summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/access_requests.rb2
-rw-r--r--lib/api/admin/ci/variables.rb2
-rw-r--r--lib/api/admin/instance_clusters.rb4
-rw-r--r--lib/api/admin/sidekiq.rb2
-rw-r--r--lib/api/api.rb10
-rw-r--r--lib/api/api_guard.rb39
-rw-r--r--lib/api/appearance.rb2
-rw-r--r--lib/api/applications.rb2
-rw-r--r--lib/api/avatar.rb2
-rw-r--r--lib/api/award_emoji.rb2
-rw-r--r--lib/api/badges.rb2
-rw-r--r--lib/api/base.rb6
-rw-r--r--lib/api/boards.rb2
-rw-r--r--lib/api/boards_responses.rb2
-rw-r--r--lib/api/branches.rb2
-rw-r--r--lib/api/broadcast_messages.rb2
-rw-r--r--lib/api/ci/pipeline_schedules.rb2
-rw-r--r--lib/api/ci/pipelines.rb6
-rw-r--r--lib/api/ci/runner.rb6
-rw-r--r--lib/api/ci/runners.rb2
-rw-r--r--lib/api/commit_statuses.rb2
-rw-r--r--lib/api/commits.rb18
-rw-r--r--lib/api/composer_packages.rb8
-rw-r--r--lib/api/conan_instance_packages.rb2
-rw-r--r--lib/api/conan_package_endpoints.rb2
-rw-r--r--lib/api/conan_project_packages.rb2
-rw-r--r--lib/api/container_registry_event.rb2
-rw-r--r--lib/api/debian_group_packages.rb21
-rw-r--r--lib/api/debian_package_endpoints.rb124
-rw-r--r--lib/api/debian_project_packages.rb56
-rw-r--r--lib/api/deploy_keys.rb2
-rw-r--r--lib/api/deploy_tokens.rb2
-rw-r--r--lib/api/deployments.rb2
-rw-r--r--lib/api/discussions.rb2
-rw-r--r--lib/api/entities/ci/lint/result.rb16
-rw-r--r--lib/api/entities/cluster.rb2
-rw-r--r--lib/api/entities/container_registry.rb1
-rw-r--r--lib/api/entities/feature_flag.rb16
-rw-r--r--lib/api/entities/feature_flag/detailed_legacy_scope.rb11
-rw-r--r--lib/api/entities/feature_flag/legacy_scope.rb16
-rw-r--r--lib/api/entities/feature_flag/scope.rb12
-rw-r--r--lib/api/entities/feature_flag/strategy.rb14
-rw-r--r--lib/api/entities/feature_flag/user_list.rb27
-rw-r--r--lib/api/entities/job_request/cache.rb2
-rw-r--r--lib/api/entities/member.rb1
-rw-r--r--lib/api/entities/package.rb14
-rw-r--r--lib/api/entities/project.rb1
-rw-r--r--lib/api/entities/snippet.rb6
-rw-r--r--lib/api/entities/unleash_feature.rb32
-rw-r--r--lib/api/entities/unleash_gitlab_user_list_strategy.rb14
-rw-r--r--lib/api/entities/unleash_legacy_strategy.rb14
-rw-r--r--lib/api/entities/unleash_strategy.rb10
-rw-r--r--lib/api/entities/user_with_admin.rb2
-rw-r--r--lib/api/environments.rb2
-rw-r--r--lib/api/error_tracking.rb2
-rw-r--r--lib/api/events.rb2
-rw-r--r--lib/api/feature_flag_scopes.rb158
-rw-r--r--lib/api/feature_flags.rb266
-rw-r--r--lib/api/feature_flags_user_lists.rb100
-rw-r--r--lib/api/features.rb2
-rw-r--r--lib/api/files.rb2
-rw-r--r--lib/api/freeze_periods.rb2
-rw-r--r--lib/api/generic_packages.rb90
-rw-r--r--lib/api/github/entities.rb4
-rwxr-xr-xlib/api/go_proxy.rb2
-rw-r--r--lib/api/group_boards.rb2
-rw-r--r--lib/api/group_clusters.rb4
-rw-r--r--lib/api/group_container_repositories.rb6
-rw-r--r--lib/api/group_export.rb2
-rw-r--r--lib/api/group_import.rb2
-rw-r--r--lib/api/group_labels.rb2
-rw-r--r--lib/api/group_milestones.rb2
-rw-r--r--lib/api/group_packages.rb2
-rw-r--r--lib/api/group_variables.rb2
-rw-r--r--lib/api/groups.rb22
-rw-r--r--lib/api/helpers.rb3
-rw-r--r--lib/api/helpers/groups_helpers.rb1
-rw-r--r--lib/api/helpers/merge_requests_helpers.rb7
-rw-r--r--lib/api/helpers/packages/conan/api_helpers.rb4
-rw-r--r--lib/api/helpers/packages/dependency_proxy_helpers.rb1
-rw-r--r--lib/api/helpers/packages_helpers.rb5
-rw-r--r--lib/api/helpers/pagination.rb4
-rw-r--r--lib/api/helpers/presentable.rb2
-rw-r--r--lib/api/helpers/projects_helpers.rb10
-rw-r--r--lib/api/helpers/runner.rb4
-rw-r--r--lib/api/helpers/services_helpers.rb6
-rw-r--r--lib/api/helpers/settings_helpers.rb1
-rw-r--r--lib/api/helpers/snippets_helpers.rb2
-rw-r--r--lib/api/import_bitbucket_server.rb2
-rw-r--r--lib/api/import_github.rb2
-rw-r--r--lib/api/internal/base.rb92
-rw-r--r--lib/api/internal/kubernetes.rb2
-rw-r--r--lib/api/internal/lfs.rb54
-rw-r--r--lib/api/internal/pages.rb2
-rw-r--r--lib/api/issue_links.rb2
-rw-r--r--lib/api/issues.rb7
-rw-r--r--lib/api/job_artifacts.rb2
-rw-r--r--lib/api/jobs.rb2
-rw-r--r--lib/api/keys.rb2
-rw-r--r--lib/api/labels.rb2
-rw-r--r--lib/api/lint.rb38
-rw-r--r--lib/api/markdown.rb2
-rw-r--r--lib/api/maven_packages.rb18
-rw-r--r--lib/api/members.rb32
-rw-r--r--lib/api/merge_request_approvals.rb2
-rw-r--r--lib/api/merge_request_diffs.rb2
-rw-r--r--lib/api/merge_requests.rb2
-rw-r--r--lib/api/metrics/dashboard/annotations.rb2
-rw-r--r--lib/api/metrics/user_starred_dashboards.rb2
-rw-r--r--lib/api/namespaces.rb6
-rw-r--r--lib/api/notes.rb2
-rw-r--r--lib/api/notification_settings.rb2
-rw-r--r--lib/api/npm_packages.rb6
-rw-r--r--lib/api/nuget_packages.rb12
-rw-r--r--lib/api/package_files.rb2
-rw-r--r--lib/api/pages.rb2
-rw-r--r--lib/api/pages_domains.rb2
-rw-r--r--lib/api/project_clusters.rb4
-rw-r--r--lib/api/project_container_repositories.rb14
-rw-r--r--lib/api/project_events.rb2
-rw-r--r--lib/api/project_export.rb4
-rw-r--r--lib/api/project_hooks.rb6
-rw-r--r--lib/api/project_import.rb4
-rw-r--r--lib/api/project_milestones.rb2
-rw-r--r--lib/api/project_packages.rb2
-rw-r--r--lib/api/project_repository_storage_moves.rb4
-rw-r--r--lib/api/project_snapshots.rb2
-rw-r--r--lib/api/project_snippets.rb14
-rw-r--r--lib/api/project_statistics.rb2
-rw-r--r--lib/api/project_templates.rb2
-rw-r--r--lib/api/projects.rb4
-rw-r--r--lib/api/protected_branches.rb2
-rw-r--r--lib/api/protected_tags.rb2
-rw-r--r--lib/api/pypi_packages.rb10
-rw-r--r--lib/api/release/links.rb2
-rw-r--r--lib/api/releases.rb16
-rw-r--r--lib/api/remote_mirrors.rb2
-rw-r--r--lib/api/repositories.rb2
-rw-r--r--lib/api/resource_label_events.rb2
-rw-r--r--lib/api/resource_milestone_events.rb2
-rw-r--r--lib/api/resource_state_events.rb2
-rw-r--r--lib/api/search.rb16
-rw-r--r--lib/api/services.rb2
-rw-r--r--lib/api/settings.rb11
-rw-r--r--lib/api/sidekiq_metrics.rb2
-rw-r--r--lib/api/snippets.rb16
-rw-r--r--lib/api/statistics.rb2
-rw-r--r--lib/api/submodules.rb2
-rw-r--r--lib/api/subscriptions.rb2
-rw-r--r--lib/api/suggestions.rb2
-rw-r--r--lib/api/system_hooks.rb6
-rw-r--r--lib/api/tags.rb2
-rw-r--r--lib/api/templates.rb2
-rw-r--r--lib/api/terraform/state.rb2
-rw-r--r--lib/api/terraform/state_version.rb68
-rw-r--r--lib/api/todos.rb2
-rw-r--r--lib/api/triggers.rb2
-rw-r--r--lib/api/unleash.rb77
-rw-r--r--lib/api/usage_data.rb4
-rw-r--r--lib/api/user_counts.rb2
-rw-r--r--lib/api/users.rb33
-rw-r--r--lib/api/v3/github.rb4
-rw-r--r--lib/api/variables.rb2
-rw-r--r--lib/api/version.rb2
-rw-r--r--lib/api/wikis.rb2
-rw-r--r--lib/backup/artifacts.rb4
-rw-r--r--lib/backup/builds.rb4
-rw-r--r--lib/backup/lfs.rb4
-rw-r--r--lib/backup/pages.rb4
-rw-r--r--lib/backup/registry.rb4
-rw-r--r--lib/backup/repositories.rb310
-rw-r--r--lib/backup/repository.rb265
-rw-r--r--lib/backup/uploads.rb4
-rw-r--r--lib/banzai/filter/design_reference_filter.rb14
-rw-r--r--lib/banzai/reference_parser.rb4
-rw-r--r--lib/banzai/reference_parser/mentioned_group_parser.rb2
-rw-r--r--lib/banzai/reference_redactor.rb1
-rw-r--r--lib/feature.rb2
-rw-r--r--lib/feature/definition.rb33
-rw-r--r--lib/feature/shared.rb12
-rw-r--r--lib/gitlab/alert_management/alert_params.rb46
-rw-r--r--lib/gitlab/alert_management/alert_status_counts.rb10
-rw-r--r--lib/gitlab/alert_management/payload/base.rb22
-rw-r--r--lib/gitlab/alert_management/payload/generic.rb4
-rw-r--r--lib/gitlab/alerting/alert.rb215
-rw-r--r--lib/gitlab/alerting/alert_annotation.rb11
-rw-r--r--lib/gitlab/alerting/notification_payload_parser.rb107
-rw-r--r--lib/gitlab/application_context.rb4
-rw-r--r--lib/gitlab/auth.rb2
-rw-r--r--lib/gitlab/auth/auth_finders.rb2
-rw-r--r--lib/gitlab/auth/otp/strategies/base.rb32
-rw-r--r--lib/gitlab/auth/otp/strategies/devise.rb15
-rw-r--r--lib/gitlab/auth/otp/strategies/forti_authenticator.rb41
-rw-r--r--lib/gitlab/auth/user_access_denied_reason.rb4
-rw-r--r--lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb73
-rw-r--r--lib/gitlab/background_migration/backfill_snippet_repositories.rb2
-rw-r--r--lib/gitlab/background_migration/migrate_u2f_webauthn.rb46
-rw-r--r--lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb2
-rw-r--r--lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb13
-rw-r--r--lib/gitlab/background_migration/replace_blocked_by_links.rb29
-rw-r--r--lib/gitlab/background_migration/sync_blocking_issues_count.rb13
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb25
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb25
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb30
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb5
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb74
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/group.rb91
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/namespace.rb33
-rw-r--r--lib/gitlab/bulk_import/client.rb72
-rw-r--r--lib/gitlab/checks/matching_merge_request.rb2
-rw-r--r--lib/gitlab/checks/post_push_message.rb6
-rw-r--r--lib/gitlab/ci/ansi2json/converter.rb23
-rw-r--r--lib/gitlab/ci/ansi2json/line.rb7
-rw-r--r--lib/gitlab/ci/ansi2json/state.rb3
-rw-r--r--lib/gitlab/ci/artifact_file_reader.rb27
-rw-r--r--lib/gitlab/ci/config.rb4
-rw-r--r--lib/gitlab/ci/config/entry/bridge.rb19
-rw-r--r--lib/gitlab/ci/config/entry/cache.rb24
-rw-r--r--lib/gitlab/ci/config/entry/include.rb6
-rw-r--r--lib/gitlab/ci/config/entry/jobs.rb29
-rw-r--r--lib/gitlab/ci/config/entry/needs.rb23
-rw-r--r--lib/gitlab/ci/config/entry/ports.rb26
-rw-r--r--lib/gitlab/ci/config/entry/product/matrix.rb2
-rw-r--r--lib/gitlab/ci/config/entry/product/variables.rb6
-rw-r--r--lib/gitlab/ci/config/entry/reports.rb3
-rw-r--r--lib/gitlab/ci/config/entry/rules.rb21
-rw-r--r--lib/gitlab/ci/config/entry/services.rb26
-rw-r--r--lib/gitlab/ci/config/entry/variables.rb22
-rw-r--r--lib/gitlab/ci/features.rb30
-rw-r--r--lib/gitlab/ci/lint.rb7
-rw-r--r--lib/gitlab/ci/parsers/test/junit.rb34
-rw-r--r--lib/gitlab/ci/pipeline/chain/command.rb2
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/process.rb2
-rw-r--r--lib/gitlab/ci/pipeline/chain/create.rb2
-rw-r--r--lib/gitlab/ci/pipeline/seed/build/cache.rb4
-rw-r--r--lib/gitlab/ci/reports/test_case.rb9
-rw-r--r--lib/gitlab/ci/reports/test_suite.rb16
-rw-r--r--lib/gitlab/ci/runner/backoff.rb75
-rw-r--r--lib/gitlab/ci/status/bridge/action.rb12
-rw-r--r--lib/gitlab/ci/status/bridge/common.rb1
-rw-r--r--lib/gitlab/ci/status/bridge/factory.rb5
-rw-r--r--lib/gitlab/ci/status/bridge/manual.rb12
-rw-r--r--lib/gitlab/ci/status/bridge/play.rb19
-rw-r--r--lib/gitlab/ci/status/canceled.rb4
-rw-r--r--lib/gitlab/ci/status/created.rb4
-rw-r--r--lib/gitlab/ci/status/failed.rb4
-rw-r--r--lib/gitlab/ci/status/manual.rb4
-rw-r--r--lib/gitlab/ci/status/pending.rb4
-rw-r--r--lib/gitlab/ci/status/preparing.rb4
-rw-r--r--lib/gitlab/ci/status/running.rb4
-rw-r--r--lib/gitlab/ci/status/scheduled.rb4
-rw-r--r--lib/gitlab/ci/status/skipped.rb4
-rw-r--r--lib/gitlab/ci/status/success.rb4
-rw-r--r--lib/gitlab/ci/status/waiting_for_resource.rb4
-rw-r--r--lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml11
-rw-r--r--lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Bash.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Clojure.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Crystal.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Django.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Elixir.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml14
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml16
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml3
-rw-r--r--lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml9
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml39
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/templates/Laravel.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/PHP.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Ruby.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Rust.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml43
-rw-r--r--lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml22
-rw-r--r--lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml53
-rw-r--r--lib/gitlab/ci/trace.rb13
-rw-r--r--lib/gitlab/ci/trace/checksum.rb83
-rw-r--r--lib/gitlab/ci/trace/metrics.rb33
-rw-r--r--lib/gitlab/ci/variables/collection/item.rb4
-rw-r--r--lib/gitlab/ci/yaml_processor/result.rb8
-rw-r--r--lib/gitlab/cleanup/orphan_lfs_file_references.rb5
-rw-r--r--lib/gitlab/cluster/puma_worker_killer_initializer.rb4
-rw-r--r--lib/gitlab/code_navigation_path.rb1
-rw-r--r--lib/gitlab/config/entry/composable_array.rb58
-rw-r--r--lib/gitlab/config/entry/composable_hash.rb47
-rw-r--r--lib/gitlab/config/entry/factory.rb2
-rw-r--r--lib/gitlab/config/entry/legacy_validation_helpers.rb14
-rw-r--r--lib/gitlab/config/entry/simplifiable.rb2
-rw-r--r--lib/gitlab/config/entry/validators.rb8
-rw-r--r--lib/gitlab/conflict/file_collection.rb8
-rw-r--r--lib/gitlab/danger/commit_linter.rb4
-rw-r--r--lib/gitlab/danger/helper.rb11
-rw-r--r--lib/gitlab/danger/roulette.rb15
-rw-r--r--lib/gitlab/danger/teammate.rb7
-rw-r--r--lib/gitlab/data_builder/deployment.rb4
-rw-r--r--lib/gitlab/database.rb40
-rw-r--r--lib/gitlab/database/batch_count.rb48
-rw-r--r--lib/gitlab/database/bulk_update.rb168
-rw-r--r--lib/gitlab/database/concurrent_reindex.rb143
-rw-r--r--lib/gitlab/database/count/reltuples_count_strategy.rb3
-rw-r--r--lib/gitlab/database/migration_helpers.rb196
-rw-r--r--lib/gitlab/database/partitioning/partition_creator.rb4
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb2
-rw-r--r--lib/gitlab/database/postgres_index.rb31
-rw-r--r--lib/gitlab/database/reindexing.rb18
-rw-r--r--lib/gitlab/database/reindexing/concurrent_reindex.rb127
-rw-r--r--lib/gitlab/database/reindexing/coordinator.rb41
-rw-r--r--lib/gitlab/database/reindexing/reindex_action.rb35
-rw-r--r--lib/gitlab/database/schema_helpers.rb17
-rw-r--r--lib/gitlab/database/similarity_score.rb11
-rw-r--r--lib/gitlab/database/with_lock_retries.rb8
-rw-r--r--lib/gitlab/design_management/copy_design_collection_model_attributes.yml43
-rw-r--r--lib/gitlab/diff/file_collection/merge_request_diff_base.rb9
-rw-r--r--lib/gitlab/diff/highlight_cache.rb47
-rw-r--r--lib/gitlab/exclusive_lease_helpers.rb2
-rw-r--r--lib/gitlab/experimentation.rb46
-rw-r--r--lib/gitlab/git/diff.rb12
-rw-r--r--lib/gitlab/git/diff_collection.rb34
-rw-r--r--lib/gitlab/git/diff_stats_collection.rb4
-rw-r--r--lib/gitlab/git/repository.rb19
-rw-r--r--lib/gitlab/git/wiki.rb10
-rw-r--r--lib/gitlab/git_access.rb36
-rw-r--r--lib/gitlab/git_access_result/custom_action.rb2
-rw-r--r--lib/gitlab/git_access_snippet.rb25
-rw-r--r--lib/gitlab/git_access_wiki.rb10
-rw-r--r--lib/gitlab/gitaly_client/diff_stitcher.rb10
-rw-r--r--lib/gitlab/gitaly_client/operation_service.rb5
-rw-r--r--lib/gitlab/gitpod.rb8
-rw-r--r--lib/gitlab/gl_repository.rb10
-rw-r--r--lib/gitlab/gl_repository/identifier.rb13
-rw-r--r--lib/gitlab/gl_repository/repo_type.rb6
-rw-r--r--lib/gitlab/gon_helper.rb10
-rw-r--r--lib/gitlab/graphql/authorize/authorize_resource.rb4
-rw-r--r--lib/gitlab/graphql/global_id_compatibility.rb20
-rw-r--r--lib/gitlab/graphql/markdown_field.rb8
-rw-r--r--lib/gitlab/graphql/markdown_field/resolver.rb22
-rw-r--r--lib/gitlab/graphql/pagination/keyset/connection.rb3
-rw-r--r--lib/gitlab/graphql/pagination/keyset/last_items.rb57
-rw-r--r--lib/gitlab/graphql/pagination/keyset/order_info.rb16
-rw-r--r--lib/gitlab/graphql/query_analyzers/logger_analyzer.rb11
-rw-r--r--lib/gitlab/group_search_results.rb6
-rw-r--r--lib/gitlab/health_checks/unicorn_check.rb2
-rw-r--r--lib/gitlab/import_export/attribute_cleaner.rb4
-rw-r--r--lib/gitlab/import_export/base/relation_factory.rb30
-rw-r--r--lib/gitlab/import_export/file_importer.rb4
-rw-r--r--lib/gitlab/import_export/json/ndjson_reader.rb6
-rw-r--r--lib/gitlab/import_export/lfs_saver.rb8
-rw-r--r--lib/gitlab/import_export/members_mapper.rb6
-rw-r--r--lib/gitlab/import_export/project/import_export.yml6
-rw-r--r--lib/gitlab/import_export/project/sample/date_calculator.rb37
-rw-r--r--lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb51
-rw-r--r--lib/gitlab/import_export/project/tree_restorer.rb10
-rw-r--r--lib/gitlab/import_export/project/tree_saver.rb2
-rw-r--r--lib/gitlab/import_export/repo_restorer.rb14
-rw-r--r--lib/gitlab/import_export/saver.rb4
-rw-r--r--lib/gitlab/import_export/uploads_manager.rb2
-rw-r--r--lib/gitlab/import_export/version_checker.rb4
-rw-r--r--lib/gitlab/instrumentation/redis.rb2
-rw-r--r--lib/gitlab/issuables_count_for_state.rb57
-rw-r--r--lib/gitlab/job_waiter.rb17
-rw-r--r--lib/gitlab/kubernetes/kube_client.rb15
-rw-r--r--lib/gitlab/kubernetes/pod.rb36
-rw-r--r--lib/gitlab/lfs/client.rb44
-rw-r--r--lib/gitlab/manifest_import/metadata.rb49
-rw-r--r--lib/gitlab/marginalia.rb10
-rw-r--r--lib/gitlab/marginalia/active_record_instrumentation.rb2
-rw-r--r--lib/gitlab/markdown_cache.rb2
-rw-r--r--lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb49
-rw-r--r--lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb24
-rw-r--r--lib/gitlab/metrics/dashboard/stages/url_validator.rb2
-rw-r--r--lib/gitlab/metrics/dashboard/transformers/errors.rb6
-rw-r--r--lib/gitlab/metrics/requests_rack_middleware.rb43
-rw-r--r--lib/gitlab/metrics/samplers/unicorn_sampler.rb2
-rw-r--r--lib/gitlab/metrics/subscribers/active_record.rb2
-rw-r--r--lib/gitlab/middleware/go.rb9
-rw-r--r--lib/gitlab/middleware/handle_null_bytes.rb61
-rw-r--r--lib/gitlab/middleware/multipart.rb3
-rw-r--r--lib/gitlab/pagination/offset_pagination.rb10
-rw-r--r--lib/gitlab/project_search_results.rb4
-rw-r--r--lib/gitlab/project_template.rb58
-rw-r--r--lib/gitlab/quick_actions/issuable_actions.rb12
-rw-r--r--lib/gitlab/quick_actions/merge_request_actions.rb14
-rw-r--r--lib/gitlab/redis/hll.rb20
-rw-r--r--lib/gitlab/regex.rb90
-rw-r--r--lib/gitlab/relative_positioning/item_context.rb4
-rw-r--r--lib/gitlab/repo_path.rb15
-rw-r--r--lib/gitlab/repository_size_checker.rb11
-rw-r--r--lib/gitlab/sample_data_template.rb22
-rw-r--r--lib/gitlab/search/recent_items.rb21
-rw-r--r--lib/gitlab/search_results.rb67
-rw-r--r--lib/gitlab/seeder.rb11
-rw-r--r--lib/gitlab/sidekiq_daemon/memory_killer.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/worker_context/server.rb6
-rw-r--r--lib/gitlab/slash_commands/presenters/help.rb2
-rw-r--r--lib/gitlab/static_site_editor/config/file_config.rb35
-rw-r--r--lib/gitlab/static_site_editor/config/file_config/entry/global.rb39
-rw-r--r--lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb26
-rw-r--r--lib/gitlab/static_site_editor/config/file_config/entry/mount.rb39
-rw-r--r--lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb33
-rw-r--r--lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb26
-rw-r--r--lib/gitlab/static_site_editor/config/generated_config.rb12
-rw-r--r--lib/gitlab/subscription_portal.rb11
-rw-r--r--lib/gitlab/template/gitlab_ci_yml_template.rb2
-rw-r--r--lib/gitlab/themes.rb24
-rw-r--r--lib/gitlab/usage_data.rb104
-rw-r--r--lib/gitlab/usage_data_counters/hll_redis_counter.rb3
-rw-r--r--lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb124
-rw-r--r--lib/gitlab/usage_data_counters/known_events.yml97
-rw-r--r--lib/gitlab/usage_data_counters/static_site_editor_counter.rb16
-rw-r--r--lib/gitlab/usage_data_queries.rb2
-rw-r--r--lib/gitlab/utils/usage_data.rb5
-rw-r--r--lib/gitlab/view/presenter/factory.rb2
-rw-r--r--lib/gitlab/visibility_level_checker.rb2
-rw-r--r--lib/gitlab/webpack/dev_server_middleware.rb9
-rw-r--r--lib/gitlab/webpack/manifest.rb108
-rw-r--r--lib/gitlab/whats_new.rb28
-rw-r--r--lib/gitlab/workhorse.rb3
-rw-r--r--lib/gitlab_danger.rb3
-rw-r--r--lib/google_api/cloud_platform/client.rb2
-rw-r--r--lib/grafana/client.rb4
-rw-r--r--lib/pager_duty/validator/schemas/message.json47
-rw-r--r--lib/pager_duty/webhook_payload_parser.rb12
-rw-r--r--lib/peek/views/active_record.rb25
-rw-r--r--lib/peek/views/detailed_view.rb2
-rw-r--r--lib/safe_zip/extract.rb22
-rw-r--r--lib/sentry/client/issue.rb4
-rw-r--r--lib/system_check/app/redis_version_check.rb2
-rw-r--r--lib/tasks/gettext.rake2
-rw-r--r--lib/tasks/gitlab/assets.rake2
-rw-r--r--lib/tasks/gitlab/backup.rake4
-rw-r--r--lib/tasks/gitlab/db.rake34
-rw-r--r--lib/tasks/gitlab/web_hook.rake5
-rw-r--r--lib/uploaded_file.rb7
441 files changed, 6244 insertions, 1893 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb
index 5305b25538f..7e3d70a210a 100644
--- a/lib/api/access_requests.rb
+++ b/lib/api/access_requests.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class AccessRequests < Grape::API::Instance
+ class AccessRequests < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb
index 8721d94d642..44c389d6f94 100644
--- a/lib/api/admin/ci/variables.rb
+++ b/lib/api/admin/ci/variables.rb
@@ -3,7 +3,7 @@
module API
module Admin
module Ci
- class Variables < Grape::API::Instance
+ class Variables < ::API::Base
include PaginationParams
before { authenticated_as_admin! }
diff --git a/lib/api/admin/instance_clusters.rb b/lib/api/admin/instance_clusters.rb
index 8208d10c089..ce1bdd65eff 100644
--- a/lib/api/admin/instance_clusters.rb
+++ b/lib/api/admin/instance_clusters.rb
@@ -2,7 +2,7 @@
module API
module Admin
- class InstanceClusters < Grape::API::Instance
+ class InstanceClusters < ::API::Base
include PaginationParams
before do
@@ -37,6 +37,7 @@ module API
requires :name, type: String, desc: 'Cluster name'
optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true'
optional :environment_scope, default: '*', type: String, desc: 'The associated environment to the cluster'
+ optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace'
optional :domain, type: String, desc: 'Cluster base domain'
optional :management_project_id, type: Integer, desc: 'The ID of the management project'
optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true'
@@ -70,6 +71,7 @@ module API
optional :name, type: String, desc: 'Cluster name'
optional :enabled, type: Boolean, desc: 'Enable or disable Gitlab\'s connection to your Kubernetes cluster'
optional :environment_scope, type: String, desc: 'The associated environment to the cluster'
+ optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace'
optional :domain, type: String, desc: 'Cluster base domain'
optional :management_project_id, type: Integer, desc: 'The ID of the management project'
optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do
diff --git a/lib/api/admin/sidekiq.rb b/lib/api/admin/sidekiq.rb
index f4c84f2eee8..c2e9de5fb4e 100644
--- a/lib/api/admin/sidekiq.rb
+++ b/lib/api/admin/sidekiq.rb
@@ -2,7 +2,7 @@
module API
module Admin
- class Sidekiq < Grape::API::Instance
+ class Sidekiq < ::API::Base
before { authenticated_as_admin! }
namespace 'admin' do
diff --git a/lib/api/api.rb b/lib/api/api.rb
index b37751e1b47..84b4d5a5835 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class API < Grape::API::Instance
+ class API < ::API::Base
include APIGuard
LOG_FILENAME = Rails.root.join("log", "api_json.log")
@@ -153,6 +153,9 @@ module API
mount ::API::Environments
mount ::API::ErrorTracking
mount ::API::Events
+ mount ::API::FeatureFlags
+ mount ::API::FeatureFlagScopes
+ mount ::API::FeatureFlagsUserLists
mount ::API::Features
mount ::API::Files
mount ::API::FreezePeriods
@@ -196,6 +199,8 @@ module API
mount ::API::ComposerPackages
mount ::API::ConanProjectPackages
mount ::API::ConanInstancePackages
+ mount ::API::DebianGroupPackages
+ mount ::API::DebianProjectPackages
mount ::API::MavenPackages
mount ::API::NpmPackages
mount ::API::GenericPackages
@@ -216,6 +221,7 @@ module API
mount ::API::ProjectStatistics
mount ::API::ProjectTemplates
mount ::API::Terraform::State
+ mount ::API::Terraform::StateVersion
mount ::API::ProtectedBranches
mount ::API::ProtectedTags
mount ::API::Releases
@@ -236,6 +242,7 @@ module API
mount ::API::Templates
mount ::API::Todos
mount ::API::Triggers
+ mount ::API::Unleash
mount ::API::UsageData
mount ::API::UserCounts
mount ::API::Users
@@ -245,6 +252,7 @@ module API
end
mount ::API::Internal::Base
+ mount ::API::Internal::Lfs
mount ::API::Internal::Pages
mount ::API::Internal::Kubernetes
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index bf5044b4832..0a486307653 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -19,6 +19,7 @@ module API
end
use AdminModeMiddleware
+ use ResponseCoercerMiddleware
helpers HelperMethods
@@ -188,6 +189,44 @@ module API
end
end
+ # Prior to Rack v2.1.x, returning a body of [nil] or [201] worked
+ # because the body was coerced to a string. However, this no longer
+ # works in Rack v2.1.0+. The Rack spec
+ # (https://github.com/rack/rack/blob/master/SPEC.rdoc#the-body-)
+ # says:
+ #
+ # The Body must respond to `each` and must only yield String values
+ #
+ # Because it's easy to return the wrong body type, this middleware
+ # will:
+ #
+ # 1. Inspect each element of the body if it is an Array.
+ # 2. Coerce each value to a string if necessary.
+ # 3. Flag a test and development error.
+ class ResponseCoercerMiddleware < ::Grape::Middleware::Base
+ def call(env)
+ response = super(env)
+
+ status = response[0]
+ body = response[2]
+
+ return response if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY[status]
+ return response unless body.is_a?(Array)
+
+ body.map! do |part|
+ if part.is_a?(String)
+ part
+ else
+ err = ArgumentError.new("The response body should be a String, but it is of type #{part.class}")
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(err)
+ part.to_s
+ end
+ end
+
+ response
+ end
+ end
+
class AdminModeMiddleware < ::Grape::Middleware::Base
def after
# Use a Grape middleware since the Grape `after` blocks might run
diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb
index f98004af480..00b495bbc1e 100644
--- a/lib/api/appearance.rb
+++ b/lib/api/appearance.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Appearance < Grape::API::Instance
+ class Appearance < ::API::Base
before { authenticated_as_admin! }
helpers do
diff --git a/lib/api/applications.rb b/lib/api/applications.rb
index 4f2c3ee79ef..2afe8763d9d 100644
--- a/lib/api/applications.rb
+++ b/lib/api/applications.rb
@@ -2,7 +2,7 @@
module API
# External applications API
- class Applications < Grape::API::Instance
+ class Applications < ::API::Base
before { authenticated_as_admin! }
resource :applications do
diff --git a/lib/api/avatar.rb b/lib/api/avatar.rb
index 9501e777fff..5a9b9940fcf 100644
--- a/lib/api/avatar.rb
+++ b/lib/api/avatar.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Avatar < Grape::API::Instance
+ class Avatar < ::API::Base
resource :avatar do
desc 'Return avatar url for a user' do
success Entities::Avatar
diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb
index 0a3df3ed96e..6d40ae8f5ff 100644
--- a/lib/api/award_emoji.rb
+++ b/lib/api/award_emoji.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class AwardEmoji < Grape::API::Instance
+ class AwardEmoji < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/badges.rb b/lib/api/badges.rb
index f9728ffc446..fc00594c9ec 100644
--- a/lib/api/badges.rb
+++ b/lib/api/badges.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Badges < Grape::API::Instance
+ class Badges < ::API::Base
include PaginationParams
before { authenticate_non_get! }
diff --git a/lib/api/base.rb b/lib/api/base.rb
new file mode 100644
index 00000000000..e174cef3bad
--- /dev/null
+++ b/lib/api/base.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module API
+ class Base < Grape::API::Instance # rubocop:disable API/Base
+ end
+end
diff --git a/lib/api/boards.rb b/lib/api/boards.rb
index 1f5086127a8..d2d1628aff4 100644
--- a/lib/api/boards.rb
+++ b/lib/api/boards.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Boards < Grape::API::Instance
+ class Boards < ::API::Base
include BoardsResponses
include PaginationParams
diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb
index 68497a08fb8..6a86c02bf4a 100644
--- a/lib/api/boards_responses.rb
+++ b/lib/api/boards_responses.rb
@@ -45,7 +45,7 @@ module API
def destroy_list(list)
destroy_conditionally!(list) do |list|
service = ::Boards::Lists::DestroyService.new(board_parent, current_user)
- unless service.execute(list)
+ if service.execute(list).error?
render_api_error!({ error: 'List could not be deleted!' }, 400)
end
end
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 44f7610384e..37cce6eafba 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -3,7 +3,7 @@
require 'mime/types'
module API
- class Branches < Grape::API::Instance
+ class Branches < ::API::Base
include PaginationParams
BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX)
diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb
index dcf950d7a03..8ce7694bbfd 100644
--- a/lib/api/broadcast_messages.rb
+++ b/lib/api/broadcast_messages.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class BroadcastMessages < Grape::API::Instance
+ class BroadcastMessages < ::API::Base
include PaginationParams
resource :broadcast_messages do
diff --git a/lib/api/ci/pipeline_schedules.rb b/lib/api/ci/pipeline_schedules.rb
index 1afdb0ad34c..18caf85f109 100644
--- a/lib/api/ci/pipeline_schedules.rb
+++ b/lib/api/ci/pipeline_schedules.rb
@@ -2,7 +2,7 @@
module API
module Ci
- class PipelineSchedules < Grape::API::Instance
+ class PipelineSchedules < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb
index 045f81074a7..61e03ed1a95 100644
--- a/lib/api/ci/pipelines.rb
+++ b/lib/api/ci/pipelines.rb
@@ -2,7 +2,7 @@
module API
module Ci
- class Pipelines < Grape::API::Instance
+ class Pipelines < ::API::Base
include PaginationParams
before { authenticate_non_get! }
@@ -128,7 +128,7 @@ module API
pipeline = user_project.all_pipelines.find(params[:pipeline_id])
- if Feature.enabled?(:ci_jobs_finder_refactor)
+ if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true)
builds = ::Ci::JobsFinder
.new(current_user: current_user, pipeline: pipeline, params: params)
.execute
@@ -157,7 +157,7 @@ module API
pipeline = user_project.all_pipelines.find(params[:pipeline_id])
- if Feature.enabled?(:ci_jobs_finder_refactor)
+ if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true)
bridges = ::Ci::JobsFinder
.new(current_user: current_user, pipeline: pipeline, params: params, type: ::Ci::Bridge)
.execute
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index 08903dce3dc..ef679147c9f 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -2,7 +2,7 @@
module API
module Ci
- class Runner < Grape::API::Instance
+ class Runner < ::API::Base
helpers ::API::Helpers::Runner
resource :runners do
@@ -72,6 +72,7 @@ module API
post '/verify' do
authenticate_runner!
status 200
+ body "200"
end
end
@@ -181,7 +182,9 @@ module API
.new(job, declared_params(include_missing: false))
service.execute.then do |result|
+ header 'X-GitLab-Trace-Update-Interval', result.backoff
status result.status
+ body result.status.to_s
end
end
@@ -292,6 +295,7 @@ module API
if result[:status] == :success
status :created
+ body "201"
else
render_api_error!(result[:message], result[:http_status])
end
diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb
index 7bca72f8028..d37f10fe631 100644
--- a/lib/api/ci/runners.rb
+++ b/lib/api/ci/runners.rb
@@ -2,7 +2,7 @@
module API
module Ci
- class Runners < Grape::API::Instance
+ class Runners < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb
index 9f5a6e87505..af103b8c1f8 100644
--- a/lib/api/commit_statuses.rb
+++ b/lib/api/commit_statuses.rb
@@ -3,7 +3,7 @@
require 'mime/types'
module API
- class CommitStatuses < Grape::API::Instance
+ class CommitStatuses < ::API::Base
params do
requires :id, type: String, desc: 'The ID of a project'
end
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 20877fb5c5f..582ccd41847 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -3,7 +3,7 @@
require 'mime/types'
module API
- class Commits < Grape::API::Instance
+ class Commits < ::API::Base
include PaginationParams
before do
@@ -62,19 +62,15 @@ module API
first_parent: first_parent,
order: order)
- commit_count =
- if all || path || before || after || first_parent
- user_project.repository.count_commits(ref: ref, path: path, before: before, after: after, all: all, first_parent: first_parent)
- else
- # Cacheable commit count.
- user_project.repository.commit_count_for_ref(ref)
- end
+ serializer = with_stats ? Entities::CommitWithStats : Entities::Commit
+ # This tells kaminari that there is 1 more commit after the one we've
+ # loaded, meaning there will be a next page, if the currently loaded set
+ # of commits is equal to the requested page size.
+ commit_count = offset + commits.size + 1
paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count)
- serializer = with_stats ? Entities::CommitWithStats : Entities::Commit
-
- present paginate(paginated_commits), with: serializer
+ present paginate(paginated_commits, exclude_total_headers: true), with: serializer
end
desc 'Commit multiple file changes as one commit' do
diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb
index 31d097c4bea..1becbd668a3 100644
--- a/lib/api/composer_packages.rb
+++ b/lib/api/composer_packages.rb
@@ -2,7 +2,7 @@
# PHP composer support (https://getcomposer.org/)
module API
- class ComposerPackages < Grape::API::Instance
+ class ComposerPackages < ::API::Base
helpers ::API::Helpers::PackagesManagerClientsHelpers
helpers ::API::Helpers::RelatedResourcesHelpers
helpers ::API::Helpers::Packages::BasicAuthHelpers
@@ -26,6 +26,10 @@ module API
render_api_error!(e.message, 400)
end
+ rescue_from Packages::Composer::ComposerJsonService::InvalidJson do |e|
+ render_api_error!(e.message, 422)
+ end
+
helpers do
def packages
strong_memoize(:packages) do
@@ -123,7 +127,7 @@ module API
bad_request!
end
- package_event('push_package')
+ track_package_event('push_package', :composer)
::Packages::Composer::CreatePackageService
.new(authorized_user_project, current_user, declared_params)
diff --git a/lib/api/conan_instance_packages.rb b/lib/api/conan_instance_packages.rb
index 209748d79fa..08265201328 100644
--- a/lib/api/conan_instance_packages.rb
+++ b/lib/api/conan_instance_packages.rb
@@ -2,7 +2,7 @@
# Conan Instance-Level Package Manager Client API
module API
- class ConanInstancePackages < Grape::API::Instance
+ class ConanInstancePackages < ::API::Base
namespace 'packages/conan/v1' do
include ConanPackageEndpoints
end
diff --git a/lib/api/conan_package_endpoints.rb b/lib/api/conan_package_endpoints.rb
index 445447cfcd2..9b6867a328b 100644
--- a/lib/api/conan_package_endpoints.rb
+++ b/lib/api/conan_package_endpoints.rb
@@ -246,7 +246,7 @@ module API
delete do
authorize!(:destroy_package, project)
- package_event('delete_package', category: 'API::ConanPackages')
+ track_package_event('delete_package', :conan, category: 'API::ConanPackages')
package.destroy
end
diff --git a/lib/api/conan_project_packages.rb b/lib/api/conan_project_packages.rb
index c51992231a7..db8cd187811 100644
--- a/lib/api/conan_project_packages.rb
+++ b/lib/api/conan_project_packages.rb
@@ -2,7 +2,7 @@
# Conan Project-Level Package Manager Client API
module API
- class ConanProjectPackages < Grape::API::Instance
+ class ConanProjectPackages < ::API::Base
params do
requires :id, type: Integer, desc: 'The ID of a project', regexp: %r{\A[1-9]\d*\z}
end
diff --git a/lib/api/container_registry_event.rb b/lib/api/container_registry_event.rb
index 0b7c35cadbd..6c4b80b612a 100644
--- a/lib/api/container_registry_event.rb
+++ b/lib/api/container_registry_event.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ContainerRegistryEvent < Grape::API::Instance
+ class ContainerRegistryEvent < ::API::Base
DOCKER_DISTRIBUTION_EVENTS_V1_JSON = 'application/vnd.docker.distribution.events.v1+json'
before { authenticate_registry_notification! }
diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb
new file mode 100644
index 00000000000..e3cacc4132f
--- /dev/null
+++ b/lib/api/debian_group_packages.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module API
+ class DebianGroupPackages < ::API::Base
+ params do
+ requires :id, type: String, desc: 'The ID of a group'
+ end
+
+ resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ before do
+ not_found! unless ::Feature.enabled?(:debian_packages, user_group)
+
+ authorize_read_package!(user_group)
+ end
+
+ namespace ':id/-/packages/debian' do
+ include DebianPackageEndpoints
+ end
+ end
+ end
+end
diff --git a/lib/api/debian_package_endpoints.rb b/lib/api/debian_package_endpoints.rb
new file mode 100644
index 00000000000..168b3ca7a4f
--- /dev/null
+++ b/lib/api/debian_package_endpoints.rb
@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module API
+ module DebianPackageEndpoints
+ extend ActiveSupport::Concern
+
+ DISTRIBUTION_REGEX = %r{[a-zA-Z0-9][a-zA-Z0-9.-]*}.freeze
+ COMPONENT_REGEX = %r{[a-z-]+}.freeze
+ ARCHITECTURE_REGEX = %r{[a-z][a-z0-9]*}.freeze
+ LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze
+ PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX
+ DISTRIBUTION_REQUIREMENTS = {
+ distribution: DISTRIBUTION_REGEX
+ }.freeze
+ COMPONENT_ARCHITECTURE_REQUIREMENTS = {
+ component: COMPONENT_REGEX,
+ architecture: ARCHITECTURE_REGEX
+ }.freeze
+ COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = {
+ component: COMPONENT_REGEX,
+ letter: LETTER_REGEX,
+ source_package: PACKAGE_REGEX
+ }.freeze
+ FILE_NAME_REQUIREMENTS = {
+ file_name: API::NO_SLASH_URL_PART_REGEX
+ }.freeze
+
+ included do
+ helpers ::API::Helpers::PackagesHelpers
+ helpers ::API::Helpers::Packages::BasicAuthHelpers
+
+ format :txt
+
+ rescue_from ArgumentError do |e|
+ render_api_error!(e.message, 400)
+ end
+
+ rescue_from ActiveRecord::RecordInvalid do |e|
+ render_api_error!(e.message, 400)
+ end
+
+ before do
+ require_packages_enabled!
+ end
+
+ params do
+ requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex
+ end
+
+ namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do
+ # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release.gpg
+ desc 'The Release file signature' do
+ detail 'This feature was introduced in GitLab 13.5'
+ end
+
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+ get 'Release.gpg' do
+ not_found!
+ end
+
+ # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/Release
+ desc 'The unsigned Release file' do
+ detail 'This feature was introduced in GitLab 13.5'
+ end
+
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+ get 'Release' do
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
+ 'TODO Release'
+ end
+
+ # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/InRelease
+ desc 'The signed Release file' do
+ detail 'This feature was introduced in GitLab 13.5'
+ end
+
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+ get 'InRelease' do
+ not_found!
+ end
+
+ params do
+ requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex
+ requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex
+ end
+
+ namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do
+ # GET {projects|groups}/:id/-/packages/debian/dists/*distribution/:component/binary-:architecture/Packages
+ desc 'The binary files index' do
+ detail 'This feature was introduced in GitLab 13.5'
+ end
+
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+ get 'Packages' do
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
+ 'TODO Packages'
+ end
+ end
+ end
+
+ params do
+ requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex
+ requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)'
+ requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex
+ end
+
+ namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do
+ # GET {projects|groups}/:id/-/packages/debian/pool/:component/:letter/:source_package/:file_name
+ params do
+ requires :file_name, type: String, desc: 'The Debian File Name'
+ end
+ desc 'The package' do
+ detail 'This feature was introduced in GitLab 13.5'
+ end
+
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+ get ':file_name', requirements: FILE_NAME_REQUIREMENTS do
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286
+ 'TODO File'
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb
new file mode 100644
index 00000000000..bcb4e8c8cbc
--- /dev/null
+++ b/lib/api/debian_project_packages.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module API
+ class DebianProjectPackages < ::API::Base
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ before do
+ not_found! unless ::Feature.enabled?(:debian_packages, user_project)
+
+ authorize_read_package!
+ end
+
+ namespace ':id/-/packages/debian' do
+ include DebianPackageEndpoints
+
+ params do
+ requires :file_name, type: String, desc: 'The file name'
+ end
+
+ namespace 'incoming/:file_name', requirements: FILE_NAME_REQUIREMENTS do
+ # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name
+ params do
+ requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)'
+ end
+
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+ put do
+ authorize_upload!(authorized_user_project)
+ bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:debian_max_file_size, params[:file].size)
+
+ track_package_event('push_package', :debian)
+
+ created!
+ rescue ObjectStorage::RemoteStoreError => e
+ Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: authorized_user_project.id })
+
+ forbidden!
+ end
+
+ # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name/authorize
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+ post 'authorize' do
+ authorize_workhorse!(
+ subject: authorized_user_project,
+ has_length: false,
+ maximum_size: authorized_user_project.actual_limits.debian_max_file_size
+ )
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index ad37b7578ad..314f5b6ee1d 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class DeployKeys < Grape::API::Instance
+ class DeployKeys < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb
index 96aa2445f56..1c156b8b3bb 100644
--- a/lib/api/deploy_tokens.rb
+++ b/lib/api/deploy_tokens.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class DeployTokens < Grape::API::Instance
+ class DeployTokens < ::API::Base
include PaginationParams
helpers do
diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb
index 87144fd31cc..ff06bdbae16 100644
--- a/lib/api/deployments.rb
+++ b/lib/api/deployments.rb
@@ -2,7 +2,7 @@
module API
# Deployments RESTful API endpoints
- class Deployments < Grape::API::Instance
+ class Deployments < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb
index c431ec8e1e4..3d2608c8c5a 100644
--- a/lib/api/discussions.rb
+++ b/lib/api/discussions.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Discussions < Grape::API::Instance
+ class Discussions < ::API::Base
include PaginationParams
helpers ::API::Helpers::NotesHelpers
helpers ::RendersNotes
diff --git a/lib/api/entities/ci/lint/result.rb b/lib/api/entities/ci/lint/result.rb
new file mode 100644
index 00000000000..0e4aa238ba2
--- /dev/null
+++ b/lib/api/entities/ci/lint/result.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ module Lint
+ class Result < Grape::Entity
+ expose :valid?, as: :valid
+ expose :errors
+ expose :warnings
+ expose :merged_yaml
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/cluster.rb b/lib/api/entities/cluster.rb
index 4cb54e988ce..67459092a33 100644
--- a/lib/api/entities/cluster.rb
+++ b/lib/api/entities/cluster.rb
@@ -4,7 +4,7 @@ module API
module Entities
class Cluster < Grape::Entity
expose :id, :name, :created_at, :domain
- expose :provider_type, :platform_type, :environment_scope, :cluster_type
+ expose :provider_type, :platform_type, :environment_scope, :cluster_type, :namespace_per_environment
expose :user, using: Entities::UserBasic
expose :platform_kubernetes, using: Entities::Platform::Kubernetes
expose :provider_gcp, using: Entities::Provider::Gcp
diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb
index cff627ab50a..c430b73580b 100644
--- a/lib/api/entities/container_registry.rb
+++ b/lib/api/entities/container_registry.rb
@@ -16,6 +16,7 @@ module API
expose :project_id
expose :location
expose :created_at
+ expose :expiration_policy_started_at, as: :cleanup_policy_started_at
expose :tags_count, if: -> (_, options) { options[:tags_count] }
expose :tags, using: Tag, if: -> (_, options) { options[:tags] }
end
diff --git a/lib/api/entities/feature_flag.rb b/lib/api/entities/feature_flag.rb
new file mode 100644
index 00000000000..82fdb20af00
--- /dev/null
+++ b/lib/api/entities/feature_flag.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class FeatureFlag < Grape::Entity
+ expose :name
+ expose :description
+ expose :active
+ expose :version, if: :feature_flags_new_version_enabled
+ expose :created_at
+ expose :updated_at
+ expose :scopes, using: FeatureFlag::LegacyScope
+ expose :strategies, using: FeatureFlag::Strategy, if: :feature_flags_new_version_enabled
+ end
+ end
+end
diff --git a/lib/api/entities/feature_flag/detailed_legacy_scope.rb b/lib/api/entities/feature_flag/detailed_legacy_scope.rb
new file mode 100644
index 00000000000..47078c1dfde
--- /dev/null
+++ b/lib/api/entities/feature_flag/detailed_legacy_scope.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class FeatureFlag < Grape::Entity
+ class DetailedLegacyScope < LegacyScope
+ expose :name
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/feature_flag/legacy_scope.rb b/lib/api/entities/feature_flag/legacy_scope.rb
new file mode 100644
index 00000000000..7329f71c599
--- /dev/null
+++ b/lib/api/entities/feature_flag/legacy_scope.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class FeatureFlag < Grape::Entity
+ class LegacyScope < Grape::Entity
+ expose :id
+ expose :active
+ expose :environment_scope
+ expose :strategies
+ expose :created_at
+ expose :updated_at
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/feature_flag/scope.rb b/lib/api/entities/feature_flag/scope.rb
new file mode 100644
index 00000000000..906fe718257
--- /dev/null
+++ b/lib/api/entities/feature_flag/scope.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class FeatureFlag < Grape::Entity
+ class Scope < Grape::Entity
+ expose :id
+ expose :environment_scope
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/feature_flag/strategy.rb b/lib/api/entities/feature_flag/strategy.rb
new file mode 100644
index 00000000000..32699be0ee3
--- /dev/null
+++ b/lib/api/entities/feature_flag/strategy.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class FeatureFlag < Grape::Entity
+ class Strategy < Grape::Entity
+ expose :id
+ expose :name
+ expose :parameters
+ expose :scopes, using: FeatureFlag::Scope
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/feature_flag/user_list.rb b/lib/api/entities/feature_flag/user_list.rb
new file mode 100644
index 00000000000..bc8b12ea22e
--- /dev/null
+++ b/lib/api/entities/feature_flag/user_list.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class FeatureFlag < Grape::Entity
+ class UserList < Grape::Entity
+ include RequestAwareEntity
+
+ expose :id
+ expose :iid
+ expose :project_id
+ expose :created_at
+ expose :updated_at
+ expose :name
+ expose :user_xids
+
+ expose :path do |list|
+ project_feature_flags_user_list_path(list.project, list)
+ end
+
+ expose :edit_path do |list|
+ edit_project_feature_flags_user_list_path(list.project, list)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/job_request/cache.rb b/lib/api/entities/job_request/cache.rb
index a75affbaf84..cd533d7e5b3 100644
--- a/lib/api/entities/job_request/cache.rb
+++ b/lib/api/entities/job_request/cache.rb
@@ -4,7 +4,7 @@ module API
module Entities
module JobRequest
class Cache < Grape::Entity
- expose :key, :untracked, :paths, :policy
+ expose :key, :untracked, :paths, :policy, :when
end
end
end
diff --git a/lib/api/entities/member.rb b/lib/api/entities/member.rb
index 14e97f41e77..ad62f92e5a0 100644
--- a/lib/api/entities/member.rb
+++ b/lib/api/entities/member.rb
@@ -5,6 +5,7 @@ module API
class Member < Grape::Entity
expose :user, merge: true, using: UserBasic
expose :access_level
+ expose :created_at
expose :expires_at
end
end
diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb
index d903f50befa..b54f0e04a9d 100644
--- a/lib/api/entities/package.rb
+++ b/lib/api/entities/package.rb
@@ -7,7 +7,19 @@ module API
extend ::API::Entities::EntityHelpers
expose :id
- expose :name
+
+ expose :name do |package|
+ if package.conan?
+ package.conan_recipe
+ else
+ package.name
+ end
+ end
+
+ expose :conan_package_name, if: ->(package) { package.conan? } do |package|
+ package.name
+ end
+
expose :version
expose :package_type
diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb
index fb599d68d72..82a44c75382 100644
--- a/lib/api/entities/project.rb
+++ b/lib/api/entities/project.rb
@@ -84,6 +84,7 @@ module API
expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) }
expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
expose :ci_default_git_depth
+ expose :ci_forward_deployment_enabled
expose :public_builds, as: :public_jobs
expose :build_git_strategy, if: lambda { |project, options| options[:user_can_admin_project] } do |project, options|
project.build_allow_git_fetch ? 'fetch' : 'clone'
diff --git a/lib/api/entities/snippet.rb b/lib/api/entities/snippet.rb
index 40488eb882d..85148c03d18 100644
--- a/lib/api/entities/snippet.rb
+++ b/lib/api/entities/snippet.rb
@@ -17,7 +17,7 @@ module API
expose :file_name do |snippet|
snippet.file_name_on_repo || snippet.file_name
end
- expose :files, if: ->(snippet, options) { snippet_multiple_files?(snippet, options[:current_user]) } do |snippet, options|
+ expose :files do |snippet, options|
snippet.list_files.map do |file|
{
path: file,
@@ -25,10 +25,6 @@ module API
}
end
end
-
- def snippet_multiple_files?(snippet, current_user)
- ::Feature.enabled?(:snippet_multiple_files, current_user) && snippet.repository_exists?
- end
end
end
end
diff --git a/lib/api/entities/unleash_feature.rb b/lib/api/entities/unleash_feature.rb
new file mode 100644
index 00000000000..8ee87d1fc11
--- /dev/null
+++ b/lib/api/entities/unleash_feature.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class UnleashFeature < Grape::Entity
+ expose :name
+ expose :description, unless: ->(feature) { feature.description.nil? }
+ expose :active, as: :enabled
+ expose :strategies do |flag|
+ flag.strategies.map do |strategy|
+ if legacy_strategy?(strategy)
+ UnleashLegacyStrategy.represent(strategy)
+ elsif gitlab_user_list_strategy?(strategy)
+ UnleashGitlabUserListStrategy.represent(strategy)
+ else
+ UnleashStrategy.represent(strategy)
+ end
+ end
+ end
+
+ private
+
+ def legacy_strategy?(strategy)
+ !strategy.respond_to?(:name)
+ end
+
+ def gitlab_user_list_strategy?(strategy)
+ strategy.name == ::Operations::FeatureFlags::Strategy::STRATEGY_GITLABUSERLIST
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/unleash_gitlab_user_list_strategy.rb b/lib/api/entities/unleash_gitlab_user_list_strategy.rb
new file mode 100644
index 00000000000..5617f8002d9
--- /dev/null
+++ b/lib/api/entities/unleash_gitlab_user_list_strategy.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class UnleashGitlabUserListStrategy < Grape::Entity
+ expose :name do |_strategy|
+ ::Operations::FeatureFlags::Strategy::STRATEGY_USERWITHID
+ end
+ expose :parameters do |strategy|
+ { userIds: strategy.user_list.user_xids }
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/unleash_legacy_strategy.rb b/lib/api/entities/unleash_legacy_strategy.rb
new file mode 100644
index 00000000000..5d5954f8da0
--- /dev/null
+++ b/lib/api/entities/unleash_legacy_strategy.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class UnleashLegacyStrategy < Grape::Entity
+ expose :name do |strategy|
+ strategy['name']
+ end
+ expose :parameters do |strategy|
+ strategy['parameters']
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/unleash_strategy.rb b/lib/api/entities/unleash_strategy.rb
new file mode 100644
index 00000000000..7627ce3873c
--- /dev/null
+++ b/lib/api/entities/unleash_strategy.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class UnleashStrategy < Grape::Entity
+ expose :name
+ expose :parameters
+ end
+ end
+end
diff --git a/lib/api/entities/user_with_admin.rb b/lib/api/entities/user_with_admin.rb
index c225ade6eb6..ab7bc738ff8 100644
--- a/lib/api/entities/user_with_admin.rb
+++ b/lib/api/entities/user_with_admin.rb
@@ -8,3 +8,5 @@ module API
end
end
end
+
+API::Entities::UserWithAdmin.prepend_if_ee('EE::API::Entities::UserWithAdmin')
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
index b825904e2c5..0e780d4ef36 100644
--- a/lib/api/environments.rb
+++ b/lib/api/environments.rb
@@ -2,7 +2,7 @@
module API
# Environments RESTfull API endpoints
- class Environments < Grape::API::Instance
+ class Environments < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/error_tracking.rb b/lib/api/error_tracking.rb
index 64ec6f0a57a..03f83477954 100644
--- a/lib/api/error_tracking.rb
+++ b/lib/api/error_tracking.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ErrorTracking < Grape::API::Instance
+ class ErrorTracking < ::API::Base
before { authenticate! }
params do
diff --git a/lib/api/events.rb b/lib/api/events.rb
index 0b79431a76d..43efacf9c0b 100644
--- a/lib/api/events.rb
+++ b/lib/api/events.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Events < Grape::API::Instance
+ class Events < ::API::Base
include PaginationParams
include APIGuard
helpers ::API::Helpers::EventsHelpers
diff --git a/lib/api/feature_flag_scopes.rb b/lib/api/feature_flag_scopes.rb
new file mode 100644
index 00000000000..d77e243aa88
--- /dev/null
+++ b/lib/api/feature_flag_scopes.rb
@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module API
+ class FeatureFlagScopes < ::API::Base
+ include PaginationParams
+
+ ENVIRONMENT_SCOPE_ENDPOINT_REQUIREMENTS = FeatureFlags::FEATURE_FLAG_ENDPOINT_REQUIREMENTS
+ .merge(environment_scope: API::NO_SLASH_URL_PART_REGEX)
+
+ before do
+ authorize_read_feature_flags!
+ end
+
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+ resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ resource :feature_flag_scopes do
+ desc 'Get all effective feature flags under the environment' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag::DetailedLegacyScope
+ end
+ params do
+ requires :environment, type: String, desc: 'The environment name'
+ end
+ get do
+ present scopes_for_environment, with: ::API::Entities::FeatureFlag::DetailedLegacyScope
+ end
+ end
+
+ params do
+ requires :name, type: String, desc: 'The name of the feature flag'
+ end
+ resource 'feature_flags/:name', requirements: FeatureFlags::FEATURE_FLAG_ENDPOINT_REQUIREMENTS do
+ resource :scopes do
+ desc 'Get all scopes of a feature flag' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag::LegacyScope
+ end
+ params do
+ use :pagination
+ end
+ get do
+ present paginate(feature_flag.scopes), with: ::API::Entities::FeatureFlag::LegacyScope
+ end
+
+ desc 'Create a scope of a feature flag' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag::LegacyScope
+ end
+ params do
+ requires :environment_scope, type: String, desc: 'The environment scope of the scope'
+ requires :active, type: Boolean, desc: 'Whether the scope is active'
+ requires :strategies, type: JSON, desc: 'The strategies of the scope'
+ end
+ post do
+ authorize_update_feature_flag!
+
+ result = ::FeatureFlags::UpdateService
+ .new(user_project, current_user, scopes_attributes: [declared_params])
+ .execute(feature_flag)
+
+ if result[:status] == :success
+ present scope, with: ::API::Entities::FeatureFlag::LegacyScope
+ else
+ render_api_error!(result[:message], result[:http_status])
+ end
+ end
+
+ params do
+ requires :environment_scope, type: String, desc: 'URL-encoded environment scope'
+ end
+ resource ':environment_scope', requirements: ENVIRONMENT_SCOPE_ENDPOINT_REQUIREMENTS do
+ desc 'Get a scope of a feature flag' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag::LegacyScope
+ end
+ get do
+ present scope, with: ::API::Entities::FeatureFlag::LegacyScope
+ end
+
+ desc 'Update a scope of a feature flag' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag::LegacyScope
+ end
+ params do
+ optional :active, type: Boolean, desc: 'Whether the scope is active'
+ optional :strategies, type: JSON, desc: 'The strategies of the scope'
+ end
+ put do
+ authorize_update_feature_flag!
+
+ scope_attributes = declared_params.merge(id: scope.id)
+
+ result = ::FeatureFlags::UpdateService
+ .new(user_project, current_user, scopes_attributes: [scope_attributes])
+ .execute(feature_flag)
+
+ if result[:status] == :success
+ updated_scope = result[:feature_flag].scopes
+ .find { |scope| scope.environment_scope == params[:environment_scope] }
+
+ present updated_scope, with: ::API::Entities::FeatureFlag::LegacyScope
+ else
+ render_api_error!(result[:message], result[:http_status])
+ end
+ end
+
+ desc 'Delete a scope from a feature flag' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag::LegacyScope
+ end
+ delete do
+ authorize_update_feature_flag!
+
+ param = { scopes_attributes: [{ id: scope.id, _destroy: true }] }
+
+ result = ::FeatureFlags::UpdateService
+ .new(user_project, current_user, param)
+ .execute(feature_flag)
+
+ if result[:status] == :success
+ status :no_content
+ else
+ render_api_error!(result[:message], result[:http_status])
+ end
+ end
+ end
+ end
+ end
+ end
+
+ helpers do
+ def authorize_read_feature_flags!
+ authorize! :read_feature_flag, user_project
+ end
+
+ def authorize_update_feature_flag!
+ authorize! :update_feature_flag, feature_flag
+ end
+
+ def feature_flag
+ @feature_flag ||= user_project.operations_feature_flags
+ .find_by_name!(params[:name])
+ end
+
+ def scope
+ @scope ||= feature_flag.scopes
+ .find_by_environment_scope!(CGI.unescape(params[:environment_scope]))
+ end
+
+ def scopes_for_environment
+ Operations::FeatureFlagScope
+ .for_unleash_client(user_project, params[:environment])
+ end
+ end
+ end
+end
diff --git a/lib/api/feature_flags.rb b/lib/api/feature_flags.rb
new file mode 100644
index 00000000000..613c3fb0f5b
--- /dev/null
+++ b/lib/api/feature_flags.rb
@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module API
+ class FeatureFlags < ::API::Base
+ include PaginationParams
+
+ FEATURE_FLAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS
+ .merge(name: API::NO_SLASH_URL_PART_REGEX)
+
+ before do
+ authorize_read_feature_flags!
+ end
+
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+ resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ resource :feature_flags do
+ desc 'Get all feature flags of a project' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag
+ end
+ params do
+ optional :scope, type: String, desc: 'The scope of feature flags',
+ values: %w[enabled disabled]
+ use :pagination
+ end
+ get do
+ feature_flags = ::FeatureFlagsFinder
+ .new(user_project, current_user, declared_params(include_missing: false))
+ .execute
+
+ present_entity(paginate(feature_flags))
+ end
+
+ desc 'Create a new feature flag' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag
+ end
+ params do
+ requires :name, type: String, desc: 'The name of feature flag'
+ optional :description, type: String, desc: 'The description of the feature flag'
+ optional :active, type: Boolean, desc: 'Active/inactive value of the flag'
+ optional :version, type: String, desc: 'The version of the feature flag'
+ optional :scopes, type: Array do
+ requires :environment_scope, type: String, desc: 'The environment scope of the scope'
+ requires :active, type: Boolean, desc: 'Active/inactive of the scope'
+ requires :strategies, type: JSON, desc: 'The strategies of the scope'
+ end
+ optional :strategies, type: Array do
+ requires :name, type: String, desc: 'The strategy name'
+ requires :parameters, type: JSON, desc: 'The strategy parameters'
+ optional :scopes, type: Array do
+ requires :environment_scope, type: String, desc: 'The environment scope of the scope'
+ end
+ end
+ end
+ post do
+ authorize_create_feature_flag!
+
+ attrs = declared_params(include_missing: false)
+
+ ensure_post_version_2_flags_enabled! if attrs[:version] == 'new_version_flag'
+
+ rename_key(attrs, :scopes, :scopes_attributes)
+ rename_key(attrs, :strategies, :strategies_attributes)
+ update_value(attrs, :strategies_attributes) do |strategies|
+ strategies.map { |s| rename_key(s, :scopes, :scopes_attributes) }
+ end
+
+ result = ::FeatureFlags::CreateService
+ .new(user_project, current_user, attrs)
+ .execute
+
+ if result[:status] == :success
+ present_entity(result[:feature_flag])
+ else
+ render_api_error!(result[:message], result[:http_status])
+ end
+ end
+ end
+
+ params do
+ requires :feature_flag_name, type: String, desc: 'The name of the feature flag'
+ end
+ resource 'feature_flags/:feature_flag_name', requirements: FEATURE_FLAG_ENDPOINT_REQUIREMENTS do
+ desc 'Get a feature flag of a project' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag
+ end
+ get do
+ authorize_read_feature_flag!
+
+ present_entity(feature_flag)
+ end
+
+ desc 'Enable a strategy for a feature flag on an environment' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag
+ end
+ params do
+ requires :environment_scope, type: String, desc: 'The environment scope of the feature flag'
+ requires :strategy, type: JSON, desc: 'The strategy to be enabled on the scope'
+ end
+ post :enable do
+ not_found! unless Feature.enabled?(:feature_flag_api, user_project)
+ render_api_error!('Version 2 flags not supported', :unprocessable_entity) if new_version_flag_present?
+
+ result = ::FeatureFlags::EnableService
+ .new(user_project, current_user, params).execute
+
+ if result[:status] == :success
+ status :ok
+ present_entity(result[:feature_flag])
+ else
+ render_api_error!(result[:message], result[:http_status])
+ end
+ end
+
+ desc 'Disable a strategy for a feature flag on an environment' do
+ detail 'This feature is going to be introduced in GitLab 12.5 if `feature_flag_api` feature flag is removed'
+ success ::API::Entities::FeatureFlag
+ end
+ params do
+ requires :environment_scope, type: String, desc: 'The environment scope of the feature flag'
+ requires :strategy, type: JSON, desc: 'The strategy to be disabled on the scope'
+ end
+ post :disable do
+ not_found! unless Feature.enabled?(:feature_flag_api, user_project)
+ render_api_error!('Version 2 flags not supported', :unprocessable_entity) if feature_flag.new_version_flag?
+
+ result = ::FeatureFlags::DisableService
+ .new(user_project, current_user, params).execute
+
+ if result[:status] == :success
+ status :ok
+ present_entity(result[:feature_flag])
+ else
+ render_api_error!(result[:message], result[:http_status])
+ end
+ end
+
+ desc 'Update a feature flag' do
+ detail 'This feature will be introduced in GitLab 13.1 if feature_flags_new_version feature flag is removed'
+ success ::API::Entities::FeatureFlag
+ end
+ params do
+ optional :name, type: String, desc: 'The name of the feature flag'
+ optional :description, type: String, desc: 'The description of the feature flag'
+ optional :active, type: Boolean, desc: 'Active/inactive value of the flag'
+ optional :strategies, type: Array do
+ optional :id, type: Integer, desc: 'The strategy id'
+ optional :name, type: String, desc: 'The strategy type'
+ optional :parameters, type: JSON, desc: 'The strategy parameters'
+ optional :_destroy, type: Boolean, desc: 'Delete the strategy when true'
+ optional :scopes, type: Array do
+ optional :id, type: Integer, desc: 'The environment scope id'
+ optional :environment_scope, type: String, desc: 'The environment scope of the scope'
+ optional :_destroy, type: Boolean, desc: 'Delete the scope when true'
+ end
+ end
+ end
+ put do
+ not_found! unless feature_flags_new_version_enabled?
+ authorize_update_feature_flag!
+ render_api_error!('PUT operations are not supported for legacy feature flags', :unprocessable_entity) if feature_flag.legacy_flag?
+
+ attrs = declared_params(include_missing: false)
+
+ rename_key(attrs, :strategies, :strategies_attributes)
+ update_value(attrs, :strategies_attributes) do |strategies|
+ strategies.map { |s| rename_key(s, :scopes, :scopes_attributes) }
+ end
+
+ result = ::FeatureFlags::UpdateService
+ .new(user_project, current_user, attrs)
+ .execute(feature_flag)
+
+ if result[:status] == :success
+ present_entity(result[:feature_flag])
+ else
+ render_api_error!(result[:message], result[:http_status])
+ end
+ end
+
+ desc 'Delete a feature flag' do
+ detail 'This feature was introduced in GitLab 12.5'
+ success ::API::Entities::FeatureFlag
+ end
+ delete do
+ authorize_destroy_feature_flag!
+
+ result = ::FeatureFlags::DestroyService
+ .new(user_project, current_user, declared_params(include_missing: false))
+ .execute(feature_flag)
+
+ if result[:status] == :success
+ present_entity(result[:feature_flag])
+ else
+ render_api_error!(result[:message], result[:http_status])
+ end
+ end
+ end
+ end
+
+ helpers do
+ def authorize_read_feature_flags!
+ authorize! :read_feature_flag, user_project
+ end
+
+ def authorize_read_feature_flag!
+ authorize! :read_feature_flag, feature_flag
+ end
+
+ def authorize_create_feature_flag!
+ authorize! :create_feature_flag, user_project
+ end
+
+ def authorize_update_feature_flag!
+ authorize! :update_feature_flag, feature_flag
+ end
+
+ def authorize_destroy_feature_flag!
+ authorize! :destroy_feature_flag, feature_flag
+ end
+
+ def present_entity(result)
+ present result,
+ with: ::API::Entities::FeatureFlag,
+ feature_flags_new_version_enabled: feature_flags_new_version_enabled?
+ end
+
+ def ensure_post_version_2_flags_enabled!
+ unless feature_flags_new_version_enabled?
+ render_api_error!('Version 2 flags are not enabled for this project', :unprocessable_entity)
+ end
+ end
+
+ def feature_flag
+ @feature_flag ||= if feature_flags_new_version_enabled?
+ user_project.operations_feature_flags.find_by_name!(params[:feature_flag_name])
+ else
+ user_project.operations_feature_flags.legacy_flag.find_by_name!(params[:feature_flag_name])
+ end
+ end
+
+ def new_version_flag_present?
+ user_project.operations_feature_flags.new_version_flag.find_by_name(params[:name]).present?
+ end
+
+ def feature_flags_new_version_enabled?
+ Feature.enabled?(:feature_flags_new_version, user_project, default_enabled: true)
+ end
+
+ def rename_key(hash, old_key, new_key)
+ hash[new_key] = hash.delete(old_key) if hash.key?(old_key)
+ hash
+ end
+
+ def update_value(hash, key)
+ hash[key] = yield(hash[key]) if hash.key?(key)
+ hash
+ end
+ end
+ end
+end
diff --git a/lib/api/feature_flags_user_lists.rb b/lib/api/feature_flags_user_lists.rb
new file mode 100644
index 00000000000..e5218cfd7f1
--- /dev/null
+++ b/lib/api/feature_flags_user_lists.rb
@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module API
+ class FeatureFlagsUserLists < ::API::Base
+ include PaginationParams
+
+ error_formatter :json, -> (message, _backtrace, _options, _env, _original_exception) {
+ message.is_a?(String) ? { message: message }.to_json : message.to_json
+ }
+
+ before do
+ authorize_admin_feature_flags_user_lists!
+ end
+
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+ resource 'projects/:id', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ resource :feature_flags_user_lists do
+ desc 'Get all feature flags user lists of a project' do
+ detail 'This feature was introduced in GitLab 12.10'
+ success ::API::Entities::FeatureFlag::UserList
+ end
+ params do
+ use :pagination
+ end
+ get do
+ present paginate(user_project.operations_feature_flags_user_lists),
+ with: ::API::Entities::FeatureFlag::UserList
+ end
+
+ desc 'Create a feature flags user list for a project' do
+ detail 'This feature was introduced in GitLab 12.10'
+ success ::API::Entities::FeatureFlag::UserList
+ end
+ params do
+ requires :name, type: String, desc: 'The name of the list'
+ requires :user_xids, type: String, desc: 'A comma separated list of external user ids'
+ end
+ post do
+ list = user_project.operations_feature_flags_user_lists.create(declared_params)
+
+ if list.save
+ present list, with: ::API::Entities::FeatureFlag::UserList
+ else
+ render_api_error!(list.errors.full_messages, :bad_request)
+ end
+ end
+ end
+
+ params do
+ requires :iid, type: String, desc: 'The internal id of the user list'
+ end
+ resource 'feature_flags_user_lists/:iid' do
+ desc 'Get a single feature flag user list belonging to a project' do
+ detail 'This feature was introduced in GitLab 12.10'
+ success ::API::Entities::FeatureFlag::UserList
+ end
+ get do
+ present user_project.operations_feature_flags_user_lists.find_by_iid!(params[:iid]),
+ with: ::API::Entities::FeatureFlag::UserList
+ end
+
+ desc 'Update a feature flag user list' do
+ detail 'This feature was introduced in GitLab 12.10'
+ success ::API::Entities::FeatureFlag::UserList
+ end
+ params do
+ optional :name, type: String, desc: 'The name of the list'
+ optional :user_xids, type: String, desc: 'A comma separated list of external user ids'
+ end
+ put do
+ list = user_project.operations_feature_flags_user_lists.find_by_iid!(params[:iid])
+
+ if list.update(declared_params(include_missing: false))
+ present list, with: ::API::Entities::FeatureFlag::UserList
+ else
+ render_api_error!(list.errors.full_messages, :bad_request)
+ end
+ end
+
+ desc 'Delete a feature flag user list' do
+ detail 'This feature was introduced in GitLab 12.10'
+ end
+ delete do
+ list = user_project.operations_feature_flags_user_lists.find_by_iid!(params[:iid])
+ unless list.destroy
+ render_api_error!(list.errors.full_messages, :conflict)
+ end
+ end
+ end
+ end
+
+ helpers do
+ def authorize_admin_feature_flags_user_lists!
+ authorize! :admin_feature_flags_user_lists, user_project
+ end
+ end
+ end
+end
diff --git a/lib/api/features.rb b/lib/api/features.rb
index 9d011d658f6..5d2e545abd6 100644
--- a/lib/api/features.rb
+++ b/lib/api/features.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Features < Grape::API::Instance
+ class Features < ::API::Base
before { authenticated_as_admin! }
helpers do
diff --git a/lib/api/files.rb b/lib/api/files.rb
index 748bdfa894d..6833fc429e2 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Files < Grape::API::Instance
+ class Files < ::API::Base
include APIGuard
FILE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(file_path: API::NO_SLASH_URL_PART_REGEX)
diff --git a/lib/api/freeze_periods.rb b/lib/api/freeze_periods.rb
index b8254ee9ab4..a83e36165a2 100644
--- a/lib/api/freeze_periods.rb
+++ b/lib/api/freeze_periods.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class FreezePeriods < Grape::API::Instance
+ class FreezePeriods < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb
index 98b8a40c7c9..a0c33ab65b9 100644
--- a/lib/api/generic_packages.rb
+++ b/lib/api/generic_packages.rb
@@ -1,7 +1,12 @@
# frozen_string_literal: true
module API
- class GenericPackages < Grape::API::Instance
+ class GenericPackages < ::API::Base
+ GENERIC_PACKAGES_REQUIREMENTS = {
+ package_name: API::NO_SLASH_URL_PART_REGEX,
+ file_name: API::NO_SLASH_URL_PART_REGEX
+ }.freeze
+
before do
require_packages_enabled!
authenticate!
@@ -17,17 +22,94 @@ module API
route_setting :authentication, job_token_allowed: true
namespace ':id/packages/generic' do
- get 'ping' do
- :pong
+ namespace ':package_name/*package_version/:file_name', requirements: GENERIC_PACKAGES_REQUIREMENTS do
+ desc 'Workhorse authorize generic package file' do
+ detail 'This feature was introduced in GitLab 13.5'
+ end
+
+ route_setting :authentication, job_token_allowed: true
+
+ params do
+ requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true
+ requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex
+ requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true
+ end
+
+ put 'authorize' do
+ authorize_workhorse!(subject: project, maximum_size: project.actual_limits.generic_packages_max_file_size)
+ end
+
+ desc 'Upload package file' do
+ detail 'This feature was introduced in GitLab 13.5'
+ end
+
+ params do
+ requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true
+ requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex
+ requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true
+ requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)'
+ end
+
+ route_setting :authentication, job_token_allowed: true
+
+ put do
+ authorize_upload!(project)
+ bad_request!('File is too large') if max_file_size_exceeded?
+
+ track_event('push_package')
+
+ create_package_file_params = declared_params.merge(build: current_authenticated_job)
+ ::Packages::Generic::CreatePackageFileService
+ .new(project, current_user, create_package_file_params)
+ .execute
+
+ created!
+ rescue ObjectStorage::RemoteStoreError => e
+ Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: project.id })
+
+ forbidden!
+ end
+
+ desc 'Download package file' do
+ detail 'This feature was introduced in GitLab 13.5'
+ end
+
+ params do
+ requires :package_name, type: String, desc: 'Package name', regexp: Gitlab::Regex.generic_package_name_regex, file_path: true
+ requires :package_version, type: String, desc: 'Package version', regexp: Gitlab::Regex.generic_package_version_regex
+ requires :file_name, type: String, desc: 'Package file name', regexp: Gitlab::Regex.generic_package_file_name_regex, file_path: true
+ end
+
+ route_setting :authentication, job_token_allowed: true
+
+ get do
+ authorize_read_package!(project)
+
+ package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version])
+ package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute!
+
+ track_event('pull_package')
+
+ present_carrierwave_file!(package_file.file)
+ end
end
end
end
helpers do
include ::API::Helpers::PackagesHelpers
+ include ::API::Helpers::Packages::BasicAuthHelpers
def require_generic_packages_available!
- not_found! unless Feature.enabled?(:generic_packages, user_project)
+ not_found! unless Feature.enabled?(:generic_packages, project, default_enabled: true)
+ end
+
+ def project
+ authorized_user_project
+ end
+
+ def max_file_size_exceeded?
+ project.actual_limits.exceeded?(:generic_packages_max_file_size, params[:file].size)
end
end
end
diff --git a/lib/api/github/entities.rb b/lib/api/github/entities.rb
index c28a0b8eb7e..fe228c9a2d2 100644
--- a/lib/api/github/entities.rb
+++ b/lib/api/github/entities.rb
@@ -119,7 +119,9 @@ module API
expose :username, as: :login
expose :user_url, as: :url
expose :user_url, as: :html_url
- expose :avatar_url
+ expose :avatar_url do |user|
+ user.avatar_url(only_path: false)
+ end
private
diff --git a/lib/api/go_proxy.rb b/lib/api/go_proxy.rb
index c0207f9169c..30f0cfb4dfd 100755
--- a/lib/api/go_proxy.rb
+++ b/lib/api/go_proxy.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
module API
- class GoProxy < Grape::API::Instance
+ class GoProxy < ::API::Base
helpers Gitlab::Golang
helpers ::API::Helpers::PackagesHelpers
diff --git a/lib/api/group_boards.rb b/lib/api/group_boards.rb
index 7efc12121d2..d4574b22d99 100644
--- a/lib/api/group_boards.rb
+++ b/lib/api/group_boards.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class GroupBoards < Grape::API::Instance
+ class GroupBoards < ::API::Base
include BoardsResponses
include PaginationParams
diff --git a/lib/api/group_clusters.rb b/lib/api/group_clusters.rb
index ae41d9f13b8..75429cf7a5c 100644
--- a/lib/api/group_clusters.rb
+++ b/lib/api/group_clusters.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class GroupClusters < Grape::API::Instance
+ class GroupClusters < ::API::Base
include PaginationParams
before { authenticate! }
@@ -41,6 +41,7 @@ module API
requires :name, type: String, desc: 'Cluster name'
optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true'
optional :environment_scope, default: '*', type: String, desc: 'The associated environment to the cluster'
+ optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace'
optional :domain, type: String, desc: 'Cluster base domain'
optional :management_project_id, type: Integer, desc: 'The ID of the management project'
optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true'
@@ -74,6 +75,7 @@ module API
optional :name, type: String, desc: 'Cluster name'
optional :domain, type: String, desc: 'Cluster base domain'
optional :environment_scope, type: String, desc: 'The associated environment to the cluster'
+ optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace'
optional :management_project_id, type: Integer, desc: 'The ID of the management project'
optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do
optional :api_url, type: String, desc: 'URL to access the Kubernetes API'
diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb
index 25b3059f63b..1bb26b3931c 100644
--- a/lib/api/group_container_repositories.rb
+++ b/lib/api/group_container_repositories.rb
@@ -1,9 +1,11 @@
# frozen_string_literal: true
module API
- class GroupContainerRepositories < Grape::API::Instance
+ class GroupContainerRepositories < ::API::Base
include PaginationParams
+ helpers ::API::Helpers::PackagesHelpers
+
before { authorize_read_group_container_images! }
REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(
@@ -27,7 +29,7 @@ module API
user: current_user, subject: user_group
).execute
- track_event('list_repositories')
+ track_package_event('list_repositories', :container)
present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count]
end
diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb
index dc14813eefc..6ebaa8de185 100644
--- a/lib/api/group_export.rb
+++ b/lib/api/group_export.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class GroupExport < Grape::API::Instance
+ class GroupExport < ::API::Base
helpers Helpers::RateLimiter
before do
diff --git a/lib/api/group_import.rb b/lib/api/group_import.rb
index b82d9fc519a..e703a217fd5 100644
--- a/lib/api/group_import.rb
+++ b/lib/api/group_import.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class GroupImport < Grape::API::Instance
+ class GroupImport < ::API::Base
helpers Helpers::FileUploadHelpers
helpers do
diff --git a/lib/api/group_labels.rb b/lib/api/group_labels.rb
index 56f2b769464..8443ddf10ce 100644
--- a/lib/api/group_labels.rb
+++ b/lib/api/group_labels.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class GroupLabels < Grape::API::Instance
+ class GroupLabels < ::API::Base
include PaginationParams
helpers ::API::Helpers::LabelHelpers
diff --git a/lib/api/group_milestones.rb b/lib/api/group_milestones.rb
index 82f5df79356..aef9877b84c 100644
--- a/lib/api/group_milestones.rb
+++ b/lib/api/group_milestones.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class GroupMilestones < Grape::API::Instance
+ class GroupMilestones < ::API::Base
include MilestoneResponses
include PaginationParams
diff --git a/lib/api/group_packages.rb b/lib/api/group_packages.rb
index aa047e260f5..5b6290df0dd 100644
--- a/lib/api/group_packages.rb
+++ b/lib/api/group_packages.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class GroupPackages < Grape::API::Instance
+ class GroupPackages < ::API::Base
include PaginationParams
before do
diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb
index e7b8cd10197..ee110d67fa5 100644
--- a/lib/api/group_variables.rb
+++ b/lib/api/group_variables.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class GroupVariables < Grape::API::Instance
+ class GroupVariables < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 813e41b4d39..bf3d6c3c7e0 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Groups < Grape::API::Instance
+ class Groups < ::API::Base
include PaginationParams
include Helpers::CustomAttributes
@@ -29,7 +29,12 @@ module API
# rubocop: disable CodeReuse/ActiveRecord
def find_groups(params, parent_id = nil)
- find_params = params.slice(:all_available, :custom_attributes, :owned, :min_access_level)
+ find_params = params.slice(
+ :all_available,
+ :custom_attributes,
+ :owned, :min_access_level,
+ :include_parent_descendants
+ )
find_params[:parent] = if params[:top_level_only]
[nil]
@@ -309,6 +314,19 @@ module API
present_groups params, groups
end
+ desc 'Get a list of descendant groups of this group.' do
+ success Entities::Group
+ end
+ params do
+ use :group_list_params
+ use :with_custom_attributes
+ end
+ get ":id/descendant_groups" do
+ finder_params = declared_params(include_missing: false).merge(include_parent_descendants: true)
+ groups = find_groups(finder_params, params[:id])
+ present_groups params, groups
+ end
+
desc 'Transfer a project to the group namespace. Available only for admin.' do
success Entities::GroupDetail
end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 1912a06682e..c8aee1f3479 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -522,7 +522,7 @@ module API
else
header(*Gitlab::Workhorse.send_url(file.url))
status :ok
- body
+ body ""
end
end
@@ -544,7 +544,6 @@ module API
feature_name = "usage_data_#{event_name}"
return unless Feature.enabled?(feature_name)
- return unless Gitlab::CurrentSettings.usage_ping_enabled?
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name)
rescue => error
diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb
index f3dfc093926..ba07a70ee32 100644
--- a/lib/api/helpers/groups_helpers.rb
+++ b/lib/api/helpers/groups_helpers.rb
@@ -24,6 +24,7 @@ module API
optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to master'
+ optional :shared_runners_setting, type: String, values: ::Namespace::SHARED_RUNNERS_SETTINGS, desc: 'Enable/disable shared runners for the group and its subgroups and projects'
end
params :optional_params_ee do
diff --git a/lib/api/helpers/merge_requests_helpers.rb b/lib/api/helpers/merge_requests_helpers.rb
index e4163c63575..9b38eeb1e72 100644
--- a/lib/api/helpers/merge_requests_helpers.rb
+++ b/lib/api/helpers/merge_requests_helpers.rb
@@ -73,6 +73,13 @@ module API
optional :not, type: Hash, desc: 'Parameters to negate' do
use :merge_requests_negatable_params
end
+
+ optional :deployed_before,
+ 'Return merge requests deployed before the given date/time'
+ optional :deployed_after,
+ 'Return merge requests deployed after the given date/time'
+ optional :environment,
+ 'Returns merge requests deployed to the given environment'
end
params :optional_scope_param do
diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb
index dcbf933a4e1..934e18bdd0a 100644
--- a/lib/api/helpers/packages/conan/api_helpers.rb
+++ b/lib/api/helpers/packages/conan/api_helpers.rb
@@ -158,7 +158,7 @@ module API
conan_package_reference: params[:conan_package_reference]
).execute!
- package_event('pull_package', category: 'API::ConanPackages') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY
+ track_package_event('pull_package', :conan, category: 'API::ConanPackages') if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY
present_carrierwave_file!(package_file.file)
end
@@ -169,7 +169,7 @@ module API
def track_push_package_event
if params[:file_name] == ::Packages::Conan::FileMetadatum::PACKAGE_BINARY && params[:file].size > 0 # rubocop: disable Style/ZeroLengthPredicate
- package_event('push_package', category: 'API::ConanPackages')
+ track_package_event('push_package', :conan, category: 'API::ConanPackages')
end
end
diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb
index 254af7690a2..577ba97d68a 100644
--- a/lib/api/helpers/packages/dependency_proxy_helpers.rb
+++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb
@@ -10,6 +10,7 @@ module API
def redirect_registry_request(forward_to_registry, package_type, options)
if forward_to_registry && redirect_registry_request_available?
+ track_event("#{package_type}_request_forward")
redirect(registry_url(package_type, options))
else
yield
diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb
index 403f5ea3851..e1898d28ef7 100644
--- a/lib/api/helpers/packages_helpers.rb
+++ b/lib/api/helpers/packages_helpers.rb
@@ -40,7 +40,7 @@ module API
params = { has_length: has_length }
params[:maximum_size] = maximum_size unless has_length
- ::Packages::PackageFileUploader.workhorse_authorize(params)
+ ::Packages::PackageFileUploader.workhorse_authorize(**params)
end
def authorize_upload!(subject = user_project)
@@ -48,7 +48,8 @@ module API
require_gitlab_workhorse!
end
- def package_event(event_name, **args)
+ def track_package_event(event_name, scope, **args)
+ ::Packages::CreateEventService.new(nil, current_user, event_name: event_name, scope: scope).execute
track_event(event_name, **args)
end
end
diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb
index a6ae9a87f98..227aec224e5 100644
--- a/lib/api/helpers/pagination.rb
+++ b/lib/api/helpers/pagination.rb
@@ -3,8 +3,8 @@
module API
module Helpers
module Pagination
- def paginate(relation)
- Gitlab::Pagination::OffsetPagination.new(self).paginate(relation)
+ def paginate(*args)
+ Gitlab::Pagination::OffsetPagination.new(self).paginate(*args)
end
end
end
diff --git a/lib/api/helpers/presentable.rb b/lib/api/helpers/presentable.rb
index a5186cc56ea..40e1b266df5 100644
--- a/lib/api/helpers/presentable.rb
+++ b/lib/api/helpers/presentable.rb
@@ -23,7 +23,7 @@ module API
def initialize(object, options = {})
options = options.opts_hash if options.is_a?(Grape::Entity::Options)
- super(object.present(options), options)
+ super(object.present(**options), options)
end
end
end
diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb
index 8c20f5b8fc2..0364ba2ad9e 100644
--- a/lib/api/helpers/projects_helpers.rb
+++ b/lib/api/helpers/projects_helpers.rb
@@ -83,9 +83,18 @@ module API
params :optional_filter_params_ee do
end
+ params :optional_update_params_ce do
+ optional :ci_forward_deployment_enabled, type: Boolean, desc: 'Skip older deployment jobs that are still pending'
+ end
+
params :optional_update_params_ee do
end
+ params :optional_update_params do
+ use :optional_update_params_ce
+ use :optional_update_params_ee
+ end
+
params :optional_container_expiration_policy_params do
optional :cadence, type: String, desc: 'Container expiration policy cadence for recurring job'
optional :keep_n, type: String, desc: 'Container expiration policy number of images to keep'
@@ -108,6 +117,7 @@ module API
:builds_access_level,
:ci_config_path,
:ci_default_git_depth,
+ :ci_forward_deployment_enabled,
:container_registry_enabled,
:container_expiration_policy_attributes,
:default_branch,
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 34a2fb09875..1c85669a626 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -51,9 +51,7 @@ module API
job_forbidden!(job, 'Job is not running') unless job.running?
end
- if Gitlab::Ci::Features.job_heartbeats_runner?(job.project)
- job.runner&.heartbeat(get_runner_ip)
- end
+ job.runner&.heartbeat(get_runner_ip)
job
end
diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb
index 4bceda51900..4adb27a7414 100644
--- a/lib/api/helpers/services_helpers.rb
+++ b/lib/api/helpers/services_helpers.rb
@@ -381,6 +381,12 @@ module API
type: String,
desc: 'The Hangouts Chat webhook. e.g. https://chat.googleapis.com/v1/spaces…'
},
+ {
+ required: false,
+ name: :branches_to_be_notified,
+ type: String,
+ desc: 'Branches for which notifications are to be sent'
+ },
chat_notification_events
].flatten,
'hipchat' => [
diff --git a/lib/api/helpers/settings_helpers.rb b/lib/api/helpers/settings_helpers.rb
index 65aec6ae2e7..451e578fdd6 100644
--- a/lib/api/helpers/settings_helpers.rb
+++ b/lib/api/helpers/settings_helpers.rb
@@ -12,6 +12,7 @@ module API
def self.optional_attributes
[*::ApplicationSettingsHelper.visible_attributes,
*::ApplicationSettingsHelper.external_authorization_service_attributes,
+ *::ApplicationSettingsHelper.deprecated_attributes,
:performance_bar_allowed_group_id].freeze
end
end
diff --git a/lib/api/helpers/snippets_helpers.rb b/lib/api/helpers/snippets_helpers.rb
index 9224381735f..42f56680ded 100644
--- a/lib/api/helpers/snippets_helpers.rb
+++ b/lib/api/helpers/snippets_helpers.rb
@@ -93,7 +93,7 @@ module API
def validate_params_for_multiple_files(snippet)
return unless params[:content] || params[:file_name]
- if Feature.enabled?(:snippet_multiple_files, current_user) && snippet.multiple_files?
+ if snippet.multiple_files?
render_api_error!({ error: _('To update Snippets with multiple files, you must use the `files` parameter') }, 400)
end
end
diff --git a/lib/api/import_bitbucket_server.rb b/lib/api/import_bitbucket_server.rb
index df3235420e9..a0238c24f3b 100644
--- a/lib/api/import_bitbucket_server.rb
+++ b/lib/api/import_bitbucket_server.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ImportBitbucketServer < Grape::API::Instance
+ class ImportBitbucketServer < ::API::Base
helpers do
def client
@client ||= BitbucketServer::Client.new(credentials)
diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb
index 0bab891eada..61fce7a2c1b 100644
--- a/lib/api/import_github.rb
+++ b/lib/api/import_github.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ImportGithub < Grape::API::Instance
+ class ImportGithub < ::API::Base
rescue_from Octokit::Unauthorized, with: :provider_unauthorized
before do
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index ff687a57888..6d8f13c36e6 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -3,7 +3,7 @@
module API
# Internal access API
module Internal
- class Base < Grape::API::Instance
+ class Base < ::API::Base
before { authenticate_by_gitlab_shell_token! }
before do
@@ -99,6 +99,14 @@ module API
@project = @container = access_checker.container
end
end
+
+ def validate_actor_key(actor, key_id)
+ return 'Could not find a user without a key' unless key_id
+
+ return 'Could not find the given key' unless actor.key
+
+ 'Could not find a user for the given key' unless actor.user
+ end
end
namespace 'internal' do
@@ -163,28 +171,23 @@ module API
redis: redis_ping
}
end
+
post '/two_factor_recovery_codes' do
status 200
actor.update_last_used_at!
user = actor.user
- if params[:key_id]
- unless actor.key
- break { success: false, message: 'Could not find the given key' }
- end
-
- if actor.key.is_a?(DeployKey)
- break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' }
- end
+ error_message = validate_actor_key(actor, params[:key_id])
- unless user
- break { success: false, message: 'Could not find a user for the given key' }
- end
- elsif params[:user_id] && user.nil?
+ if params[:user_id] && user.nil?
break { success: false, message: 'Could not find the given user' }
+ elsif error_message
+ break { success: false, message: error_message }
end
+ break { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } if actor.key.is_a?(DeployKey)
+
unless user.two_factor_enabled?
break { success: false, message: 'Two-factor authentication is not enabled for this user' }
end
@@ -204,20 +207,14 @@ module API
actor.update_last_used_at!
user = actor.user
- if params[:key_id]
- unless actor.key
- break { success: false, message: 'Could not find the given key' }
- end
+ error_message = validate_actor_key(actor, params[:key_id])
- if actor.key.is_a?(DeployKey)
- break { success: false, message: 'Deploy keys cannot be used to create personal access tokens' }
- end
+ break { success: false, message: 'Deploy keys cannot be used to create personal access tokens' } if actor.key.is_a?(DeployKey)
- unless user
- break { success: false, message: 'Could not find a user for the given key' }
- end
- elsif params[:user_id] && user.nil?
+ if params[:user_id] && user.nil?
break { success: false, message: 'Could not find the given user' }
+ elsif error_message
+ break { success: false, message: error_message }
end
if params[:name].blank?
@@ -269,6 +266,53 @@ module API
present response, with: Entities::InternalPostReceive::Response
end
+
+ post '/two_factor_config' do
+ status 200
+
+ break { success: false } unless Feature.enabled?(:two_factor_for_cli)
+
+ actor.update_last_used_at!
+ user = actor.user
+
+ error_message = validate_actor_key(actor, params[:key_id])
+
+ if error_message
+ { success: false, message: error_message }
+ elsif actor.key.is_a?(DeployKey)
+ { success: true, two_factor_required: false }
+ else
+ {
+ success: true,
+ two_factor_required: user.two_factor_enabled?
+ }
+ end
+ end
+
+ post '/two_factor_otp_check' do
+ status 200
+
+ break { success: false } unless Feature.enabled?(:two_factor_for_cli)
+
+ actor.update_last_used_at!
+ user = actor.user
+
+ error_message = validate_actor_key(actor, params[:key_id])
+
+ break { success: false, message: error_message } if error_message
+
+ break { success: false, message: 'Deploy keys cannot be used for Two Factor' } if actor.key.is_a?(DeployKey)
+
+ break { success: false, message: 'Two-factor authentication is not enabled for this user' } unless user.two_factor_enabled?
+
+ otp_validation_result = ::Users::ValidateOtpService.new(user).execute(params.fetch(:otp_attempt))
+
+ if otp_validation_result[:status] == :success
+ { success: true }
+ else
+ { success: false, message: 'Invalid OTP' }
+ end
+ end
end
end
end
diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb
index 6d5dfd086e7..8175b81f900 100644
--- a/lib/api/internal/kubernetes.rb
+++ b/lib/api/internal/kubernetes.rb
@@ -3,7 +3,7 @@
module API
# Kubernetes Internal API
module Internal
- class Kubernetes < Grape::API::Instance
+ class Kubernetes < ::API::Base
before do
check_feature_enabled
authenticate_gitlab_kas_request!
diff --git a/lib/api/internal/lfs.rb b/lib/api/internal/lfs.rb
new file mode 100644
index 00000000000..630f0ec77a8
--- /dev/null
+++ b/lib/api/internal/lfs.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module API
+ module Internal
+ class Lfs < ::API::Base
+ use Rack::Sendfile
+
+ before { authenticate_by_gitlab_shell_token! }
+
+ helpers do
+ def find_lfs_object(lfs_oid)
+ LfsObject.find_by_oid(lfs_oid)
+ end
+ end
+
+ namespace 'internal' do
+ namespace 'lfs' do
+ desc 'Get LFS URL for object ID' do
+ detail 'This feature was introduced in GitLab 13.5.'
+ end
+ params do
+ requires :oid, type: String, desc: 'The object ID to query'
+ requires :gl_repository, type: String, desc: "Project identifier (e.g. project-1)"
+ end
+ get "/" do
+ lfs_object = find_lfs_object(params[:oid])
+
+ not_found! unless lfs_object
+
+ _, project, repo_type = Gitlab::GlRepository.parse(params[:gl_repository])
+
+ not_found! unless repo_type.project? && project
+ not_found! unless lfs_object.project_allowed_access?(project)
+
+ file = lfs_object.file
+
+ not_found! unless file&.exists?
+
+ content_type 'application/octet-stream'
+
+ if file.file_storage?
+ sendfile file.path
+ else
+ workhorse_headers = Gitlab::Workhorse.send_url(file.url)
+ header workhorse_headers[0], workhorse_headers[1]
+ env['api.format'] = :binary
+ body ""
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb
index 5f8d23f15fa..51136144c19 100644
--- a/lib/api/internal/pages.rb
+++ b/lib/api/internal/pages.rb
@@ -3,7 +3,7 @@
module API
# Pages Internal API
module Internal
- class Pages < Grape::API::Instance
+ class Pages < ::API::Base
before do
authenticate_gitlab_pages_request!
end
diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb
index 6cc5b344f47..db4979c9052 100644
--- a/lib/api/issue_links.rb
+++ b/lib/api/issue_links.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class IssueLinks < Grape::API::Instance
+ class IssueLinks < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 0e5b0fae6e2..143f9e40736 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Issues < Grape::API::Instance
+ class Issues < ::API::Base
include PaginationParams
helpers Helpers::IssuesHelpers
helpers Helpers::RateLimiter
@@ -231,9 +231,6 @@ module API
authorize! :create_issue, user_project
- params.delete(:created_at) unless current_user.can?(:set_issue_created_at, user_project)
- params.delete(:iid) unless current_user.can?(:set_issue_iid, user_project)
-
issue_params = declared_params(include_missing: false)
issue_params[:system_note_timestamp] = params[:created_at]
@@ -279,8 +276,6 @@ module API
issue = user_project.issues.find_by!(iid: params.delete(:issue_iid))
authorize! :update_issue, issue
- # Setting updated_at is allowed only for admins and owners
- params.delete(:updated_at) unless current_user.can?(:set_issue_updated_at, user_project)
issue.system_note_timestamp = params[:updated_at]
update_params = declared_params(include_missing: false).merge(request: request, api: true)
diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb
index bc7bc956580..536b361b308 100644
--- a/lib/api/job_artifacts.rb
+++ b/lib/api/job_artifacts.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class JobArtifacts < Grape::API::Instance
+ class JobArtifacts < ::API::Base
before { authenticate_non_get! }
# EE::API::JobArtifacts would override the following helpers
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index ad46d948f3b..bdb23b4a9be 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Jobs < Grape::API::Instance
+ class Jobs < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/keys.rb b/lib/api/keys.rb
index c014641ca04..2e4568029b5 100644
--- a/lib/api/keys.rb
+++ b/lib/api/keys.rb
@@ -2,7 +2,7 @@
module API
# Keys API
- class Keys < Grape::API::Instance
+ class Keys < ::API::Base
before { authenticate! }
resource :keys do
diff --git a/lib/api/labels.rb b/lib/api/labels.rb
index edf4a8ca14e..0cc9f33bd07 100644
--- a/lib/api/labels.rb
+++ b/lib/api/labels.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Labels < Grape::API::Instance
+ class Labels < ::API::Base
include PaginationParams
helpers ::API::Helpers::LabelHelpers
diff --git a/lib/api/lint.rb b/lib/api/lint.rb
index f7796b1e969..bfd152f70b1 100644
--- a/lib/api/lint.rb
+++ b/lib/api/lint.rb
@@ -1,24 +1,48 @@
# frozen_string_literal: true
module API
- class Lint < Grape::API::Instance
+ class Lint < ::API::Base
namespace :ci do
desc 'Validation of .gitlab-ci.yml content'
params do
requires :content, type: String, desc: 'Content of .gitlab-ci.yml'
+ optional :include_merged_yaml, type: Boolean, desc: 'Whether or not to include merged CI config yaml in the response'
end
post '/lint' do
- error = Gitlab::Ci::YamlProcessor.validation_message(params[:content],
- user: current_user)
+ result = Gitlab::Ci::YamlProcessor.new(params[:content], user: current_user).execute
+ error = result.errors.first
status 200
- if error.blank?
- { status: 'valid', errors: [] }
- else
- { status: 'invalid', errors: [error] }
+ response = if error.blank?
+ { status: 'valid', errors: [] }
+ else
+ { status: 'invalid', errors: [error] }
+ end
+
+ response.tap do |response|
+ response[:merged_yaml] = result.merged_yaml if params[:include_merged_yaml]
end
end
end
+
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc 'Validation of .gitlab-ci.yml content' do
+ detail 'This feature was introduced in GitLab 13.5.'
+ end
+ params do
+ optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check.'
+ end
+ get ':id/ci/lint' do
+ authorize! :download_code, user_project
+
+ content = user_project.repository.gitlab_ci_yml_for(user_project.commit.id, user_project.ci_config_path_or_default)
+ result = Gitlab::Ci::Lint
+ .new(project: user_project, current_user: current_user)
+ .validate(content, dry_run: params[:dry_run])
+
+ present result, with: Entities::Ci::Lint::Result, current_user: current_user
+ end
+ end
end
end
diff --git a/lib/api/markdown.rb b/lib/api/markdown.rb
index a0822271cca..97549abd273 100644
--- a/lib/api/markdown.rb
+++ b/lib/api/markdown.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Markdown < Grape::API::Instance
+ class Markdown < ::API::Base
params do
requires :text, type: String, desc: "The markdown text to render"
optional :gfm, type: Boolean, desc: "Render text using GitLab Flavored Markdown"
diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb
index e6d9a9a7c20..a3e2fa84c32 100644
--- a/lib/api/maven_packages.rb
+++ b/lib/api/maven_packages.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
module API
- class MavenPackages < Grape::API::Instance
+ class MavenPackages < ::API::Base
MAVEN_ENDPOINT_REQUIREMENTS = {
file_name: API::NO_SLASH_URL_PART_REGEX
}.freeze
@@ -32,10 +32,10 @@ module API
end
def verify_package_file(package_file, uploaded_file)
- stored_sha1 = Digest::SHA256.hexdigest(package_file.file_sha1)
- expected_sha1 = uploaded_file.sha256
+ stored_sha256 = Digest::SHA256.hexdigest(package_file.file_sha1)
+ expected_sha256 = uploaded_file.sha256
- if stored_sha1 == expected_sha1
+ if stored_sha256 == expected_sha256
no_content!
else
conflict!
@@ -107,7 +107,7 @@ module API
when 'sha1'
package_file.file_sha1
else
- package_event('pull_package') if jar_file?(format)
+ track_package_event('pull_package', :maven) if jar_file?(format)
present_carrierwave_file_with_head_support!(package_file.file)
end
end
@@ -145,7 +145,7 @@ module API
when 'sha1'
package_file.file_sha1
else
- package_event('pull_package') if jar_file?(format)
+ track_package_event('pull_package', :maven) if jar_file?(format)
present_carrierwave_file_with_head_support!(package_file.file)
end
@@ -181,7 +181,7 @@ module API
when 'sha1'
package_file.file_sha1
else
- package_event('pull_package') if jar_file?(format)
+ track_package_event('pull_package', :maven) if jar_file?(format)
present_carrierwave_file_with_head_support!(package_file.file)
end
@@ -231,9 +231,9 @@ module API
verify_package_file(package_file, params[:file])
when 'md5'
- nil
+ ''
else
- package_event('push_package') if jar_file?(format)
+ track_package_event('push_package', :maven) if jar_file?(format)
file_params = {
file: params[:file],
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 4edf94c6350..c28b3b1cc7c 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Members < Grape::API::Instance
+ class Members < ::API::Base
include PaginationParams
before { authenticate! }
@@ -88,8 +88,8 @@ module API
success Entities::Member
end
params do
- requires :user_id, type: Integer, desc: 'The user ID of the new member'
requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)'
+ requires :user_id, types: [Integer, String], desc: 'The user ID of the new member or multiple IDs separated by commas.'
optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
end
# rubocop: disable CodeReuse/ActiveRecord
@@ -97,20 +97,26 @@ module API
source = find_source(source_type, params[:id])
authorize_admin_source!(source_type, source)
- member = source.members.find_by(user_id: params[:user_id])
- conflict!('Member already exists') if member
+ if params[:user_id].to_s.include?(',')
+ create_service_params = params.except(:user_id).merge({ user_ids: params[:user_id] })
- user = User.find_by_id(params[:user_id])
- not_found!('User') unless user
+ ::Members::CreateService.new(current_user, create_service_params).execute(source)
+ elsif params[:user_id].present?
+ member = source.members.find_by(user_id: params[:user_id])
+ conflict!('Member already exists') if member
- member = create_member(current_user, user, source, params)
+ user = User.find_by_id(params[:user_id])
+ not_found!('User') unless user
- if !member
- not_allowed! # This currently can only be reached in EE
- elsif member.valid? && member.persisted?
- present_members(member)
- else
- render_validation_error!(member)
+ member = create_member(current_user, user, source, params)
+
+ if !member
+ not_allowed! # This currently can only be reached in EE
+ elsif member.valid? && member.persisted?
+ present_members(member)
+ else
+ render_validation_error!(member)
+ end
end
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/api/merge_request_approvals.rb b/lib/api/merge_request_approvals.rb
index 035ed9f0e04..14d6e3995ea 100644
--- a/lib/api/merge_request_approvals.rb
+++ b/lib/api/merge_request_approvals.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class MergeRequestApprovals < ::Grape::API::Instance
+ class MergeRequestApprovals < ::API::Base
before { authenticate_non_get! }
helpers do
diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb
index 3e43fe8b257..22023888bbd 100644
--- a/lib/api/merge_request_diffs.rb
+++ b/lib/api/merge_request_diffs.rb
@@ -2,7 +2,7 @@
module API
# MergeRequestDiff API
- class MergeRequestDiffs < Grape::API::Instance
+ class MergeRequestDiffs < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 4bd72b267a9..b24dd870c8b 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class MergeRequests < Grape::API::Instance
+ class MergeRequests < ::API::Base
include PaginationParams
CONTEXT_COMMITS_POST_LIMIT = 20
diff --git a/lib/api/metrics/dashboard/annotations.rb b/lib/api/metrics/dashboard/annotations.rb
index e07762ac6d3..b6bc0af2202 100644
--- a/lib/api/metrics/dashboard/annotations.rb
+++ b/lib/api/metrics/dashboard/annotations.rb
@@ -3,7 +3,7 @@
module API
module Metrics
module Dashboard
- class Annotations < Grape::API::Instance
+ class Annotations < ::API::Base
desc 'Create a new monitoring dashboard annotation' do
success Entities::Metrics::Dashboard::Annotation
end
diff --git a/lib/api/metrics/user_starred_dashboards.rb b/lib/api/metrics/user_starred_dashboards.rb
index 263d2394276..cb6e7099247 100644
--- a/lib/api/metrics/user_starred_dashboards.rb
+++ b/lib/api/metrics/user_starred_dashboards.rb
@@ -2,7 +2,7 @@
module API
module Metrics
- class UserStarredDashboards < Grape::API::Instance
+ class UserStarredDashboards < ::API::Base
resource :projects do
desc 'Marks selected metrics dashboard as starred' do
success Entities::Metrics::UserStarredDashboard
diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb
index e1f279df045..f98a1f6dd1d 100644
--- a/lib/api/namespaces.rb
+++ b/lib/api/namespaces.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Namespaces < Grape::API::Instance
+ class Namespaces < ::API::Base
include PaginationParams
before { authenticate! }
@@ -32,7 +32,9 @@ module API
get do
namespaces = current_user.admin ? Namespace.all : current_user.namespaces
- namespaces = namespaces.include_gitlab_subscription if Gitlab.ee?
+ namespaces = namespaces.include_route
+
+ namespaces = namespaces.include_gitlab_subscription_with_hosted_plan if Gitlab.ee?
namespaces = namespaces.search(params[:search]) if params[:search].present?
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index e4989243f3d..0db537ca616 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Notes < Grape::API::Instance
+ class Notes < ::API::Base
include PaginationParams
helpers ::API::Helpers::NotesHelpers
diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb
index f8b621c1c38..bad3f5ead7a 100644
--- a/lib/api/notification_settings.rb
+++ b/lib/api/notification_settings.rb
@@ -2,7 +2,7 @@
module API
# notification_settings API
- class NotificationSettings < Grape::API::Instance
+ class NotificationSettings < ::API::Base
before { authenticate! }
helpers ::API::Helpers::MembersHelpers
diff --git a/lib/api/npm_packages.rb b/lib/api/npm_packages.rb
index fca405b76b7..1443b28c1ee 100644
--- a/lib/api/npm_packages.rb
+++ b/lib/api/npm_packages.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
module API
- class NpmPackages < Grape::API::Instance
+ class NpmPackages < ::API::Base
helpers ::API::Helpers::PackagesHelpers
helpers ::API::Helpers::Packages::DependencyProxyHelpers
@@ -141,7 +141,7 @@ module API
package_file = ::Packages::PackageFileFinder
.new(package, params[:file_name]).execute!
- package_event('pull_package')
+ track_package_event('pull_package', package)
present_carrierwave_file!(package_file.file)
end
@@ -157,7 +157,7 @@ module API
put ':id/packages/npm/:package_name', requirements: NPM_ENDPOINT_REQUIREMENTS do
authorize_create_package!(user_project)
- package_event('push_package')
+ track_package_event('push_package', :npm)
created_package = ::Packages::Npm::CreatePackageService
.new(user_project, current_user, params.merge(build: current_authenticated_job)).execute
diff --git a/lib/api/nuget_packages.rb b/lib/api/nuget_packages.rb
index f84a3acbe6d..0f2c956a9df 100644
--- a/lib/api/nuget_packages.rb
+++ b/lib/api/nuget_packages.rb
@@ -6,7 +6,7 @@
# called by the NuGet package manager client when users run commands
# like `nuget install` or `nuget push`.
module API
- class NugetPackages < Grape::API::Instance
+ class NugetPackages < ::API::Base
helpers ::API::Helpers::PackagesManagerClientsHelpers
helpers ::API::Helpers::Packages::BasicAuthHelpers
@@ -42,7 +42,7 @@ module API
def package_finder(finder_params = {})
::Packages::Nuget::PackageFinder.new(
authorized_user_project,
- finder_params.merge(package_name: params[:package_name])
+ **finder_params.merge(package_name: params[:package_name])
)
end
end
@@ -73,7 +73,7 @@ module API
get 'index', format: :json do
authorize_read_package!(authorized_user_project)
- track_event('nuget_service_index')
+ track_package_event('cli_metadata', :nuget)
present ::Packages::Nuget::ServiceIndexPresenter.new(authorized_user_project),
with: ::API::Entities::Nuget::ServiceIndex
@@ -105,7 +105,7 @@ module API
package_file = ::Packages::CreatePackageFileService.new(package, file_params)
.execute
- package_event('push_package')
+ track_package_event('push_package', :nuget)
::Packages::Nuget::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker
@@ -198,7 +198,7 @@ module API
not_found!('Package') unless package_file
- package_event('pull_package')
+ track_package_event('pull_package', :nuget)
# nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false
present_carrierwave_file!(package_file.file, supports_direct_download: false)
@@ -233,7 +233,7 @@ module API
.new(authorized_user_project, params[:q], search_options)
.execute
- package_event('search_package')
+ track_package_event('search_package', :nuget)
present ::Packages::Nuget::SearchResultsPresenter.new(search),
with: ::API::Entities::Nuget::SearchResults
diff --git a/lib/api/package_files.rb b/lib/api/package_files.rb
index 17b92df629c..c1fc9a6e4d8 100644
--- a/lib/api/package_files.rb
+++ b/lib/api/package_files.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class PackageFiles < Grape::API::Instance
+ class PackageFiles < ::API::Base
include PaginationParams
before do
diff --git a/lib/api/pages.rb b/lib/api/pages.rb
index 79a6b527581..813307c498f 100644
--- a/lib/api/pages.rb
+++ b/lib/api/pages.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Pages < Grape::API::Instance
+ class Pages < ::API::Base
before do
require_pages_config_enabled!
authenticated_with_can_read_all_resources!
diff --git a/lib/api/pages_domains.rb b/lib/api/pages_domains.rb
index 7d27b575efa..00c51298c45 100644
--- a/lib/api/pages_domains.rb
+++ b/lib/api/pages_domains.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class PagesDomains < Grape::API::Instance
+ class PagesDomains < ::API::Base
include PaginationParams
PAGES_DOMAINS_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(domain: API::NO_SLASH_URL_PART_REGEX)
diff --git a/lib/api/project_clusters.rb b/lib/api/project_clusters.rb
index 0e5605984e6..46ccb4ba1a0 100644
--- a/lib/api/project_clusters.rb
+++ b/lib/api/project_clusters.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectClusters < Grape::API::Instance
+ class ProjectClusters < ::API::Base
include PaginationParams
before { authenticate! }
@@ -45,6 +45,7 @@ module API
optional :enabled, type: Boolean, default: true, desc: 'Determines if cluster is active or not, defaults to true'
optional :domain, type: String, desc: 'Cluster base domain'
optional :environment_scope, default: '*', type: String, desc: 'The associated environment to the cluster'
+ optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace'
optional :management_project_id, type: Integer, desc: 'The ID of the management project'
optional :managed, type: Boolean, default: true, desc: 'Determines if GitLab will manage namespaces and service accounts for this cluster, defaults to true'
requires :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do
@@ -78,6 +79,7 @@ module API
optional :name, type: String, desc: 'Cluster name'
optional :domain, type: String, desc: 'Cluster base domain'
optional :environment_scope, type: String, desc: 'The associated environment to the cluster'
+ optional :namespace_per_environment, default: true, type: Boolean, desc: 'Deploy each environment to a separate Kubernetes namespace'
optional :management_project_id, type: Integer, desc: 'The ID of the management project'
optional :platform_kubernetes_attributes, type: Hash, desc: %q(Platform Kubernetes data) do
optional :api_url, type: String, desc: 'URL to access the Kubernetes API'
diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb
index 8f2a62bc5a4..d565531d372 100644
--- a/lib/api/project_container_repositories.rb
+++ b/lib/api/project_container_repositories.rb
@@ -1,13 +1,13 @@
# frozen_string_literal: true
module API
- class ProjectContainerRepositories < Grape::API::Instance
+ class ProjectContainerRepositories < ::API::Base
include PaginationParams
+ helpers ::API::Helpers::PackagesHelpers
REPOSITORY_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(
tag_name: API::NO_SLASH_URL_PART_REGEX)
- before { error!('404 Not Found', 404) unless Feature.enabled?(:container_registry_api, user_project, default_enabled: true) }
before { authorize_read_container_images! }
params do
@@ -28,7 +28,7 @@ module API
user: current_user, subject: user_project
).execute
- track_event( 'list_repositories')
+ track_package_event('list_repositories', :container)
present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count]
end
@@ -43,7 +43,7 @@ module API
authorize_admin_container_image!
DeleteContainerRepositoryWorker.perform_async(current_user.id, repository.id) # rubocop:disable CodeReuse/Worker
- track_event('delete_repository')
+ track_package_event('delete_repository', :container)
status :accepted
end
@@ -60,7 +60,7 @@ module API
authorize_read_container_image!
tags = Kaminari.paginate_array(repository.tags)
- track_event('list_tags')
+ track_package_event('list_tags', :container)
present paginate(tags), with: Entities::ContainerRegistry::Tag
end
@@ -89,7 +89,7 @@ module API
declared_params.except(:repository_id).merge(container_expiration_policy: false))
# rubocop:enable CodeReuse/Worker
- track_event('delete_tag_bulk')
+ track_package_event('delete_tag_bulk', :container)
status :accepted
end
@@ -125,7 +125,7 @@ module API
.execute(repository)
if result[:status] == :success
- track_event('delete_tag')
+ track_package_event('delete_tag', :container)
status :ok
else
diff --git a/lib/api/project_events.rb b/lib/api/project_events.rb
index 726e693826e..3765473bc0e 100644
--- a/lib/api/project_events.rb
+++ b/lib/api/project_events.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectEvents < Grape::API::Instance
+ class ProjectEvents < ::API::Base
include PaginationParams
include APIGuard
helpers ::API::Helpers::EventsHelpers
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index 377d61689b3..184f89200ab 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectExport < Grape::API::Instance
+ class ProjectExport < ::API::Base
helpers Helpers::RateLimiter
before do
@@ -55,7 +55,7 @@ module API
export_strategy = if after_export_params[:url].present?
params = after_export_params.slice(:url, :http_method).symbolize_keys
- Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(params)
+ Gitlab::ImportExport::AfterExportStrategies::WebUploadStrategy.new(**params)
end
if export_strategy&.invalid?
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index e68a3b106b1..bc2d8c816a8 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectHooks < Grape::API::Instance
+ class ProjectHooks < ::API::Base
include PaginationParams
before { authenticate! }
@@ -104,7 +104,9 @@ module API
delete ":id/hooks/:hook_id" do
hook = user_project.hooks.find(params.delete(:hook_id))
- destroy_conditionally!(hook)
+ destroy_conditionally!(hook) do
+ WebHooks::DestroyService.new(current_user).execute(hook)
+ end
end
end
end
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index 9f43c3c7993..5c4e1d73ee1 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -1,11 +1,9 @@
# frozen_string_literal: true
module API
- class ProjectImport < Grape::API::Instance
+ class ProjectImport < ::API::Base
include PaginationParams
- MAXIMUM_FILE_SIZE = 50.megabytes
-
helpers Helpers::ProjectsHelpers
helpers Helpers::FileUploadHelpers
helpers Helpers::RateLimiter
diff --git a/lib/api/project_milestones.rb b/lib/api/project_milestones.rb
index 2f8dd1085dc..a81118f44bd 100644
--- a/lib/api/project_milestones.rb
+++ b/lib/api/project_milestones.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectMilestones < Grape::API::Instance
+ class ProjectMilestones < ::API::Base
include PaginationParams
include MilestoneResponses
diff --git a/lib/api/project_packages.rb b/lib/api/project_packages.rb
index 359514f1f78..b8d97b1243a 100644
--- a/lib/api/project_packages.rb
+++ b/lib/api/project_packages.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectPackages < Grape::API::Instance
+ class ProjectPackages < ::API::Base
include PaginationParams
before do
diff --git a/lib/api/project_repository_storage_moves.rb b/lib/api/project_repository_storage_moves.rb
index c318907542b..38eb74663d3 100644
--- a/lib/api/project_repository_storage_moves.rb
+++ b/lib/api/project_repository_storage_moves.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectRepositoryStorageMoves < Grape::API::Instance
+ class ProjectRepositoryStorageMoves < ::API::Base
include PaginationParams
before { authenticated_as_admin! }
@@ -69,7 +69,7 @@ module API
success Entities::ProjectRepositoryStorageMove
end
params do
- requires :destination_storage_name, type: String, desc: 'The destination storage shard'
+ optional :destination_storage_name, type: String, desc: 'The destination storage shard'
end
post ':id/repository_storage_moves' do
storage_move = user_project.repository_storage_moves.build(
diff --git a/lib/api/project_snapshots.rb b/lib/api/project_snapshots.rb
index 360000861fc..e19afb6e8e4 100644
--- a/lib/api/project_snapshots.rb
+++ b/lib/api/project_snapshots.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectSnapshots < Grape::API::Instance
+ class ProjectSnapshots < ::API::Base
helpers ::API::Helpers::ProjectSnapshotsHelpers
before { authorize_read_git_snapshot! }
diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb
index f6e87fece89..b4de260fe49 100644
--- a/lib/api/project_snippets.rb
+++ b/lib/api/project_snippets.rb
@@ -1,10 +1,9 @@
# frozen_string_literal: true
module API
- class ProjectSnippets < Grape::API::Instance
+ class ProjectSnippets < ::API::Base
include PaginationParams
- before { authenticate! }
before { check_snippets_enabled }
params do
@@ -37,6 +36,8 @@ module API
use :pagination
end
get ":id/snippets" do
+ authenticate!
+
present paginate(snippets_for_current_user), with: Entities::ProjectSnippet, current_user: current_user
end
@@ -48,6 +49,9 @@ module API
end
get ":id/snippets/:snippet_id" do
snippet = snippets_for_current_user.find(params[:snippet_id])
+
+ not_found!('Snippet') unless snippet
+
present snippet, with: Entities::ProjectSnippet, current_user: current_user
end
@@ -63,6 +67,8 @@ module API
use :create_file_params
end
post ":id/snippets" do
+ authenticate!
+
authorize! :create_snippet, user_project
snippet_params = process_create_params(declared_params(include_missing: false))
@@ -97,6 +103,8 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
put ":id/snippets/:snippet_id" do
+ authenticate!
+
snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id))
not_found!('Snippet') unless snippet
@@ -125,6 +133,8 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
delete ":id/snippets/:snippet_id" do
+ authenticate!
+
snippet = snippets_for_current_user.find_by(id: params[:snippet_id])
not_found!('Snippet') unless snippet
diff --git a/lib/api/project_statistics.rb b/lib/api/project_statistics.rb
index 2196801096f..1ead969fc81 100644
--- a/lib/api/project_statistics.rb
+++ b/lib/api/project_statistics.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectStatistics < Grape::API::Instance
+ class ProjectStatistics < ::API::Base
before do
authenticate!
authorize! :daily_statistics, user_project
diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb
index 48c3dbed3b0..7d851de0237 100644
--- a/lib/api/project_templates.rb
+++ b/lib/api/project_templates.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProjectTemplates < Grape::API::Instance
+ class ProjectTemplates < ::API::Base
include PaginationParams
TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses metrics_dashboard_ymls issues merge_requests].freeze
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index abbdb11a3f7..ecee76ae60c 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -3,7 +3,7 @@
require_dependency 'declarative_policy'
module API
- class Projects < Grape::API::Instance
+ class Projects < ::API::Base
include PaginationParams
include Helpers::CustomAttributes
@@ -353,7 +353,7 @@ module API
optional :path, type: String, desc: 'The path of the repository'
use :optional_project_params
- use :optional_update_params_ee
+ use :optional_update_params
at_least_one_of(*Helpers::ProjectsHelpers.update_params_at_least_one_of)
end
diff --git a/lib/api/protected_branches.rb b/lib/api/protected_branches.rb
index b0a7f898eec..a448682d8bd 100644
--- a/lib/api/protected_branches.rb
+++ b/lib/api/protected_branches.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProtectedBranches < Grape::API::Instance
+ class ProtectedBranches < ::API::Base
include PaginationParams
BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX)
diff --git a/lib/api/protected_tags.rb b/lib/api/protected_tags.rb
index aaa31cb7cc6..dd3e407ffc9 100644
--- a/lib/api/protected_tags.rb
+++ b/lib/api/protected_tags.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ProtectedTags < Grape::API::Instance
+ class ProtectedTags < ::API::Base
include PaginationParams
TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: API::NO_SLASH_URL_PART_REGEX)
diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb
index c07db68f8a8..5622bc6e42d 100644
--- a/lib/api/pypi_packages.rb
+++ b/lib/api/pypi_packages.rb
@@ -6,7 +6,7 @@
# called by the PyPI package manager client when users run commands
# like `pip install` or `twine upload`.
module API
- class PypiPackages < Grape::API::Instance
+ class PypiPackages < ::API::Base
helpers ::API::Helpers::PackagesManagerClientsHelpers
helpers ::API::Helpers::RelatedResourcesHelpers
helpers ::API::Helpers::Packages::BasicAuthHelpers
@@ -33,7 +33,7 @@ module API
def find_package_versions
packages = packages_finder
- .with_name(params[:package_name])
+ .with_normalized_pypi_name(params[:package_name])
not_found!('Package') if packages.empty?
@@ -72,7 +72,7 @@ module API
package = packages_finder(project).by_file_name_and_sha256(filename, params[:sha256])
package_file = ::Packages::PackageFileFinder.new(package, filename, with_file_name_like: false).execute
- package_event('pull_package')
+ track_package_event('pull_package', :pypi)
present_carrierwave_file!(package_file.file, supports_direct_download: true)
end
@@ -91,7 +91,7 @@ module API
get 'simple/*package_name', format: :txt do
authorize_read_package!(authorized_user_project)
- package_event('list_package')
+ track_package_event('list_package', :pypi)
packages = find_package_versions
presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project)
@@ -122,7 +122,7 @@ module API
authorize_upload!(authorized_user_project)
bad_request!('File is too large') if authorized_user_project.actual_limits.exceeded?(:pypi_max_file_size, params[:content].size)
- package_event('push_package')
+ track_package_event('push_package', :pypi)
::Packages::Pypi::CreatePackageService
.new(authorized_user_project, current_user, declared_params)
diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb
index 9624b8924e5..23de9f9fc9f 100644
--- a/lib/api/release/links.rb
+++ b/lib/api/release/links.rb
@@ -2,7 +2,7 @@
module API
module Release
- class Links < Grape::API::Instance
+ class Links < ::API::Base
include PaginationParams
RELEASE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS
diff --git a/lib/api/releases.rb b/lib/api/releases.rb
index 3c38721129f..3bd6ea77403 100644
--- a/lib/api/releases.rb
+++ b/lib/api/releases.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Releases < Grape::API::Instance
+ class Releases < ::API::Base
include PaginationParams
RELEASE_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS
@@ -19,9 +19,13 @@ module API
end
params do
use :pagination
+ optional :order_by, type: String, values: %w[released_at created_at], default: 'released_at',
+ desc: 'Return releases ordered by `released_at` or `created_at`.'
+ optional :sort, type: String, values: %w[asc desc], default: 'desc',
+ desc: 'Return releases sorted in `asc` or `desc` order.'
end
get ':id/releases' do
- releases = ::ReleasesFinder.new(user_project, current_user).execute
+ releases = ::ReleasesFinder.new(user_project, current_user, declared_params.slice(:order_by, :sort)).execute
present paginate(releases), with: Entities::Release, current_user: current_user
end
@@ -152,7 +156,7 @@ module API
end
def authorize_create_evidence!
- # This is a separate method so that EE can extend its behaviour
+ # extended in EE
end
def release
@@ -160,15 +164,15 @@ module API
end
def log_release_created_audit_event(release)
- # This is a separate method so that EE can extend its behaviour
+ # extended in EE
end
def log_release_updated_audit_event
- # This is a separate method so that EE can extend its behaviour
+ # extended in EE
end
def log_release_milestones_updated_audit_event
- # This is a separate method so that EE can extend its behaviour
+ # extended in EE
end
end
end
diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb
index d1def05808b..f63ea04a529 100644
--- a/lib/api/remote_mirrors.rb
+++ b/lib/api/remote_mirrors.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class RemoteMirrors < Grape::API::Instance
+ class RemoteMirrors < ::API::Base
include PaginationParams
before do
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 81702f8f02a..38ac1f22a48 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -3,7 +3,7 @@
require 'mime/types'
module API
- class Repositories < Grape::API::Instance
+ class Repositories < ::API::Base
include PaginationParams
helpers ::API::Helpers::HeadersHelpers
diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb
index a8d3419528c..d3a219f0810 100644
--- a/lib/api/resource_label_events.rb
+++ b/lib/api/resource_label_events.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ResourceLabelEvents < Grape::API::Instance
+ class ResourceLabelEvents < ::API::Base
include PaginationParams
helpers ::API::Helpers::NotesHelpers
diff --git a/lib/api/resource_milestone_events.rb b/lib/api/resource_milestone_events.rb
index a8f221f8740..21411f68dd5 100644
--- a/lib/api/resource_milestone_events.rb
+++ b/lib/api/resource_milestone_events.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ResourceMilestoneEvents < Grape::API::Instance
+ class ResourceMilestoneEvents < ::API::Base
include PaginationParams
helpers ::API::Helpers::NotesHelpers
diff --git a/lib/api/resource_state_events.rb b/lib/api/resource_state_events.rb
index 1c1a90c09a3..9bfda39be90 100644
--- a/lib/api/resource_state_events.rb
+++ b/lib/api/resource_state_events.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class ResourceStateEvents < Grape::API::Instance
+ class ResourceStateEvents < ::API::Base
include PaginationParams
helpers ::API::Helpers::NotesHelpers
diff --git a/lib/api/search.rb b/lib/api/search.rb
index b9c6a823f4f..85f0a8e2e60 100644
--- a/lib/api/search.rb
+++ b/lib/api/search.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Search < Grape::API::Instance
+ class Search < ::API::Base
include PaginationParams
before { authenticate! }
@@ -33,6 +33,7 @@ module API
scope: params[:scope],
search: params[:search],
state: params[:state],
+ confidential: params[:confidential],
snippets: snippets?,
page: params[:page],
per_page: params[:per_page]
@@ -62,12 +63,6 @@ module API
# Defining this method here as a noop allows us to easily extend it in
# EE, without having to modify this file directly.
end
-
- def check_users_search_allowed!
- if params[:scope].to_sym == :users && Feature.disabled?(:users_search, default_enabled: true)
- render_api_error!({ error: _("Scope not supported with disabled 'users_search' feature!") }, 400)
- end
- end
end
resource :search do
@@ -81,11 +76,11 @@ module API
desc: 'The scope of the search',
values: Helpers::SearchHelpers.global_search_scopes
optional :state, type: String, desc: 'Filter results by state', values: Helpers::SearchHelpers.search_states
+ optional :confidential, type: Boolean, desc: 'Filter results by confidentiality'
use :pagination
end
get do
verify_search_scope!(resource: nil)
- check_users_search_allowed!
present search, with: entity
end
@@ -103,11 +98,11 @@ module API
desc: 'The scope of the search',
values: Helpers::SearchHelpers.group_search_scopes
optional :state, type: String, desc: 'Filter results by state', values: Helpers::SearchHelpers.search_states
+ optional :confidential, type: Boolean, desc: 'Filter results by confidentiality'
use :pagination
end
get ':id/(-/)search' do
verify_search_scope!(resource: user_group)
- check_users_search_allowed!
present search(group_id: user_group.id), with: entity
end
@@ -126,11 +121,10 @@ module API
values: Helpers::SearchHelpers.project_search_scopes
optional :ref, type: String, desc: 'The name of a repository branch or tag. If not given, the default branch is used'
optional :state, type: String, desc: 'Filter results by state', values: Helpers::SearchHelpers.search_states
+ optional :confidential, type: Boolean, desc: 'Filter results by confidentiality'
use :pagination
end
get ':id/(-/)search' do
- check_users_search_allowed!
-
present search({ project_id: user_project.id, repository_ref: params[:ref] }), with: entity
end
end
diff --git a/lib/api/services.rb b/lib/api/services.rb
index 9ee1822339c..5f3d14010a8 100644
--- a/lib/api/services.rb
+++ b/lib/api/services.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
module API
- class Services < Grape::API::Instance
+ class Services < ::API::Base
services = Helpers::ServicesHelpers.services
service_classes = Helpers::ServicesHelpers.service_classes
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 6e5534d0c9a..dc917d9c529 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Settings < Grape::API::Instance
+ class Settings < ::API::Base
before { authenticated_as_admin! }
helpers Helpers::SettingsHelpers
@@ -29,7 +29,8 @@ module API
success Entities::ApplicationSetting
end
params do
- optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.'
+ optional :admin_notification_email, type: String, desc: 'Deprecated: Use :abuse_notification_email instead. Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.'
+ optional :abuse_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.'
optional :after_sign_up_text, type: String, desc: 'Text shown after sign up'
optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out'
optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues'
@@ -73,6 +74,7 @@ module API
optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled'
optional :help_page_hide_commercial_content, type: Boolean, desc: 'Hide marketing-related entries from help'
optional :help_page_support_url, type: String, desc: 'Alternate support URL for help page and help dropdown'
+ optional :help_page_documentation_base_url, type: String, desc: 'Alternate documentation pages URL'
optional :help_page_text, type: String, desc: 'Custom text displayed on the help page'
optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page'
optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)'
@@ -194,6 +196,11 @@ module API
attrs[:allow_local_requests_from_web_hooks_and_services] = attrs.delete(:allow_local_requests_from_hooks_and_services)
end
+ # support legacy names, can be removed in v5
+ if attrs.has_key?(:admin_notification_email)
+ attrs[:abuse_notification_email] = attrs.delete(:admin_notification_email)
+ end
+
# since 13.0 it's not possible to disable hashed storage - support can be removed in 14.0
attrs.delete(:hashed_storage_enabled) if attrs.has_key?(:hashed_storage_enabled)
diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb
index 77f2b1e871e..b025dbfab37 100644
--- a/lib/api/sidekiq_metrics.rb
+++ b/lib/api/sidekiq_metrics.rb
@@ -3,7 +3,7 @@
require 'sidekiq/api'
module API
- class SidekiqMetrics < Grape::API::Instance
+ class SidekiqMetrics < ::API::Base
before { authenticated_as_admin! }
helpers do
diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb
index c6ef35875fc..2e67b9649bc 100644
--- a/lib/api/snippets.rb
+++ b/lib/api/snippets.rb
@@ -2,11 +2,9 @@
module API
# Snippets API
- class Snippets < Grape::API::Instance
+ class Snippets < ::API::Base
include PaginationParams
- before { authenticate! }
-
resource :snippets do
helpers Helpers::SnippetsHelpers
helpers do
@@ -23,7 +21,7 @@ module API
end
end
- desc 'Get a snippets list for authenticated user' do
+ desc 'Get a snippets list for an authenticated user' do
detail 'This feature was introduced in GitLab 8.15.'
success Entities::Snippet
end
@@ -31,6 +29,8 @@ module API
use :pagination
end
get do
+ authenticate!
+
present paginate(snippets_for_current_user), with: Entities::Snippet, current_user: current_user
end
@@ -42,6 +42,8 @@ module API
use :pagination
end
get 'public' do
+ authenticate!
+
present paginate(public_snippets), with: Entities::PersonalSnippet, current_user: current_user
end
@@ -74,6 +76,8 @@ module API
use :create_file_params
end
post do
+ authenticate!
+
authorize! :create_snippet
attrs = process_create_params(declared_params(include_missing: false))
@@ -109,6 +113,8 @@ module API
use :minimum_update_params
end
put ':id' do
+ authenticate!
+
snippet = snippets_for_current_user.find_by_id(params.delete(:id))
break not_found!('Snippet') unless snippet
@@ -139,6 +145,8 @@ module API
requires :id, type: Integer, desc: 'The ID of a snippet'
end
delete ':id' do
+ authenticate!
+
snippet = snippets_for_current_user.find_by_id(params.delete(:id))
break not_found!('Snippet') unless snippet
diff --git a/lib/api/statistics.rb b/lib/api/statistics.rb
index 3869fd3ac76..fa7176491ba 100644
--- a/lib/api/statistics.rb
+++ b/lib/api/statistics.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Statistics < Grape::API::Instance
+ class Statistics < ::API::Base
before { authenticated_as_admin! }
COUNTED_ITEMS = [Project, User, Group, ForkNetworkMember, ForkNetwork, Issue,
diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb
index 34d21d3d7d8..e2ceb49c119 100644
--- a/lib/api/submodules.rb
+++ b/lib/api/submodules.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Submodules < Grape::API::Instance
+ class Submodules < ::API::Base
before { authenticate! }
helpers do
diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb
index 533663fb087..35a28da4736 100644
--- a/lib/api/subscriptions.rb
+++ b/lib/api/subscriptions.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Subscriptions < Grape::API::Instance
+ class Subscriptions < ::API::Base
helpers ::API::Helpers::LabelHelpers
before { authenticate! }
diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb
index 38e96c080f2..f23d279c3f4 100644
--- a/lib/api/suggestions.rb
+++ b/lib/api/suggestions.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Suggestions < Grape::API::Instance
+ class Suggestions < ::API::Base
before { authenticate! }
resource :suggestions do
diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb
index d8e0a425625..2820d305d0f 100644
--- a/lib/api/system_hooks.rb
+++ b/lib/api/system_hooks.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class SystemHooks < Grape::API::Instance
+ class SystemHooks < ::API::Base
include PaginationParams
before do
@@ -70,7 +70,9 @@ module API
hook = SystemHook.find_by(id: params[:id])
not_found!('System hook') unless hook
- destroy_conditionally!(hook)
+ destroy_conditionally!(hook) do
+ WebHooks::DestroyService.new(current_user).execute(hook)
+ end
end
# rubocop: enable CodeReuse/ActiveRecord
end
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index c1fbd3ca7c6..b969394ec47 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Tags < Grape::API::Instance
+ class Tags < ::API::Base
include PaginationParams
TAG_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(tag_name: API::NO_SLASH_URL_PART_REGEX)
diff --git a/lib/api/templates.rb b/lib/api/templates.rb
index 80a97aae429..0b427bbf5b9 100644
--- a/lib/api/templates.rb
+++ b/lib/api/templates.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Templates < Grape::API::Instance
+ class Templates < ::API::Base
include PaginationParams
GLOBAL_TEMPLATE_TYPES = {
diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb
index 7063a3d08b5..4168cce21ef 100644
--- a/lib/api/terraform/state.rb
+++ b/lib/api/terraform/state.rb
@@ -4,7 +4,7 @@ require_dependency 'api/validations/validators/limit'
module API
module Terraform
- class State < Grape::API::Instance
+ class State < ::API::Base
include ::Gitlab::Utils::StrongMemoize
default_format :json
diff --git a/lib/api/terraform/state_version.rb b/lib/api/terraform/state_version.rb
new file mode 100644
index 00000000000..b4a0efd7a2b
--- /dev/null
+++ b/lib/api/terraform/state_version.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module API
+ module Terraform
+ class StateVersion < ::API::Base
+ default_format :json
+
+ before do
+ authenticate!
+ authorize! :read_terraform_state, user_project
+ end
+
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ namespace ':id/terraform/state/:name/versions/:serial' do
+ params do
+ requires :name, type: String, desc: 'The name of a Terraform state'
+ requires :serial, type: Integer, desc: 'The version number of the state'
+ end
+
+ helpers do
+ def remote_state_handler
+ ::Terraform::RemoteStateHandler.new(user_project, current_user, name: params[:name])
+ end
+
+ def find_version(serial)
+ remote_state_handler.find_with_lock do |state|
+ version = state.versions.find_by_version(serial)
+
+ if version.present?
+ yield version
+ else
+ not_found!
+ end
+ end
+ end
+ end
+
+ desc 'Get a terraform state version'
+ route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+ get do
+ find_version(params[:serial]) do |version|
+ env['api.format'] = :binary # Bypass json serialization
+ body version.file.read
+ status :ok
+ end
+ end
+
+ desc 'Delete a terraform state version'
+ route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
+ delete do
+ authorize! :admin_terraform_state, user_project
+
+ find_version(params[:serial]) do |version|
+ version.destroy!
+
+ body false
+ status :no_content
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/todos.rb b/lib/api/todos.rb
index 5eae92a251e..ce07d13cc9a 100644
--- a/lib/api/todos.rb
+++ b/lib/api/todos.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Todos < Grape::API::Instance
+ class Todos < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
index f398bbf3e32..960d004a04c 100644
--- a/lib/api/triggers.rb
+++ b/lib/api/triggers.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Triggers < Grape::API::Instance
+ class Triggers < ::API::Base
include PaginationParams
HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase
diff --git a/lib/api/unleash.rb b/lib/api/unleash.rb
new file mode 100644
index 00000000000..907422118f1
--- /dev/null
+++ b/lib/api/unleash.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module API
+ class Unleash < ::API::Base
+ include PaginationParams
+
+ namespace :feature_flags do
+ resource :unleash, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ params do
+ requires :project_id, type: String, desc: 'The ID of a project'
+ optional :instance_id, type: String, desc: 'The Instance ID of Unleash Client'
+ optional :app_name, type: String, desc: 'The Application Name of Unleash Client'
+ end
+ route_param :project_id do
+ before do
+ authorize_by_unleash_instance_id!
+ end
+
+ get do
+ # not supported yet
+ status :ok
+ end
+
+ desc 'Get a list of features (deprecated, v2 client support)'
+ get 'features' do
+ present :version, 1
+ present :features, feature_flags, with: ::API::Entities::UnleashFeature
+ end
+
+ desc 'Get a list of features'
+ get 'client/features' do
+ present :version, 1
+ present :features, feature_flags, with: ::API::Entities::UnleashFeature
+ end
+
+ post 'client/register' do
+ # not supported yet
+ status :ok
+ end
+
+ post 'client/metrics' do
+ # not supported yet
+ status :ok
+ end
+ end
+ end
+ end
+
+ helpers do
+ def project
+ @project ||= find_project(params[:project_id])
+ end
+
+ def unleash_instance_id
+ env['HTTP_UNLEASH_INSTANCEID'] || params[:instance_id]
+ end
+
+ def unleash_app_name
+ env['HTTP_UNLEASH_APPNAME'] || params[:app_name]
+ end
+
+ def authorize_by_unleash_instance_id!
+ unauthorized! unless Operations::FeatureFlagsClient
+ .find_for_project_and_token(project, unleash_instance_id)
+ end
+
+ def feature_flags
+ return [] unless unleash_app_name.present?
+
+ legacy_flags = Operations::FeatureFlagScope.for_unleash_client(project, unleash_app_name)
+ new_version_flags = Operations::FeatureFlag.for_unleash_client(project, unleash_app_name)
+
+ legacy_flags + new_version_flags
+ end
+ end
+ end
+end
diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb
index a1512197ee1..fa5bfc1cbe9 100644
--- a/lib/api/usage_data.rb
+++ b/lib/api/usage_data.rb
@@ -1,12 +1,12 @@
# frozen_string_literal: true
module API
- class UsageData < Grape::API::Instance
+ class UsageData < ::API::Base
before { authenticate! }
namespace 'usage_data' do
before do
- not_found! unless Feature.enabled?(:usage_data_api)
+ not_found! unless Feature.enabled?(:usage_data_api, default_enabled: true)
forbidden!('Invalid CSRF token is provided') unless verified_request?
end
diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb
index 90127ecbc73..6d9db53fec8 100644
--- a/lib/api/user_counts.rb
+++ b/lib/api/user_counts.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class UserCounts < Grape::API::Instance
+ class UserCounts < ::API::Base
resource :user_counts do
desc 'Return the user specific counts' do
detail 'Open MR Count'
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 73bb43b88fc..e7c1d644324 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Users < Grape::API::Instance
+ class Users < ::API::Base
include PaginationParams
include APIGuard
include Helpers::CustomAttributes
@@ -348,7 +348,7 @@ module API
end
# rubocop: enable CodeReuse/ActiveRecord
- desc 'Get the GPG keys of a specified user. Available only for admins.' do
+ desc 'Get the GPG keys of a specified user.' do
detail 'This feature was added in GitLab 10.0'
success Entities::GpgKey
end
@@ -358,8 +358,6 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
get ':id/gpg_keys' do
- authenticated_as_admin!
-
user = User.find_by(id: params[:id])
not_found!('User') unless user
@@ -367,6 +365,26 @@ module API
end
# rubocop: enable CodeReuse/ActiveRecord
+ desc 'Get a specific GPG key for a given user.' do
+ detail 'This feature was added in GitLab 13.5'
+ success Entities::GpgKey
+ end
+ params do
+ requires :id, type: Integer, desc: 'The ID of the user'
+ requires :key_id, type: Integer, desc: 'The ID of the GPG key'
+ end
+ # rubocop: disable CodeReuse/ActiveRecord
+ get ':id/gpg_keys/:key_id' do
+ user = User.find_by(id: params[:id])
+ not_found!('User') unless user
+
+ key = user.gpg_keys.find_by(id: params[:key_id])
+ not_found!('GPG Key') unless key
+
+ present key, with: Entities::GpgKey
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
desc 'Delete an existing GPG key from a specified user. Available only for admins.' do
detail 'This feature was added in GitLab 10.0'
end
@@ -529,10 +547,15 @@ module API
unless user.can_be_deactivated?
forbidden!('A blocked user cannot be deactivated by the API') if user.blocked?
+ forbidden!('An internal user cannot be deactivated by the API') if user.internal?
forbidden!("The user you are trying to deactivate has been active in the past #{::User::MINIMUM_INACTIVE_DAYS} days and cannot be deactivated")
end
- user.deactivate
+ if user.deactivate
+ true
+ else
+ render_api_error!(user.errors.full_messages, 400)
+ end
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb
index 593f90460ac..aed88e6091c 100644
--- a/lib/api/v3/github.rb
+++ b/lib/api/v3/github.rb
@@ -7,7 +7,7 @@
#
module API
module V3
- class Github < Grape::API::Instance
+ class Github < ::API::Base
NO_SLASH_URL_PART_REGEX = %r{[^/]+}.freeze
ENDPOINT_REQUIREMENTS = {
namespace: NO_SLASH_URL_PART_REGEX,
@@ -51,7 +51,7 @@ module API
def find_project_with_access(params)
project = find_project!(
- ::Gitlab::Jira::Dvcs.restore_full_path(params.slice(:namespace, :project).symbolize_keys)
+ ::Gitlab::Jira::Dvcs.restore_full_path(**params.slice(:namespace, :project).symbolize_keys)
)
not_found! unless can?(current_user, :download_code, project)
project
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index 0b3ec10f1b4..f5de3d844e6 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Variables < Grape::API::Instance
+ class Variables < ::API::Base
include PaginationParams
before { authenticate! }
diff --git a/lib/api/version.rb b/lib/api/version.rb
index 6a480fc2bd9..841b55f8d6c 100644
--- a/lib/api/version.rb
+++ b/lib/api/version.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Version < Grape::API::Instance
+ class Version < ::API::Base
helpers ::API::Helpers::GraphqlHelpers
include APIGuard
diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb
index 4eba12157bd..21f457046f1 100644
--- a/lib/api/wikis.rb
+++ b/lib/api/wikis.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module API
- class Wikis < Grape::API::Instance
+ class Wikis < ::API::Base
helpers ::API::Helpers::WikisHelpers
helpers do
diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb
index c2266f0bad6..6a45baa60ec 100644
--- a/lib/backup/artifacts.rb
+++ b/lib/backup/artifacts.rb
@@ -1,9 +1,7 @@
# frozen_string_literal: true
-require 'backup/files'
-
module Backup
- class Artifacts < Files
+ class Artifacts < Backup::Files
attr_reader :progress
def initialize(progress)
diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb
index 5e795a449de..9c3b7165de7 100644
--- a/lib/backup/builds.rb
+++ b/lib/backup/builds.rb
@@ -1,9 +1,7 @@
# frozen_string_literal: true
-require 'backup/files'
-
module Backup
- class Builds < Files
+ class Builds < Backup::Files
attr_reader :progress
def initialize(progress)
diff --git a/lib/backup/lfs.rb b/lib/backup/lfs.rb
index 0dfe56e214f..514d52d7f65 100644
--- a/lib/backup/lfs.rb
+++ b/lib/backup/lfs.rb
@@ -1,9 +1,7 @@
# frozen_string_literal: true
-require 'backup/files'
-
module Backup
- class Lfs < Files
+ class Lfs < Backup::Files
attr_reader :progress
def initialize(progress)
diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb
index d7aab33d7cb..ae293073ba2 100644
--- a/lib/backup/pages.rb
+++ b/lib/backup/pages.rb
@@ -1,9 +1,7 @@
# frozen_string_literal: true
-require 'backup/files'
-
module Backup
- class Pages < Files
+ class Pages < Backup::Files
attr_reader :progress
def initialize(progress)
diff --git a/lib/backup/registry.rb b/lib/backup/registry.rb
index d16ed2facf1..9645a07dfb8 100644
--- a/lib/backup/registry.rb
+++ b/lib/backup/registry.rb
@@ -1,9 +1,7 @@
# frozen_string_literal: true
-require 'backup/files'
-
module Backup
- class Registry < Files
+ class Registry < Backup::Files
attr_reader :progress
def initialize(progress)
diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb
new file mode 100644
index 00000000000..4248a86dc7c
--- /dev/null
+++ b/lib/backup/repositories.rb
@@ -0,0 +1,310 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Backup
+ class Repositories
+ attr_reader :progress
+
+ def initialize(progress)
+ @progress = progress
+ end
+
+ def dump(max_concurrency:, max_storage_concurrency:)
+ prepare
+
+ if max_concurrency <= 1 && max_storage_concurrency <= 1
+ return dump_consecutive
+ end
+
+ check_valid_storages!
+
+ semaphore = Concurrent::Semaphore.new(max_concurrency)
+ errors = Queue.new
+
+ threads = Gitlab.config.repositories.storages.keys.map do |storage|
+ Thread.new do
+ Rails.application.executor.wrap do
+ dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency)
+ rescue => e
+ errors << e
+ end
+ end
+ end
+
+ ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+ threads.each(&:join)
+ end
+
+ raise errors.pop unless errors.empty?
+ end
+
+ def restore
+ Project.find_each(batch_size: 1000) do |project|
+ restore_repository(project, Gitlab::GlRepository::PROJECT)
+ restore_repository(project, Gitlab::GlRepository::WIKI)
+ restore_repository(project, Gitlab::GlRepository::DESIGN)
+ end
+
+ invalid_ids = Snippet.find_each(batch_size: 1000)
+ .map { |snippet| restore_snippet_repository(snippet) }
+ .compact
+
+ cleanup_snippets_without_repositories(invalid_ids)
+
+ restore_object_pools
+ end
+
+ private
+
+ def check_valid_storages!
+ [ProjectRepository, SnippetRepository].each do |klass|
+ if klass.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists?
+ raise Error, "repositories.storages in gitlab.yml does not include all storages used by #{klass}"
+ end
+ end
+ end
+
+ def backup_repos_path
+ @backup_repos_path ||= File.join(Gitlab.config.backup.path, 'repositories')
+ end
+
+ def prepare
+ FileUtils.rm_rf(backup_repos_path)
+ FileUtils.mkdir_p(Gitlab.config.backup.path)
+ FileUtils.mkdir(backup_repos_path, mode: 0700)
+ end
+
+ def dump_consecutive
+ dump_consecutive_projects
+ dump_consecutive_snippets
+ end
+
+ def dump_consecutive_projects
+ project_relation.find_each(batch_size: 1000) do |project|
+ dump_project(project)
+ end
+ end
+
+ def dump_consecutive_snippets
+ Snippet.find_each(batch_size: 1000) { |snippet| dump_snippet(snippet) }
+ end
+
+ def dump_storage(storage, semaphore, max_storage_concurrency:)
+ errors = Queue.new
+ queue = InterlockSizedQueue.new(1)
+
+ threads = Array.new(max_storage_concurrency) do
+ Thread.new do
+ Rails.application.executor.wrap do
+ while container = queue.pop
+ ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+ semaphore.acquire
+ end
+
+ begin
+ case container
+ when Project
+ dump_project(container)
+ when Snippet
+ dump_snippet(container)
+ end
+ rescue => e
+ errors << e
+ break
+ ensure
+ semaphore.release
+ end
+ end
+ end
+ end
+ end
+
+ enqueue_records_for_storage(storage, queue, errors)
+
+ raise errors.pop unless errors.empty?
+ ensure
+ queue.close
+ ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+ threads.each(&:join)
+ end
+ end
+
+ def dump_project(project)
+ backup_repository(project, Gitlab::GlRepository::PROJECT)
+ backup_repository(project, Gitlab::GlRepository::WIKI)
+ backup_repository(project, Gitlab::GlRepository::DESIGN)
+ end
+
+ def dump_snippet(snippet)
+ backup_repository(snippet, Gitlab::GlRepository::SNIPPET)
+ end
+
+ def enqueue_records_for_storage(storage, queue, errors)
+ records_to_enqueue(storage).each do |relation|
+ relation.find_each(batch_size: 100) do |project|
+ break unless errors.empty?
+
+ queue.push(project)
+ end
+ end
+ end
+
+ def records_to_enqueue(storage)
+ [projects_in_storage(storage), snippets_in_storage(storage)]
+ end
+
+ def projects_in_storage(storage)
+ project_relation.id_in(ProjectRepository.for_repository_storage(storage).select(:project_id))
+ end
+
+ def project_relation
+ Project.includes(:route, :group, namespace: :owner)
+ end
+
+ def snippets_in_storage(storage)
+ Snippet.id_in(SnippetRepository.for_repository_storage(storage).select(:snippet_id))
+ end
+
+ def backup_repository(container, type)
+ BackupRestore.new(
+ progress,
+ type.repository_for(container),
+ backup_repos_path
+ ).backup
+ end
+
+ def restore_repository(container, type)
+ BackupRestore.new(
+ progress,
+ type.repository_for(container),
+ backup_repos_path
+ ).restore(always_create: type.project?)
+ end
+
+ def restore_object_pools
+ PoolRepository.includes(:source_project).find_each do |pool|
+ progress.puts " - Object pool #{pool.disk_path}..."
+
+ pool.source_project ||= pool.member_projects.first.root_of_fork_network
+ pool.state = 'none'
+ pool.save
+
+ pool.schedule
+ end
+ end
+
+ def restore_snippet_repository(snippet)
+ restore_repository(snippet, Gitlab::GlRepository::SNIPPET)
+
+ response = Snippets::RepositoryValidationService.new(nil, snippet).execute
+
+ if response.error?
+ snippet.repository.remove
+
+ progress.puts("Snippet #{snippet.full_path} can't be restored: #{response.message}")
+
+ snippet.id
+ else
+ nil
+ end
+ end
+
+ # Snippets without a repository should be removed because they failed to import
+ # due to having invalid repositories
+ def cleanup_snippets_without_repositories(ids)
+ Snippet.id_in(ids).delete_all
+ end
+
+ class BackupRestore
+ attr_accessor :progress, :repository, :backup_repos_path
+
+ def initialize(progress, repository, backup_repos_path)
+ @progress = progress
+ @repository = repository
+ @backup_repos_path = backup_repos_path
+ end
+
+ def backup
+ progress.puts " * #{display_repo_path} ... "
+
+ if repository.empty?
+ progress.puts " * #{display_repo_path} ... " + "[SKIPPED]".color(:cyan)
+ return
+ end
+
+ FileUtils.mkdir_p(repository_backup_path)
+
+ repository.bundle_to_disk(path_to_bundle)
+ repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar)
+
+ progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green)
+
+ rescue => e
+ progress.puts "[Failed] backing up #{display_repo_path}".color(:red)
+ progress.puts "Error #{e}".color(:red)
+ end
+
+ def restore(always_create: false)
+ progress.puts " * #{display_repo_path} ... "
+
+ repository.remove rescue nil
+
+ if File.exist?(path_to_bundle)
+ repository.create_from_bundle(path_to_bundle)
+ restore_custom_hooks
+ elsif always_create
+ repository.create_repository
+ end
+
+ progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green)
+
+ rescue => e
+ progress.puts "[Failed] restoring #{display_repo_path}".color(:red)
+ progress.puts "Error #{e}".color(:red)
+ end
+
+ private
+
+ def display_repo_path
+ "#{repository.full_path} (#{repository.disk_path})"
+ end
+
+ def repository_backup_path
+ @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path)
+ end
+
+ def path_to_bundle
+ @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle')
+ end
+
+ def restore_custom_hooks
+ return unless File.exist?(custom_hooks_tar)
+
+ repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar)
+ end
+
+ def custom_hooks_tar
+ File.join(repository_backup_path, "custom_hooks.tar")
+ end
+ end
+
+ class InterlockSizedQueue < SizedQueue
+ extend ::Gitlab::Utils::Override
+
+ override :pop
+ def pop(*)
+ ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+ super
+ end
+ end
+
+ override :push
+ def push(*)
+ ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+ super
+ end
+ end
+ end
+ end
+end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
deleted file mode 100644
index eb0b230904e..00000000000
--- a/lib/backup/repository.rb
+++ /dev/null
@@ -1,265 +0,0 @@
-# frozen_string_literal: true
-
-require 'yaml'
-
-module Backup
- class Repository
- attr_reader :progress
-
- def initialize(progress)
- @progress = progress
- end
-
- def dump(max_concurrency:, max_storage_concurrency:)
- prepare
-
- if max_concurrency <= 1 && max_storage_concurrency <= 1
- return dump_consecutive
- end
-
- if Project.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists?
- raise Error, 'repositories.storages in gitlab.yml is misconfigured'
- end
-
- semaphore = Concurrent::Semaphore.new(max_concurrency)
- errors = Queue.new
-
- threads = Gitlab.config.repositories.storages.keys.map do |storage|
- Thread.new do
- Rails.application.executor.wrap do
- dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency)
- rescue => e
- errors << e
- end
- end
- end
-
- ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
- threads.each(&:join)
- end
-
- raise errors.pop unless errors.empty?
- end
-
- def backup_project(project)
- path_to_project_bundle = path_to_bundle(project)
- Gitlab::GitalyClient::RepositoryService.new(project.repository)
- .create_bundle(path_to_project_bundle)
-
- backup_custom_hooks(project)
- rescue => e
- progress_warn(project, e, 'Failed to backup repo')
- end
-
- def backup_custom_hooks(project)
- FileUtils.mkdir_p(project_backup_path(project))
-
- custom_hooks_path = custom_hooks_tar(project)
- Gitlab::GitalyClient::RepositoryService.new(project.repository)
- .backup_custom_hooks(custom_hooks_path)
- end
-
- def restore_custom_hooks(project)
- return unless Dir.exist?(project_backup_path(project))
- return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none?
-
- custom_hooks_path = custom_hooks_tar(project)
- Gitlab::GitalyClient::RepositoryService.new(project.repository)
- .restore_custom_hooks(custom_hooks_path)
- end
-
- def restore
- Project.find_each(batch_size: 1000) do |project|
- progress.print " * #{project.full_path} ... "
-
- restore_repo_success =
- begin
- try_restore_repository(project)
- rescue => err
- progress.puts "Error: #{err}".color(:red)
- false
- end
-
- if restore_repo_success
- progress.puts "[DONE]".color(:green)
- else
- progress.puts "[Failed] restoring #{project.full_path} repository".color(:red)
- end
-
- wiki = ProjectWiki.new(project)
- wiki.repository.remove rescue nil
- path_to_wiki_bundle = path_to_bundle(wiki)
-
- if File.exist?(path_to_wiki_bundle)
- progress.print " * #{wiki.full_path} ... "
- begin
- wiki.repository.create_from_bundle(path_to_wiki_bundle)
- restore_custom_hooks(wiki)
-
- progress.puts "[DONE]".color(:green)
- rescue => e
- progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red)
- progress.puts "Error #{e}".color(:red)
- end
- end
- end
-
- restore_object_pools
- end
-
- protected
-
- def try_restore_repository(project)
- path_to_project_bundle = path_to_bundle(project)
- project.repository.remove rescue nil
-
- if File.exist?(path_to_project_bundle)
- project.repository.create_from_bundle(path_to_project_bundle)
- restore_custom_hooks(project)
- else
- project.repository.create_repository
- end
-
- true
- end
-
- def path_to_bundle(project)
- File.join(backup_repos_path, project.disk_path + '.bundle')
- end
-
- def project_backup_path(project)
- File.join(backup_repos_path, project.disk_path)
- end
-
- def custom_hooks_tar(project)
- File.join(project_backup_path(project), "custom_hooks.tar")
- end
-
- def backup_repos_path
- File.join(Gitlab.config.backup.path, 'repositories')
- end
-
- def prepare
- FileUtils.rm_rf(backup_repos_path)
- FileUtils.mkdir_p(Gitlab.config.backup.path)
- FileUtils.mkdir(backup_repos_path, mode: 0700)
- end
-
- private
-
- def dump_consecutive
- Project.includes(:route, :group, namespace: :owner).find_each(batch_size: 1000) do |project|
- dump_project(project)
- end
- end
-
- def dump_storage(storage, semaphore, max_storage_concurrency:)
- errors = Queue.new
- queue = InterlockSizedQueue.new(1)
-
- threads = Array.new(max_storage_concurrency) do
- Thread.new do
- Rails.application.executor.wrap do
- while project = queue.pop
- ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
- semaphore.acquire
- end
-
- begin
- dump_project(project)
- rescue => e
- errors << e
- break
- ensure
- semaphore.release
- end
- end
- end
- end
- end
-
- Project.for_repository_storage(storage).includes(:route, :group, namespace: :owner).find_each(batch_size: 100) do |project|
- break unless errors.empty?
-
- queue.push(project)
- end
-
- raise errors.pop unless errors.empty?
- ensure
- queue.close
- ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
- threads.each(&:join)
- end
- end
-
- def dump_project(project)
- progress.puts " * #{display_repo_path(project)} ... "
-
- if project.hashed_storage?(:repository)
- FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
- else
- FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
- end
-
- if !empty_repo?(project)
- backup_project(project)
- progress.puts " * #{display_repo_path(project)} ... " + "[DONE]".color(:green)
- else
- progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED]".color(:cyan)
- end
-
- wiki = ProjectWiki.new(project)
-
- if !empty_repo?(wiki)
- backup_project(wiki)
- progress.puts " * #{display_repo_path(project)} ... " + "[DONE] Wiki".color(:green)
- else
- progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED] Wiki".color(:cyan)
- end
- end
-
- def progress_warn(project, cmd, output)
- progress.puts "[WARNING] Executing #{cmd}".color(:orange)
- progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange)
- end
-
- def empty_repo?(project_or_wiki)
- project_or_wiki.repository.expire_emptiness_caches
- project_or_wiki.repository.empty?
- end
-
- def display_repo_path(project)
- project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path
- end
-
- def restore_object_pools
- PoolRepository.includes(:source_project).find_each do |pool|
- progress.puts " - Object pool #{pool.disk_path}..."
-
- pool.source_project ||= pool.member_projects.first.root_of_fork_network
- pool.state = 'none'
- pool.save
-
- pool.schedule
- end
- end
-
- class InterlockSizedQueue < SizedQueue
- extend ::Gitlab::Utils::Override
-
- override :pop
- def pop(*)
- ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
- super
- end
- end
-
- override :push
- def push(*)
- ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
- super
- end
- end
- end
- end
-end
diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb
index b6a62bc3f29..9665624f71b 100644
--- a/lib/backup/uploads.rb
+++ b/lib/backup/uploads.rb
@@ -1,9 +1,7 @@
# frozen_string_literal: true
-require 'backup/files'
-
module Backup
- class Uploads < Files
+ class Uploads < Backup::Files
attr_reader :progress
def initialize(progress)
diff --git a/lib/banzai/filter/design_reference_filter.rb b/lib/banzai/filter/design_reference_filter.rb
index 2ab47c5c6db..1754fec93d4 100644
--- a/lib/banzai/filter/design_reference_filter.rb
+++ b/lib/banzai/filter/design_reference_filter.rb
@@ -3,8 +3,6 @@
module Banzai
module Filter
class DesignReferenceFilter < AbstractReferenceFilter
- FEATURE_FLAG = :design_management_reference_filter_gfm_pipeline
-
class Identifier
include Comparable
attr_reader :issue_iid, :filename
@@ -35,14 +33,6 @@ module Banzai
self.reference_type = :design
- # This filter must be enabled by setting the
- # design_management_reference_filter_gfm_pipeline flag
- def call
- return doc unless enabled?
-
- super
- end
-
def find_object(project, identifier)
records_per_parent[project][identifier]
end
@@ -112,10 +102,6 @@ module Banzai
.in_groups_of(100, false) # limitation of by_issue_id_and_filename, so we batch
.flat_map { |ids| DesignManagement::Design.by_issue_id_and_filename(ids) }
end
-
- def enabled?
- Feature.enabled?(FEATURE_FLAG, parent, default_enabled: true)
- end
end
end
end
diff --git a/lib/banzai/reference_parser.rb b/lib/banzai/reference_parser.rb
index c08d3364a87..8b5d689a984 100644
--- a/lib/banzai/reference_parser.rb
+++ b/lib/banzai/reference_parser.rb
@@ -2,6 +2,8 @@
module Banzai
module ReferenceParser
+ InvalidReferenceType = Class.new(StandardError)
+
# Returns the reference parser class for the given type
#
# Example:
@@ -11,6 +13,8 @@ module Banzai
# This would return the `Banzai::ReferenceParser::IssueParser` class.
def self.[](name)
const_get("#{name.to_s.camelize}Parser", false)
+ rescue NameError
+ raise InvalidReferenceType
end
end
end
diff --git a/lib/banzai/reference_parser/mentioned_group_parser.rb b/lib/banzai/reference_parser/mentioned_group_parser.rb
index a0892e15df8..75d05ef59f9 100644
--- a/lib/banzai/reference_parser/mentioned_group_parser.rb
+++ b/lib/banzai/reference_parser/mentioned_group_parser.rb
@@ -16,7 +16,7 @@ module Banzai
end
def nodes_visible_to_user(user, nodes)
- groups = lazy { grouped_objects_for_nodes(nodes, Group, GROUP_ATTR) }
+ groups = lazy { grouped_objects_for_nodes(nodes, references_relation, GROUP_ATTR) }
nodes.select do |node|
node.has_attribute?(GROUP_ATTR) && can_read_group_reference?(node, user, groups)
diff --git a/lib/banzai/reference_redactor.rb b/lib/banzai/reference_redactor.rb
index 936436982e7..81e4fd45966 100644
--- a/lib/banzai/reference_redactor.rb
+++ b/lib/banzai/reference_redactor.rb
@@ -111,6 +111,7 @@ module Banzai
parser = Banzai::ReferenceParser[type].new(context)
visible.merge(parser.nodes_visible_to_user(user, nodes))
+ rescue Banzai::ReferenceParser::InvalidReferenceType
end
visible
diff --git a/lib/feature.rb b/lib/feature.rb
index 71241e98723..1f8c530bee5 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -138,7 +138,7 @@ class Feature
def register_definitions
return unless check_feature_flags_definition?
- Feature::Definition.load_all!
+ Feature::Definition.reload!
end
def register_hot_reloader
diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb
index ee779a86952..0ba1bdc4799 100644
--- a/lib/feature/definition.rb
+++ b/lib/feature/definition.rb
@@ -84,17 +84,14 @@ class Feature
end
def definitions
- @definitions ||= {}
+ # We lazily load all definitions
+ # The hot reloading might request a feature flag
+ # before we can properly call `load_all!`
+ @definitions ||= load_all!
end
- def load_all!
- definitions.clear
-
- paths.each do |glob_path|
- load_all_from_path!(glob_path)
- end
-
- definitions
+ def reload!
+ @definitions = load_all!
end
def valid_usage!(key, type:, default_enabled:)
@@ -110,9 +107,7 @@ class Feature
def register_hot_reloader!
# Reload feature flags on change of this file or any `.yml`
file_watcher = Rails.configuration.file_watcher.new(reload_files, reload_directories) do
- # We use `Feature::Definition` as on Ruby code-reload
- # a new class definition is created
- Feature::Definition.load_all!
+ Feature::Definition.reload!
end
Rails.application.reloaders << file_watcher
@@ -123,6 +118,16 @@ class Feature
private
+ def load_all!
+ # We currently do not load feature flag definitions
+ # in production environments
+ return [] unless Gitlab.dev_or_test_env?
+
+ paths.each_with_object({}) do |glob_path, definitions|
+ load_all_from_path!(definitions, glob_path)
+ end
+ end
+
def load_from_file(path)
definition = File.read(path)
definition = YAML.safe_load(definition)
@@ -133,7 +138,7 @@ class Feature
raise Feature::InvalidFeatureFlagError, "Invalid definition for `#{path}`: #{e.message}"
end
- def load_all_from_path!(glob_path)
+ def load_all_from_path!(definitions, glob_path)
Dir.glob(glob_path).each do |path|
definition = load_from_file(path)
@@ -146,7 +151,7 @@ class Feature
end
def reload_files
- [File.expand_path(__FILE__)]
+ []
end
def reload_directories
diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb
index c06f699ef27..9ec56ee6b52 100644
--- a/lib/feature/shared.rb
+++ b/lib/feature/shared.rb
@@ -9,12 +9,14 @@ class Feature
# optional: defines if a on-disk definition is required for this feature flag type
# rollout_issue: defines if `bin/feature-flag` asks for rollout issue
# default_enabled: defines a default state of a feature flag when created by `bin/feature-flag`
+ # ee_only: defines that a feature flag can only be created in a context of EE
# example: usage being shown when exception is raised
TYPES = {
development: {
description: 'Short lived, used to enable unfinished code to be deployed',
- optional: true,
+ optional: false,
rollout_issue: true,
+ ee_only: false,
default_enabled: false,
example: <<-EOS
Feature.enabled?(:my_feature_flag, project)
@@ -26,6 +28,7 @@ class Feature
description: "Long-lived feature flags that control operational aspects of GitLab's behavior",
optional: true,
rollout_issue: false,
+ ee_only: false,
default_enabled: false,
example: <<-EOS
Feature.enabled?(:my_ops_flag, type: ops)
@@ -36,6 +39,7 @@ class Feature
description: 'Permanent feature flags used to temporarily disable licensed features.',
optional: true,
rollout_issue: false,
+ ee_only: true,
default_enabled: true,
example: <<-EOS
project.feature_available?(:my_licensed_feature)
@@ -44,13 +48,15 @@ class Feature
}
}.freeze
+ # The ordering of PARAMS defines an order in YAML
+ # This is done to ease the file comparison
PARAMS = %i[
name
- default_enabled
- type
introduced_by_url
rollout_issue_url
+ type
group
+ default_enabled
].freeze
end
end
diff --git a/lib/gitlab/alert_management/alert_params.rb b/lib/gitlab/alert_management/alert_params.rb
deleted file mode 100644
index 3bb839c1114..00000000000
--- a/lib/gitlab/alert_management/alert_params.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module AlertManagement
- class AlertParams
- MONITORING_TOOLS = {
- prometheus: 'Prometheus'
- }.freeze
-
- def self.from_generic_alert(project:, payload:)
- parsed_payload = Gitlab::Alerting::NotificationPayloadParser.call(payload, project).with_indifferent_access
- annotations = parsed_payload[:annotations]
-
- {
- project_id: project.id,
- title: annotations[:title],
- description: annotations[:description],
- monitoring_tool: annotations[:monitoring_tool],
- service: annotations[:service],
- hosts: Array(annotations[:hosts]),
- payload: payload,
- started_at: parsed_payload['startsAt'],
- ended_at: parsed_payload['endsAt'],
- severity: annotations[:severity],
- fingerprint: annotations[:fingerprint],
- environment: annotations[:environment]
- }
- end
-
- def self.from_prometheus_alert(project:, parsed_alert:)
- {
- project_id: project.id,
- title: parsed_alert.title,
- description: parsed_alert.description,
- monitoring_tool: MONITORING_TOOLS[:prometheus],
- payload: parsed_alert.payload,
- started_at: parsed_alert.starts_at,
- ended_at: parsed_alert.ends_at,
- fingerprint: parsed_alert.gitlab_fingerprint,
- environment: parsed_alert.environment,
- prometheus_alert: parsed_alert.gitlab_alert
- }
- end
- end
- end
-end
diff --git a/lib/gitlab/alert_management/alert_status_counts.rb b/lib/gitlab/alert_management/alert_status_counts.rb
index 382026236e0..e55e0016599 100644
--- a/lib/gitlab/alert_management/alert_status_counts.rb
+++ b/lib/gitlab/alert_management/alert_status_counts.rb
@@ -6,8 +6,6 @@ module Gitlab
class AlertStatusCounts
include Gitlab::Utils::StrongMemoize
- STATUSES = ::AlertManagement::Alert::STATUSES
-
attr_reader :project
def self.declarative_policy_class
@@ -21,7 +19,7 @@ module Gitlab
end
# Define method for each status
- STATUSES.each_key do |status|
+ ::AlertManagement::Alert.status_names.each do |status|
define_method(status) { counts[status] }
end
@@ -30,7 +28,7 @@ module Gitlab
end
def all
- counts.values.sum # rubocop:disable CodeReuse/ActiveRecord
+ counts.values.sum
end
private
@@ -44,9 +42,7 @@ module Gitlab
end
def counts_by_status
- ::AlertManagement::AlertsFinder
- .counts_by_status(current_user, project, params)
- .transform_keys { |status_id| STATUSES.key(status_id) }
+ ::AlertManagement::AlertsFinder.counts_by_status(current_user, project, params)
end
end
end
diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb
index 74e47e5226e..0fd593a3780 100644
--- a/lib/gitlab/alert_management/payload/base.rb
+++ b/lib/gitlab/alert_management/payload/base.rb
@@ -88,19 +88,19 @@ module Gitlab
# AlertManagement::Alert directly for read operations.
def alert_params
{
- description: description,
+ description: description&.truncate(::AlertManagement::Alert::DESCRIPTION_MAX_LENGTH),
ended_at: ends_at,
environment: environment,
fingerprint: gitlab_fingerprint,
- hosts: Array(hosts),
- monitoring_tool: monitoring_tool,
+ hosts: truncate_hosts(Array(hosts).flatten),
+ monitoring_tool: monitoring_tool&.truncate(::AlertManagement::Alert::TOOL_MAX_LENGTH),
payload: payload,
project_id: project.id,
prometheus_alert: gitlab_alert,
- service: service,
+ service: service&.truncate(::AlertManagement::Alert::SERVICE_MAX_LENGTH),
severity: severity,
started_at: starts_at,
- title: title
+ title: title&.truncate(::AlertManagement::Alert::TITLE_MAX_LENGTH)
}.transform_values(&:presence).compact
end
@@ -135,6 +135,18 @@ module Gitlab
def plain_gitlab_fingerprint; end
+ def truncate_hosts(hosts)
+ return hosts if hosts.join.length <= ::AlertManagement::Alert::HOSTS_MAX_LENGTH
+
+ hosts.inject([]) do |new_hosts, host|
+ remaining_length = ::AlertManagement::Alert::HOSTS_MAX_LENGTH - new_hosts.join.length
+
+ break new_hosts unless remaining_length > 0
+
+ new_hosts << host.to_s.truncate(remaining_length, omission: '')
+ end
+ end
+
def value_for_paths(paths)
target_path = paths.find { |path| payload&.dig(*path) }
diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb
index 7efdfac75dc..e8e85155bef 100644
--- a/lib/gitlab/alert_management/payload/generic.rb
+++ b/lib/gitlab/alert_management/payload/generic.rb
@@ -8,6 +8,8 @@ module Gitlab
DEFAULT_TITLE = 'New: Incident'
DEFAULT_SEVERITY = 'critical'
+ attribute :description, paths: 'description'
+ attribute :ends_at, paths: 'end_time', type: :time
attribute :environment_name, paths: 'gitlab_environment_name'
attribute :hosts, paths: 'hosts'
attribute :monitoring_tool, paths: 'monitoring_tool'
@@ -23,3 +25,5 @@ module Gitlab
end
end
end
+
+Gitlab::AlertManagement::Payload::Generic.prepend_if_ee('EE::Gitlab::AlertManagement::Payload::Generic')
diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb
deleted file mode 100644
index 94b81b7d290..00000000000
--- a/lib/gitlab/alerting/alert.rb
+++ /dev/null
@@ -1,215 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Alerting
- class Alert
- include ActiveModel::Model
- include Gitlab::Utils::StrongMemoize
- include Presentable
-
- attr_accessor :project, :payload, :am_alert
-
- def self.for_alert_management_alert(project:, alert:)
- params = if alert.prometheus?
- alert.payload
- else
- Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project)
- end
-
- self.new(project: project, payload: params, am_alert: alert)
- end
-
- def gitlab_alert
- strong_memoize(:gitlab_alert) do
- parse_gitlab_alert_from_payload
- end
- end
-
- def metric_id
- strong_memoize(:metric_id) do
- payload&.dig('labels', 'gitlab_alert_id')
- end
- end
-
- def gitlab_prometheus_alert_id
- strong_memoize(:gitlab_prometheus_alert_id) do
- payload&.dig('labels', 'gitlab_prometheus_alert_id')
- end
- end
-
- def title
- strong_memoize(:title) do
- gitlab_alert&.title || parse_title_from_payload
- end
- end
-
- def description
- strong_memoize(:description) do
- parse_description_from_payload
- end
- end
-
- def environment
- strong_memoize(:environment) do
- gitlab_alert&.environment || parse_environment_from_payload
- end
- end
-
- def annotations
- strong_memoize(:annotations) do
- parse_annotations_from_payload || []
- end
- end
-
- def starts_at
- strong_memoize(:starts_at) do
- parse_datetime_from_payload('startsAt')
- end
- end
-
- def starts_at_raw
- strong_memoize(:starts_at_raw) do
- payload&.dig('startsAt')
- end
- end
-
- def ends_at
- strong_memoize(:ends_at) do
- parse_datetime_from_payload('endsAt')
- end
- end
-
- def full_query
- strong_memoize(:full_query) do
- gitlab_alert&.full_query || parse_expr_from_payload
- end
- end
-
- def y_label
- strong_memoize(:y_label) do
- parse_y_label_from_payload || title
- end
- end
-
- def alert_markdown
- strong_memoize(:alert_markdown) do
- parse_alert_markdown_from_payload
- end
- end
-
- def status
- strong_memoize(:status) do
- payload&.dig('status')
- end
- end
-
- def firing?
- status == 'firing'
- end
-
- def resolved?
- status == 'resolved'
- end
-
- def gitlab_managed?
- metric_id.present?
- end
-
- def gitlab_fingerprint
- Gitlab::AlertManagement::Fingerprint.generate(plain_gitlab_fingerprint)
- end
-
- def valid?
- payload.respond_to?(:dig) && project && title && starts_at
- end
-
- def present
- super(presenter_class: Projects::Prometheus::AlertPresenter)
- end
-
- private
-
- def plain_gitlab_fingerprint
- if gitlab_managed?
- [metric_id, starts_at_raw].join('/')
- else # self managed
- [starts_at_raw, title, full_query].join('/')
- end
- end
-
- def parse_environment_from_payload
- environment_name = payload&.dig('labels', 'gitlab_environment_name')
-
- return unless environment_name
-
- EnvironmentsFinder.new(project, nil, { name: environment_name })
- .find
- &.first
- end
-
- def parse_gitlab_alert_from_payload
- alerts_found = matching_gitlab_alerts
-
- return if alerts_found.blank? || alerts_found.size > 1
-
- alerts_found.first
- end
-
- def matching_gitlab_alerts
- return unless metric_id || gitlab_prometheus_alert_id
-
- Projects::Prometheus::AlertsFinder
- .new(project: project, metric: metric_id, id: gitlab_prometheus_alert_id)
- .execute
- end
-
- def parse_title_from_payload
- payload&.dig('annotations', 'title') ||
- payload&.dig('annotations', 'summary') ||
- payload&.dig('labels', 'alertname')
- end
-
- def parse_description_from_payload
- payload&.dig('annotations', 'description')
- end
-
- def parse_annotations_from_payload
- payload&.dig('annotations')&.map do |label, value|
- Alerting::AlertAnnotation.new(label: label, value: value)
- end
- end
-
- def parse_datetime_from_payload(field)
- value = payload&.dig(field)
- return unless value
-
- # value is a rfc3339 timestamp
- # Timestamps from Prometheus and Alertmanager are UTC RFC3339 timestamps like: '2018-03-12T09:06:00Z' (Z represents 0 offset or UTC)
- # .utc sets the datetime zone to `UTC`
- Time.rfc3339(value).utc
- rescue ArgumentError
- end
-
- # Parses `g0.expr` from `generatorURL`.
- #
- # Example: http://localhost:9090/graph?g0.expr=vector%281%29&g0.tab=1
- def parse_expr_from_payload
- url = payload&.dig('generatorURL')
- return unless url
-
- uri = URI(url)
-
- Rack::Utils.parse_query(uri.query).fetch('g0.expr')
- rescue URI::InvalidURIError, KeyError
- end
-
- def parse_alert_markdown_from_payload
- payload&.dig('annotations', 'gitlab_incident_markdown')
- end
-
- def parse_y_label_from_payload
- payload&.dig('annotations', 'gitlab_y_label')
- end
- end
- end
-end
diff --git a/lib/gitlab/alerting/alert_annotation.rb b/lib/gitlab/alerting/alert_annotation.rb
deleted file mode 100644
index a4b3a97b08c..00000000000
--- a/lib/gitlab/alerting/alert_annotation.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Alerting
- class AlertAnnotation
- include ActiveModel::Model
-
- attr_accessor :label, :value
- end
- end
-end
diff --git a/lib/gitlab/alerting/notification_payload_parser.rb b/lib/gitlab/alerting/notification_payload_parser.rb
deleted file mode 100644
index 348f851f551..00000000000
--- a/lib/gitlab/alerting/notification_payload_parser.rb
+++ /dev/null
@@ -1,107 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Alerting
- class NotificationPayloadParser
- BadPayloadError = Class.new(StandardError)
-
- DEFAULT_TITLE = 'New: Incident'
- DEFAULT_SEVERITY = 'critical'
-
- def initialize(payload, project)
- @payload = payload.to_h.with_indifferent_access
- @project = project
- end
-
- def self.call(payload, project)
- new(payload, project).call
- end
-
- def call
- {
- 'annotations' => annotations,
- 'startsAt' => starts_at,
- 'endsAt' => ends_at
- }.compact
- end
-
- private
-
- attr_reader :payload, :project
-
- def title
- payload[:title].presence || DEFAULT_TITLE
- end
-
- def severity
- payload[:severity].presence || DEFAULT_SEVERITY
- end
-
- def fingerprint
- Gitlab::AlertManagement::Fingerprint.generate(payload[:fingerprint])
- end
-
- def annotations
- primary_params
- .reverse_merge(flatten_secondary_params)
- .transform_values(&:presence)
- .compact
- end
-
- def primary_params
- {
- 'title' => title,
- 'description' => payload[:description],
- 'monitoring_tool' => payload[:monitoring_tool],
- 'service' => payload[:service],
- 'hosts' => hosts.presence,
- 'severity' => severity,
- 'fingerprint' => fingerprint,
- 'environment' => environment
- }
- end
-
- def hosts
- Array(payload[:hosts]).reject(&:blank?)
- end
-
- def current_time
- Time.current.change(usec: 0).rfc3339
- end
-
- def starts_at
- Time.parse(payload[:start_time].to_s).rfc3339
- rescue ArgumentError
- current_time
- end
-
- def ends_at
- Time.parse(payload[:end_time].to_s).rfc3339
- rescue ArgumentError
- nil
- end
-
- def environment
- environment_name = payload[:gitlab_environment_name]
-
- return unless environment_name
-
- EnvironmentsFinder.new(project, nil, { name: environment_name })
- .find
- &.first
- end
-
- def secondary_params
- payload.except(:start_time, :end_time)
- end
-
- def flatten_secondary_params
- Gitlab::Utils::SafeInlineHash.merge_keys!(secondary_params)
- rescue ArgumentError
- raise BadPayloadError, 'The payload is too big'
- end
- end
- end
-end
-
-Gitlab::Alerting::NotificationPayloadParser.prepend_if_ee('EE::Gitlab::Alerting::NotificationPayloadParser')
diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb
index 30cb74bcf54..b4bbb309c36 100644
--- a/lib/gitlab/application_context.rb
+++ b/lib/gitlab/application_context.rb
@@ -12,7 +12,8 @@ module Gitlab
Attribute.new(:namespace, Namespace),
Attribute.new(:user, User),
Attribute.new(:caller_id, String),
- Attribute.new(:related_class, String)
+ Attribute.new(:related_class, String),
+ Attribute.new(:feature_category, String)
].freeze
def self.with_context(args, &block)
@@ -45,6 +46,7 @@ module Gitlab
hash[:root_namespace] = -> { root_namespace_path } if include_namespace?
hash[:caller_id] = caller_id if set_values.include?(:caller_id)
hash[:related_class] = related_class if set_values.include?(:related_class)
+ hash[:feature_category] = feature_category if set_values.include?(:feature_category)
end
end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 609eef5e365..001c083c778 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -371,7 +371,7 @@ module Gitlab
end
def find_build_by_token(token)
- ::Ci::Build.running.find_by_token(token)
+ ::Ci::AuthJobFinder.new(token: token).execute
end
def user_auth_attempt!(user, success:)
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index ccf52bae9a5..3d3f7212053 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -290,7 +290,7 @@ module Gitlab
end
def api_request?
- current_request.path.starts_with?('/api/')
+ current_request.path.starts_with?(Gitlab::Utils.append_path(Gitlab.config.gitlab.relative_url_root, '/api/'))
end
def archive_request?
diff --git a/lib/gitlab/auth/otp/strategies/base.rb b/lib/gitlab/auth/otp/strategies/base.rb
new file mode 100644
index 00000000000..718630e0e31
--- /dev/null
+++ b/lib/gitlab/auth/otp/strategies/base.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Auth
+ module Otp
+ module Strategies
+ class Base
+ def initialize(user)
+ @user = user
+ end
+
+ private
+
+ attr_reader :user
+
+ def success
+ { status: :success }
+ end
+
+ def error(message, http_status = nil)
+ result = { message: message,
+ status: :error }
+
+ result[:http_status] = http_status if http_status
+
+ result
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/auth/otp/strategies/devise.rb b/lib/gitlab/auth/otp/strategies/devise.rb
new file mode 100644
index 00000000000..93068d6c9b0
--- /dev/null
+++ b/lib/gitlab/auth/otp/strategies/devise.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Auth
+ module Otp
+ module Strategies
+ class Devise < Base
+ def validate(otp_code)
+ user.validate_and_consume_otp!(otp_code) ? success : error('invalid OTP code')
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator.rb
new file mode 100644
index 00000000000..fbcb9fd8cdb
--- /dev/null
+++ b/lib/gitlab/auth/otp/strategies/forti_authenticator.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Auth
+ module Otp
+ module Strategies
+ class FortiAuthenticator < Base
+ def validate(otp_code)
+ body = { username: user.username,
+ token_code: otp_code }
+
+ response = Gitlab::HTTP.post(
+ auth_url,
+ headers: { 'Content-Type': 'application/json' },
+ body: body.to_json,
+ basic_auth: api_credentials)
+
+ # Successful authentication results in HTTP 200: OK
+ # https://docs.fortinet.com/document/fortiauthenticator/6.2.0/rest-api-solution-guide/704555/authentication-auth
+ response.ok? ? success : error(message: response.message, http_status: response.code)
+ end
+
+ private
+
+ def auth_url
+ host = ::Gitlab.config.forti_authenticator.host
+ port = ::Gitlab.config.forti_authenticator.port
+ path = 'api/v1/auth/'
+
+ "https://#{host}:#{port}/#{path}"
+ end
+
+ def api_credentials
+ { username: ::Gitlab.config.forti_authenticator.username,
+ password: ::Gitlab.config.forti_authenticator.token }
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb
index cc4b8d887ff..36b54ba2e46 100644
--- a/lib/gitlab/auth/user_access_denied_reason.rb
+++ b/lib/gitlab/auth/user_access_denied_reason.rb
@@ -11,6 +11,8 @@ module Gitlab
case rejection_type
when :internal
"This action cannot be performed by internal users"
+ when :blocked_pending_approval
+ "Your account is pending approval from your administrator and hence blocked."
when :terms_not_accepted
"You (#{@user.to_reference}) must accept the Terms of Service in order to perform this action. "\
"Please access GitLab from a web browser to accept these terms."
@@ -31,6 +33,8 @@ module Gitlab
def rejection_type
if @user.internal?
:internal
+ elsif @user.blocked_pending_approval?
+ :blocked_pending_approval
elsif @user.required_terms_not_accepted?
:terms_not_accepted
elsif @user.deactivated?
diff --git a/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb b/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb
new file mode 100644
index 00000000000..2148e96f6b4
--- /dev/null
+++ b/lib/gitlab/background_migration/add_modified_to_approval_merge_request_rule.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Compare all current rules to project rules
+ class AddModifiedToApprovalMergeRequestRule
+ # Stubbed class to access the Group table
+ class Group < ActiveRecord::Base
+ self.table_name = 'namespaces'
+ self.inheritance_column = :_type_disabled
+ end
+
+ # Stubbed class to access the ApprovalMergeRequestRule table
+ class ApprovalMergeRequestRule < ActiveRecord::Base
+ self.table_name = 'approval_merge_request_rules'
+
+ has_one :approval_merge_request_rule_source, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource'
+ has_one :approval_project_rule, through: :approval_merge_request_rule_source
+ has_and_belongs_to_many :groups,
+ class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups"
+ end
+
+ # Stubbed class to access the ApprovalProjectRule table
+ class ApprovalProjectRule < ActiveRecord::Base
+ self.table_name = 'approval_project_rules'
+
+ has_many :approval_merge_request_rule_sources, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRuleSource'
+ has_and_belongs_to_many :groups,
+ class_name: 'AddModifiedToApprovalMergeRequestRule::Group', join_table: "#{self.table_name}_groups"
+ end
+
+ # Stubbed class to access the ApprovalMergeRequestRuleSource table
+ class ApprovalMergeRequestRuleSource < ActiveRecord::Base
+ self.table_name = 'approval_merge_request_rule_sources'
+
+ belongs_to :approval_merge_request_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalMergeRequestRule'
+ belongs_to :approval_project_rule, class_name: 'AddModifiedToApprovalMergeRequestRule::ApprovalProjectRule'
+ end
+
+ def perform(start_id, stop_id)
+ approval_merge_requests_rules = ApprovalMergeRequestRule
+ .joins(:groups, :approval_merge_request_rule_source)
+ .where(id: start_id..stop_id)
+ .pluck(
+ 'approval_merge_request_rule_sources.id as ars_id',
+ 'approval_merge_request_rules_groups.id as amrg_id'
+ )
+
+ approval_project_rules = ApprovalProjectRule
+ .joins(:groups, approval_merge_request_rule_sources: :approval_merge_request_rule)
+ .where(approval_merge_request_rules: { id: start_id..stop_id })
+ .pluck(
+ 'approval_merge_request_rule_sources.id as ars_id',
+ 'approval_project_rules_groups.id as apg_id'
+ )
+
+ different_names_or_approval_sources = ApprovalMergeRequestRule.joins(:approval_project_rule, :approval_merge_request_rule_source)
+ .where(id: start_id..stop_id)
+ .where('approval_merge_request_rules.name != approval_project_rules.name OR ' \
+ 'approval_merge_request_rules.approvals_required != approval_project_rules.approvals_required')
+ .pluck('approval_merge_request_rule_sources.id as ars_id')
+
+ intersected_set = approval_merge_requests_rules.to_set ^ approval_project_rules.to_set
+ source_ids = intersected_set.collect { |rule| rule[0] }.uniq
+
+ rule_sources = ApprovalMergeRequestRuleSource.where(id: source_ids + different_names_or_approval_sources)
+ changed_merge_request_rules = ApprovalMergeRequestRule.where(id: rule_sources.select(:approval_merge_request_rule_id))
+
+ changed_merge_request_rules.update_all(modified_from_project_rule: true)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
index 21538000fec..8befade8c3a 100644
--- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
@@ -109,7 +109,7 @@ module Gitlab
end
def create_commit(snippet)
- snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), commit_attrs)
+ snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), **commit_attrs)
end
# If the user is not allowed to access git or update the snippet
diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb
new file mode 100644
index 00000000000..b8c14aa2573
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+require "webauthn/u2f_migrator"
+
+module Gitlab
+ module BackgroundMigration
+ class MigrateU2fWebauthn
+ class U2fRegistration < ActiveRecord::Base
+ self.table_name = 'u2f_registrations'
+ end
+
+ class WebauthnRegistration < ActiveRecord::Base
+ self.table_name = 'webauthn_registrations'
+ end
+
+ def perform(start_id, end_id)
+ old_registrations = U2fRegistration.where(id: start_id..end_id)
+ old_registrations.each_slice(100) do |slice|
+ now = Time.now
+ values = slice.map do |u2f_registration|
+ converted_credential = WebAuthn::U2fMigrator.new(
+ app_id: Gitlab.config.gitlab.url,
+ certificate: u2f_registration.certificate,
+ key_handle: u2f_registration.key_handle,
+ public_key: u2f_registration.public_key,
+ counter: u2f_registration.counter
+ ).credential
+
+ {
+ credential_xid: Base64.strict_encode64(converted_credential.id),
+ public_key: Base64.strict_encode64(converted_credential.public_key),
+ counter: u2f_registration.counter || 0,
+ name: u2f_registration.name || '',
+ user_id: u2f_registration.user_id,
+ u2f_registration_id: u2f_registration.id,
+ created_at: now,
+ updated_at: now
+ }
+ end
+
+ WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb
index ca64d13b118..bbe2164ae4e 100644
--- a/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb
+++ b/lib/gitlab/background_migration/migrate_users_bio_to_user_details.rb
@@ -13,8 +13,6 @@ module Gitlab
end
def perform(start_id, stop_id)
- return if Feature.disabled?(:migrate_bio_to_user_details, default_enabled: true)
-
relation = User
.select("id AS user_id", "substring(COALESCE(bio, '') from 1 for 255) AS bio")
.where("(COALESCE(bio, '') IS DISTINCT FROM '')")
diff --git a/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb
new file mode 100644
index 00000000000..cd305adc7cd
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_duplicated_cs_findings_without_vulnerability_id.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+ module BackgroundMigration
+ class RemoveDuplicatedCsFindingsWithoutVulnerabilityId
+ def perform(start_id, stop_id)
+ end
+ end
+ end
+end
+
+Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveDuplicatedCsFindingsWithoutVulnerabilityId')
diff --git a/lib/gitlab/background_migration/replace_blocked_by_links.rb b/lib/gitlab/background_migration/replace_blocked_by_links.rb
new file mode 100644
index 00000000000..26626aaef79
--- /dev/null
+++ b/lib/gitlab/background_migration/replace_blocked_by_links.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+ module BackgroundMigration
+ class ReplaceBlockedByLinks
+ class IssueLink < ActiveRecord::Base
+ self.table_name = 'issue_links'
+ end
+
+ def perform(start_id, stop_id)
+ blocked_by_links = IssueLink.where(id: start_id..stop_id).where(link_type: 2)
+
+ ActiveRecord::Base.transaction do
+ # if there is duplicit bi-directional relation (issue2 is blocked by issue1
+ # and issue1 already links issue2), then we can just delete 'blocked by'.
+ # This should be rare as we have a pre-create check which checks if issues are
+ # already linked
+ blocked_by_links
+ .joins('INNER JOIN issue_links as opposite_links ON issue_links.source_id = opposite_links.target_id AND issue_links.target_id = opposite_links.source_id')
+ .where('opposite_links.link_type': 1)
+ .delete_all
+
+ blocked_by_links.update_all('source_id=target_id,target_id=source_id,link_type=1')
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/sync_blocking_issues_count.rb b/lib/gitlab/background_migration/sync_blocking_issues_count.rb
new file mode 100644
index 00000000000..6262320128c
--- /dev/null
+++ b/lib/gitlab/background_migration/sync_blocking_issues_count.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+ module BackgroundMigration
+ class SyncBlockingIssuesCount
+ def perform(start_id, end_id)
+ end
+ end
+ end
+end
+
+Gitlab::BackgroundMigration::SyncBlockingIssuesCount.prepend_if_ee('EE::Gitlab::BackgroundMigration::SyncBlockingIssuesCount')
diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb
new file mode 100644
index 00000000000..3def5eb3369
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Lib
+ module Banzai
+ # isolated Banzai::ReferenceParser
+ module ReferenceParser
+ # Returns the reference parser class for the given type
+ #
+ # Example:
+ #
+ # Banzai::ReferenceParser['isolated_mentioned_group']
+ #
+ # This would return the `::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::IsolatedMentionedGroupParser` class.
+ def self.[](name)
+ const_get("::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser::#{name.to_s.camelize}Parser", false)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb
new file mode 100644
index 00000000000..d3d032ba433
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_group_parser.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Lib
+ module Banzai
+ module ReferenceParser
+ # isolated Banzai::ReferenceParser::MentionedGroupParser
+ class IsolatedMentionedGroupParser < ::Banzai::ReferenceParser::MentionedGroupParser
+ extend ::Gitlab::Utils::Override
+
+ self.reference_type = :user
+
+ override :references_relation
+ def references_relation
+ ::Gitlab::BackgroundMigration::UserMentions::Models::Group
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
new file mode 100644
index 00000000000..1d3a3af81a1
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Lib
+ module Gitlab
+ # Extract possible GFM references from an arbitrary String for further processing.
+ class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor
+ REFERABLES = %i(isolated_mentioned_group).freeze
+
+ REFERABLES.each do |type|
+ define_method("#{type}s") do
+ @references[type] ||= isolated_references(type)
+ end
+ end
+
+ def isolated_references(type)
+ context = ::Banzai::RenderContext.new(project, current_user)
+ processor = ::Gitlab::BackgroundMigration::UserMentions::Lib::Banzai::ReferenceParser[type].new(context)
+
+ refs = processor.process(html_documents)
+ refs[:visible]
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
index 69ba3f9132b..be9c0ad2b3a 100644
--- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
@@ -36,7 +36,8 @@ module Gitlab
if extractor
extractors[current_user] = extractor
else
- extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user)
+ extractor = extractors[current_user] ||=
+ Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedReferenceExtractor.new(project, current_user)
extractor.reset_memoized_values
end
@@ -71,7 +72,7 @@ module Gitlab
mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id))
mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id))
- mentioned_groups_ids = array_to_sql(refs.mentioned_groups.pluck(:id))
+ mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id))
return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank?
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
new file mode 100644
index 00000000000..5cadfa45b5b
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Models
+ module Concerns
+ module Namespace
+ # extracted methods for recursive traversing of namespace hierarchy
+ module RecursiveTraversal
+ extend ActiveSupport::Concern
+
+ def root_ancestor
+ return self if persisted? && parent_id.nil?
+
+ strong_memoize(:root_ancestor) do
+ Gitlab::ObjectHierarchy
+ .new(self.class.where(id: id))
+ .base_and_ancestors
+ .reorder(nil)
+ .find_by(parent_id: nil)
+ end
+ end
+
+ # Returns all ancestors, self, and descendants of the current namespace.
+ def self_and_hierarchy
+ Gitlab::ObjectHierarchy
+ .new(self.class.where(id: id))
+ .all_objects
+ end
+
+ # Returns all the ancestors of the current namespaces.
+ def ancestors
+ return self.class.none unless parent_id
+
+ Gitlab::ObjectHierarchy
+ .new(self.class.where(id: parent_id))
+ .base_and_ancestors
+ end
+
+ # returns all ancestors upto but excluding the given namespace
+ # when no namespace is given, all ancestors upto the top are returned
+ def ancestors_upto(top = nil, hierarchy_order: nil)
+ Gitlab::ObjectHierarchy.new(self.class.where(id: id))
+ .ancestors(upto: top, hierarchy_order: hierarchy_order)
+ end
+
+ def self_and_ancestors(hierarchy_order: nil)
+ return self.class.where(id: id) unless parent_id
+
+ Gitlab::ObjectHierarchy
+ .new(self.class.where(id: id))
+ .base_and_ancestors(hierarchy_order: hierarchy_order)
+ end
+
+ # Returns all the descendants of the current namespace.
+ def descendants
+ Gitlab::ObjectHierarchy
+ .new(self.class.where(parent_id: id))
+ .base_and_descendants
+ end
+
+ def self_and_descendants
+ Gitlab::ObjectHierarchy
+ .new(self.class.where(id: id))
+ .base_and_descendants
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb
new file mode 100644
index 00000000000..bc04172b9a2
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/group.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Models
+ # isolated Group model
+ class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace
+ self.store_full_sti_class = false
+ has_one :saml_provider
+
+ def self.declarative_policy_class
+ "GroupPolicy"
+ end
+
+ def max_member_access_for_user(user)
+ return GroupMember::NO_ACCESS unless user
+
+ return GroupMember::OWNER if user.admin?
+
+ max_member_access = members_with_parents.where(user_id: user)
+ .reorder(access_level: :desc)
+ .first
+ &.access_level
+
+ max_member_access || GroupMember::NO_ACCESS
+ end
+
+ def members_with_parents
+ # Avoids an unnecessary SELECT when the group has no parents
+ source_ids =
+ if has_parent?
+ self_and_ancestors.reorder(nil).select(:id)
+ else
+ id
+ end
+
+ group_hierarchy_members = GroupMember.active_without_invites_and_requests
+ .where(source_id: source_ids)
+
+ GroupMember.from_union([group_hierarchy_members,
+ members_from_self_and_ancestor_group_shares])
+ end
+
+ # rubocop: disable Metrics/AbcSize
+ def members_from_self_and_ancestor_group_shares
+ group_group_link_table = GroupGroupLink.arel_table
+ group_member_table = GroupMember.arel_table
+
+ source_ids =
+ if has_parent?
+ self_and_ancestors.reorder(nil).select(:id)
+ else
+ id
+ end
+
+ group_group_links_query = GroupGroupLink.where(shared_group_id: source_ids)
+ cte = Gitlab::SQL::CTE.new(:group_group_links_cte, group_group_links_query)
+ cte_alias = cte.table.alias(GroupGroupLink.table_name)
+
+ # Instead of members.access_level, we need to maximize that access_level at
+ # the respective group_group_links.group_access.
+ member_columns = GroupMember.attribute_names.map do |column_name|
+ if column_name == 'access_level'
+ smallest_value_arel([cte_alias[:group_access], group_member_table[:access_level]],
+ 'access_level')
+ else
+ group_member_table[column_name]
+ end
+ end
+
+ GroupMember
+ .with(cte.to_arel)
+ .select(*member_columns)
+ .from([group_member_table, cte.alias_to(group_group_link_table)])
+ .where(group_member_table[:requested_at].eq(nil))
+ .where(group_member_table[:source_id].eq(group_group_link_table[:shared_with_group_id]))
+ .where(group_member_table[:source_type].eq('Namespace'))
+ end
+ # rubocop: enable Metrics/AbcSize
+
+ def smallest_value_arel(args, column_alias)
+ Arel::Nodes::As.new(
+ Arel::Nodes::NamedFunction.new('LEAST', args),
+ Arel::Nodes::SqlLiteral.new(column_alias))
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
new file mode 100644
index 00000000000..6d7b9a86e69
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Models
+ # isolated Namespace model
+ class Namespace < ApplicationRecord
+ include ::Gitlab::VisibilityLevel
+ include ::Gitlab::Utils::StrongMemoize
+ include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal
+
+ belongs_to :parent, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace"
+
+ def visibility_level_field
+ :visibility_level
+ end
+
+ def has_parent?
+ parent_id.present? || parent.present?
+ end
+
+ # Overridden in EE::Namespace
+ def feature_available?(_feature)
+ false
+ end
+ end
+ end
+ end
+ end
+end
+
+Namespace.prepend_if_ee('::EE::Namespace')
diff --git a/lib/gitlab/bulk_import/client.rb b/lib/gitlab/bulk_import/client.rb
new file mode 100644
index 00000000000..c6e77a158cd
--- /dev/null
+++ b/lib/gitlab/bulk_import/client.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BulkImport
+ class Client
+ API_VERSION = 'v4'.freeze
+ DEFAULT_PAGE = 1.freeze
+ DEFAULT_PER_PAGE = 30.freeze
+
+ ConnectionError = Class.new(StandardError)
+
+ def initialize(uri:, token:, page: DEFAULT_PAGE, per_page: DEFAULT_PER_PAGE, api_version: API_VERSION)
+ @uri = URI.parse(uri)
+ @token = token&.strip
+ @page = page
+ @per_page = per_page
+ @api_version = api_version
+ end
+
+ def get(resource, query = {})
+ response = with_error_handling do
+ Gitlab::HTTP.get(
+ resource_url(resource),
+ headers: request_headers,
+ follow_redirects: false,
+ query: query.merge(request_query)
+ )
+ end
+
+ response.parsed_response
+ end
+
+ private
+
+ def request_query
+ {
+ page: @page,
+ per_page: @per_page
+ }
+ end
+
+ def request_headers
+ {
+ 'Content-Type' => 'application/json',
+ 'Authorization' => "Bearer #{@token}"
+ }
+ end
+
+ def with_error_handling
+ response = yield
+
+ raise ConnectionError.new("Error #{response.code}") unless response.success?
+
+ response
+ rescue *Gitlab::HTTP::HTTP_ERRORS => e
+ raise ConnectionError, e
+ end
+
+ def base_uri
+ @base_uri ||= "#{@uri.scheme}://#{@uri.host}:#{@uri.port}"
+ end
+
+ def api_url
+ Gitlab::Utils.append_path(base_uri, "/api/#{@api_version}")
+ end
+
+ def resource_url(resource)
+ Gitlab::Utils.append_path(api_url, resource)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb
index 71361b12d07..db7af0088d0 100644
--- a/lib/gitlab/checks/matching_merge_request.rb
+++ b/lib/gitlab/checks/matching_merge_request.rb
@@ -20,3 +20,5 @@ module Gitlab
end
end
end
+
+Gitlab::Checks::MatchingMergeRequest.prepend_if_ee('EE::Gitlab::Checks::MatchingMergeRequest')
diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb
index b3c981d252b..0d93e7ac8a1 100644
--- a/lib/gitlab/checks/post_push_message.rb
+++ b/lib/gitlab/checks/post_push_message.rb
@@ -44,11 +44,7 @@ module Gitlab
end
def url_to_repo
- protocol == 'ssh' ? message_subject.ssh_url_to_repo : message_subject.http_url_to_repo
- end
-
- def message_subject
- repository.repo_type.wiki? ? project.wiki : container
+ protocol == 'ssh' ? container.ssh_url_to_repo : container.http_url_to_repo
end
end
end
diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb
index 0373a12ab69..6d152c052dc 100644
--- a/lib/gitlab/ci/ansi2json/converter.rb
+++ b/lib/gitlab/ci/ansi2json/converter.rb
@@ -104,23 +104,24 @@ module Gitlab
action = scanner[1]
timestamp = scanner[2]
section = scanner[3]
+ options = parse_section_options(scanner[4])
section_name = sanitize_section_name(section)
- if action == "start"
- handle_section_start(scanner, section_name, timestamp)
- elsif action == "end"
+ if action == 'start'
+ handle_section_start(scanner, section_name, timestamp, options)
+ elsif action == 'end'
handle_section_end(scanner, section_name, timestamp)
else
raise 'unsupported action'
end
end
- def handle_section_start(scanner, section, timestamp)
+ def handle_section_start(scanner, section, timestamp, options)
# We make a new line for new section
flush_current_line
- @state.open_section(section, timestamp)
+ @state.open_section(section, timestamp, options)
# we need to consume match after handling
# the open of section, as we want the section
@@ -157,6 +158,18 @@ module Gitlab
def sanitize_section_name(section)
section.to_s.downcase.gsub(/[^a-z0-9]/, '-')
end
+
+ def parse_section_options(raw_options)
+ return unless raw_options
+
+ # We need to remove the square brackets and split
+ # by comma to get a list of the options
+ options = raw_options[1...-1].split ','
+
+ # Now split each option by equals to separate
+ # each in the format [key, value]
+ options.to_h { |option| option.split '=' }
+ end
end
end
end
diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb
index 21aa1f84353..b1dee0e1ecc 100644
--- a/lib/gitlab/ci/ansi2json/line.rb
+++ b/lib/gitlab/ci/ansi2json/line.rb
@@ -32,7 +32,7 @@ module Gitlab
end
attr_reader :offset, :sections, :segments, :current_segment,
- :section_header, :section_duration
+ :section_header, :section_duration, :section_options
def initialize(offset:, style:, sections: [])
@offset = offset
@@ -68,6 +68,10 @@ module Gitlab
@sections << section
end
+ def set_section_options(options)
+ @section_options = options
+ end
+
def set_as_section_header
@section_header = true
end
@@ -90,6 +94,7 @@ module Gitlab
result[:section] = sections.last if sections.any?
result[:section_header] = true if @section_header
result[:section_duration] = @section_duration if @section_duration
+ result[:section_options] = @section_options if @section_options
end
end
end
diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb
index 38d36e6950c..b2b6ce649ed 100644
--- a/lib/gitlab/ci/ansi2json/state.rb
+++ b/lib/gitlab/ci/ansi2json/state.rb
@@ -26,10 +26,11 @@ module Gitlab
Base64.urlsafe_encode64(state.to_json)
end
- def open_section(section, timestamp)
+ def open_section(section, timestamp, options)
@open_sections[section] = timestamp
@current_line.add_section(section)
+ @current_line.set_section_options(options)
@current_line.set_as_section_header
end
diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb
index 6395a20ca99..b0fad026ec5 100644
--- a/lib/gitlab/ci/artifact_file_reader.rb
+++ b/lib/gitlab/ci/artifact_file_reader.rb
@@ -45,14 +45,6 @@ module Gitlab
end
def read_zip_file!(file_path)
- if ::Gitlab::Ci::Features.new_artifact_file_reader_enabled?(job.project)
- read_with_new_artifact_file_reader(file_path)
- else
- read_with_legacy_artifact_file_reader(file_path)
- end
- end
-
- def read_with_new_artifact_file_reader(file_path)
job.artifacts_file.use_open_file do |file|
zip_file = Zip::File.new(file, false, true)
entry = zip_file.find_entry(file_path)
@@ -69,25 +61,6 @@ module Gitlab
end
end
- def read_with_legacy_artifact_file_reader(file_path)
- job.artifacts_file.use_file do |archive_path|
- Zip::File.open(archive_path) do |zip_file|
- entry = zip_file.find_entry(file_path)
- unless entry
- raise Error, "Path `#{file_path}` does not exist inside the `#{job.name}` artifacts archive!"
- end
-
- if entry.name_is_directory?
- raise Error, "Path `#{file_path}` was expected to be a file but it was a directory!"
- end
-
- zip_file.get_input_stream(entry) do |is|
- is.read
- end
- end
- end
- end
-
def max_archive_size_in_mb
ActiveSupport::NumberHelper.number_to_human_size(MAX_ARCHIVE_SIZE)
end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 9d269831679..071a8ef830f 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -54,6 +54,10 @@ module Gitlab
root.variables_value
end
+ def variables_with_data
+ root.variables_entry.value_with_data
+ end
+
def stages
root.stages_value
end
diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb
index a8b67a1db4f..1740032e5c7 100644
--- a/lib/gitlab/ci/config/entry/bridge.rb
+++ b/lib/gitlab/ci/config/entry/bridge.rb
@@ -11,15 +11,18 @@ module Gitlab
class Bridge < ::Gitlab::Config::Entry::Node
include ::Gitlab::Ci::Config::Entry::Processable
+ ALLOWED_WHEN = %w[on_success on_failure always manual].freeze
ALLOWED_KEYS = %i[trigger].freeze
validations do
validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS
with_options allow_nil: true do
- validates :when,
- inclusion: { in: %w[on_success on_failure always],
- message: 'should be on_success, on_failure or always' }
+ validates :allow_failure, boolean: true
+ validates :when, inclusion: {
+ in: ALLOWED_WHEN,
+ message: "should be one of: #{ALLOWED_WHEN.join(', ')}"
+ }
end
validate on: :composed do
@@ -57,11 +60,19 @@ module Gitlab
true
end
+ def manual_action?
+ self.when == 'manual'
+ end
+
+ def ignored?
+ allow_failure.nil? ? manual_action? : allow_failure
+ end
+
def value
super.merge(
trigger: (trigger_value if trigger_defined?),
needs: (needs_value if needs_defined?),
- ignore: !!allow_failure,
+ ignore: ignored?,
when: self.when,
scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage
).compact
diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb
index a304d9b724f..6b036182706 100644
--- a/lib/gitlab/ci/config/entry/cache.rb
+++ b/lib/gitlab/ci/config/entry/cache.rb
@@ -9,14 +9,28 @@ module Gitlab
#
class Cache < ::Gitlab::Config::Entry::Node
include ::Gitlab::Config::Entry::Configurable
+ include ::Gitlab::Config::Entry::Validatable
include ::Gitlab::Config::Entry::Attributable
- ALLOWED_KEYS = %i[key untracked paths policy].freeze
+ ALLOWED_KEYS = %i[key untracked paths when policy].freeze
+ ALLOWED_POLICY = %w[pull-push push pull].freeze
DEFAULT_POLICY = 'pull-push'
+ ALLOWED_WHEN = %w[on_success on_failure always].freeze
+ DEFAULT_WHEN = 'on_success'
validations do
- validates :config, allowed_keys: ALLOWED_KEYS
- validates :policy, inclusion: { in: %w[pull-push push pull], message: 'should be pull-push, push, or pull' }, allow_blank: true
+ validates :config, type: Hash, allowed_keys: ALLOWED_KEYS
+ validates :policy,
+ inclusion: { in: ALLOWED_POLICY, message: 'should be pull-push, push, or pull' },
+ allow_blank: true
+
+ with_options allow_nil: true do
+ validates :when,
+ inclusion: {
+ in: ALLOWED_WHEN,
+ message: 'should be on_success, on_failure or always'
+ }
+ end
end
entry :key, Entry::Key,
@@ -28,13 +42,15 @@ module Gitlab
entry :paths, Entry::Paths,
description: 'Specify which paths should be cached across builds.'
- attributes :policy
+ attributes :policy, :when
def value
result = super
result[:key] = key_value
result[:policy] = policy || DEFAULT_POLICY
+ # Use self.when to avoid conflict with reserved word
+ result[:when] = self.when || DEFAULT_WHEN
result
end
diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb
index 9c2e5f641d0..ad0ed00aa6f 100644
--- a/lib/gitlab/ci/config/entry/include.rb
+++ b/lib/gitlab/ci/config/entry/include.rb
@@ -10,7 +10,7 @@ module Gitlab
class Include < ::Gitlab::Config::Entry::Node
include ::Gitlab::Config::Entry::Validatable
- ALLOWED_KEYS = %i[local file remote template artifact job].freeze
+ ALLOWED_KEYS = %i[local file remote template artifact job project ref].freeze
validations do
validates :config, hash_or_string: true
@@ -22,6 +22,10 @@ module Gitlab
if config[:artifact] && config[:job].blank?
errors.add(:config, "must specify the job where to fetch the artifact from")
end
+
+ if config[:project] && config[:file].blank?
+ errors.add(:config, "must specify the file where to fetch the config from")
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb
index b5ce42969a5..b0fd9cef10b 100644
--- a/lib/gitlab/ci/config/entry/jobs.rb
+++ b/lib/gitlab/ci/config/entry/jobs.rb
@@ -7,7 +7,7 @@ module Gitlab
##
# Entry that represents a set of jobs.
#
- class Jobs < ::Gitlab::Config::Entry::Node
+ class Jobs < ::Gitlab::Config::Entry::ComposableHash
include ::Gitlab::Config::Entry::Validatable
validations do
@@ -36,6 +36,10 @@ module Gitlab
end
end
+ def composable_class(name, config)
+ self.class.find_type(name, config)
+ end
+
TYPES = [Entry::Hidden, Entry::Job, Entry::Bridge].freeze
private_constant :TYPES
@@ -49,29 +53,6 @@ module Gitlab
type.matching?(name, config)
end
end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def compose!(deps = nil)
- super do
- @config.each do |name, config|
- node = self.class.find_type(name, config)
- next unless node
-
- factory = ::Gitlab::Config::Entry::Factory.new(node)
- .value(config || {})
- .metadata(name: name)
- .with(key: name, parent: self,
- description: "#{name} job definition.")
-
- @entries[name] = factory.create!
- end
-
- @entries.each_value do |entry|
- entry.compose!(deps)
- end
- end
- end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/ci/config/entry/needs.rb b/lib/gitlab/ci/config/entry/needs.rb
index d7ba8624882..66cd57b8cf3 100644
--- a/lib/gitlab/ci/config/entry/needs.rb
+++ b/lib/gitlab/ci/config/entry/needs.rb
@@ -7,7 +7,7 @@ module Gitlab
##
# Entry that represents a set of needs dependencies.
#
- class Needs < ::Gitlab::Config::Entry::Node
+ class Needs < ::Gitlab::Config::Entry::ComposableArray
include ::Gitlab::Config::Entry::Validatable
validations do
@@ -29,27 +29,16 @@ module Gitlab
end
end
- def compose!(deps = nil)
- super(deps) do
- [@config].flatten.each_with_index do |need, index|
- @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Need)
- .value(need)
- .with(key: "need", parent: self, description: "need definition.") # rubocop:disable CodeReuse/ActiveRecord
- .create!
- end
-
- @entries.each_value do |entry|
- entry.compose!(deps)
- end
- end
- end
-
def value
- values = @entries.values.select(&:type)
+ values = @entries.select(&:type)
values.group_by(&:type).transform_values do |values|
values.map(&:value)
end
end
+
+ def composable_class
+ Entry::Need
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/entry/ports.rb b/lib/gitlab/ci/config/entry/ports.rb
index 01ffcc7dd87..d26b31deca8 100644
--- a/lib/gitlab/ci/config/entry/ports.rb
+++ b/lib/gitlab/ci/config/entry/ports.rb
@@ -7,7 +7,7 @@ module Gitlab
##
# Entry that represents a configuration of the ports of a Docker service.
#
- class Ports < ::Gitlab::Config::Entry::Node
+ class Ports < ::Gitlab::Config::Entry::ComposableArray
include ::Gitlab::Config::Entry::Validatable
validations do
@@ -16,28 +16,8 @@ module Gitlab
validates :config, port_unique: true
end
- def compose!(deps = nil)
- super do
- @entries = []
- @config.each do |config|
- @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Port)
- .value(config || {})
- .with(key: "port", parent: self, description: "port definition.") # rubocop:disable CodeReuse/ActiveRecord
- .create!
- end
-
- @entries.each do |entry|
- entry.compose!(deps)
- end
- end
- end
-
- def value
- @entries.map(&:value)
- end
-
- def descendants
- @entries
+ def composable_class
+ Entry::Port
end
end
end
diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb
index 6af809d46c1..d4ee0978e1b 100644
--- a/lib/gitlab/ci/config/entry/product/matrix.rb
+++ b/lib/gitlab/ci/config/entry/product/matrix.rb
@@ -46,13 +46,11 @@ module Gitlab
end
end
- # rubocop:disable CodeReuse/ActiveRecord
def number_of_generated_jobs
value.sum do |config|
config.values.reduce(1) { |acc, values| acc * values.size }
end
end
- # rubocop:enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb
index ac4f70fb69e..2481989060e 100644
--- a/lib/gitlab/ci/config/entry/product/variables.rb
+++ b/lib/gitlab/ci/config/entry/product/variables.rb
@@ -14,7 +14,7 @@ module Gitlab
validations do
validates :config, variables: { array_values: true }
validates :config, length: {
- minimum: 2,
+ minimum: :minimum,
too_short: 'requires at least %{count} items'
}
end
@@ -28,6 +28,10 @@ module Gitlab
.map { |key, value| [key.to_s, Array(value).map(&:to_s)] }
.to_h
end
+
+ def minimum
+ ::Gitlab::Ci::Features.one_dimensional_matrix_enabled? ? 1 : 2
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb
index 0ae65f43723..f2fd8ac7fd9 100644
--- a/lib/gitlab/ci/config/entry/reports.rb
+++ b/lib/gitlab/ci/config/entry/reports.rb
@@ -15,7 +15,7 @@ module Gitlab
%i[junit codequality sast secret_detection dependency_scanning container_scanning
dast performance browser_performance load_performance license_management license_scanning metrics lsif
dotenv cobertura terraform accessibility cluster_applications
- requirements coverage_fuzzing].freeze
+ requirements coverage_fuzzing api_fuzzing].freeze
attributes ALLOWED_KEYS
@@ -25,6 +25,7 @@ module Gitlab
with_options allow_nil: true do
validates :junit, array_of_strings_or_string: true
+ validates :api_fuzzing, array_of_strings_or_string: true
validates :coverage_fuzzing, array_of_strings_or_string: true
validates :sast, array_of_strings_or_string: true
validates :sast, array_of_strings_or_string: true
diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb
index 2fbc3d9e367..bf74f995e80 100644
--- a/lib/gitlab/ci/config/entry/rules.rb
+++ b/lib/gitlab/ci/config/entry/rules.rb
@@ -4,7 +4,7 @@ module Gitlab
module Ci
class Config
module Entry
- class Rules < ::Gitlab::Config::Entry::Node
+ class Rules < ::Gitlab::Config::Entry::ComposableArray
include ::Gitlab::Config::Entry::Validatable
validations do
@@ -12,24 +12,13 @@ module Gitlab
validates :config, type: Array
end
- def compose!(deps = nil)
- super(deps) do
- @config.each_with_index do |rule, index|
- @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Rules::Rule)
- .value(rule)
- .with(key: "rule", parent: self, description: "rule definition.") # rubocop:disable CodeReuse/ActiveRecord
- .create!
- end
-
- @entries.each_value do |entry|
- entry.compose!(deps)
- end
- end
- end
-
def value
@config
end
+
+ def composable_class
+ Entry::Rules::Rule
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb
index 83baa83711f..44e2903a300 100644
--- a/lib/gitlab/ci/config/entry/services.rb
+++ b/lib/gitlab/ci/config/entry/services.rb
@@ -7,7 +7,7 @@ module Gitlab
##
# Entry that represents a configuration of Docker services.
#
- class Services < ::Gitlab::Config::Entry::Node
+ class Services < ::Gitlab::Config::Entry::ComposableArray
include ::Gitlab::Config::Entry::Validatable
validations do
@@ -15,28 +15,8 @@ module Gitlab
validates :config, services_with_ports_alias_unique: true, if: ->(record) { record.opt(:with_image_ports) }
end
- def compose!(deps = nil)
- super do
- @entries = []
- @config.each do |config|
- @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Service)
- .value(config || {})
- .with(key: "service", parent: self, description: "service definition.") # rubocop:disable CodeReuse/ActiveRecord
- .create!
- end
-
- @entries.each do |entry|
- entry.compose!(deps)
- end
- end
- end
-
- def value
- @entries.map(&:value)
- end
-
- def descendants
- @entries
+ def composable_class
+ Entry::Service
end
end
end
diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb
index c9d0c7cb568..e258f7128fc 100644
--- a/lib/gitlab/ci/config/entry/variables.rb
+++ b/lib/gitlab/ci/config/entry/variables.rb
@@ -10,16 +10,32 @@ module Gitlab
class Variables < ::Gitlab::Config::Entry::Node
include ::Gitlab::Config::Entry::Validatable
+ ALLOWED_VALUE_DATA = %i[value description].freeze
+
validations do
- validates :config, variables: true
+ validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA }
+ end
+
+ def value
+ Hash[@config.map { |key, value| [key.to_s, expand_value(value)[:value]] }]
end
def self.default(**)
{}
end
- def value
- Hash[@config.map { |key, value| [key.to_s, value.to_s] }]
+ def value_with_data
+ Hash[@config.map { |key, value| [key.to_s, expand_value(value)] }]
+ end
+
+ private
+
+ def expand_value(value)
+ if value.is_a?(Hash)
+ { value: value[:value].to_s, description: value[:description] }
+ else
+ { value: value.to_s, description: nil }
+ end
end
end
end
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index e770187b124..1b58e3ec71a 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -10,10 +10,6 @@ module Gitlab
::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true)
end
- def self.job_heartbeats_runner?(project)
- ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true)
- end
-
def self.instance_variables_ui_enabled?
::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true)
end
@@ -35,10 +31,6 @@ module Gitlab
::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true)
end
- def self.bulk_insert_on_create?(project)
- ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true)
- end
-
# NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project`
# is a safe switch to disable the feature for a parituclar project when something went wrong,
# therefore it's not supposed to be enabled by default.
@@ -54,25 +46,25 @@ module Gitlab
Feature.enabled?(:project_transactionless_destroy, project, default_enabled: false)
end
- def self.coverage_report_view?(project)
- ::Feature.enabled?(:coverage_report_view, project, default_enabled: true)
- end
-
- def self.child_of_child_pipeline_enabled?(project)
- ::Feature.enabled?(:ci_child_of_child_pipeline, project, default_enabled: true)
- end
-
def self.trace_overwrite?
::Feature.enabled?(:ci_trace_overwrite, type: :ops, default_enabled: false)
end
def self.accept_trace?(project)
::Feature.enabled?(:ci_enable_live_trace, project) &&
- ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: false)
+ ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true)
+ end
+
+ def self.log_invalid_trace_chunks?(project)
+ ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false)
+ end
+
+ def self.one_dimensional_matrix_enabled?
+ ::Feature.enabled?(:one_dimensional_matrix, default_enabled: true)
end
- def self.new_artifact_file_reader_enabled?(project)
- ::Feature.enabled?(:ci_new_artifact_file_reader, project, default_enabled: false)
+ def self.manual_bridges_enabled?(project)
+ ::Feature.enabled?(:ci_manual_bridges, project, default_enabled: true)
end
end
end
diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb
index 86a9ebfa451..44f2ac23ce3 100644
--- a/lib/gitlab/ci/lint.rb
+++ b/lib/gitlab/ci/lint.rb
@@ -4,10 +4,11 @@ module Gitlab
module Ci
class Lint
class Result
- attr_reader :jobs, :errors, :warnings
+ attr_reader :jobs, :merged_yaml, :errors, :warnings
- def initialize(jobs:, errors:, warnings:)
+ def initialize(jobs:, merged_yaml:, errors:, warnings:)
@jobs = jobs
+ @merged_yaml = merged_yaml
@errors = errors
@warnings = warnings
end
@@ -39,6 +40,7 @@ module Gitlab
Result.new(
jobs: dry_run_convert_to_jobs(pipeline.stages),
+ merged_yaml: pipeline.merged_yaml,
errors: pipeline.error_messages.map(&:content),
warnings: pipeline.warning_messages(limit: ::Gitlab::Ci::Warnings::MAX_LIMIT).map(&:content)
)
@@ -54,6 +56,7 @@ module Gitlab
Result.new(
jobs: static_validation_convert_to_jobs(result),
+ merged_yaml: result.merged_yaml,
errors: result.errors,
warnings: result.warnings.take(::Gitlab::Ci::Warnings::MAX_LIMIT) # rubocop: disable CodeReuse/ActiveRecord
)
diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb
index 5746f38ae5b..50cd703da4a 100644
--- a/lib/gitlab/ci/parsers/test/junit.rb
+++ b/lib/gitlab/ci/parsers/test/junit.rb
@@ -8,12 +8,17 @@ module Gitlab
JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/.freeze
- def parse!(xml_data, test_suite, **args)
+ def parse!(xml_data, test_suite, job:)
root = Hash.from_xml(xml_data)
+ total_parsed = 0
+ max_test_cases = job.max_test_cases_per_report
all_cases(root) do |test_case|
- test_case = create_test_case(test_case, args)
+ test_case = create_test_case(test_case, test_suite, job)
test_suite.add_test_case(test_case)
+ total_parsed += 1
+
+ ensure_test_cases_limited!(total_parsed, max_test_cases)
end
rescue Nokogiri::XML::SyntaxError => e
test_suite.set_suite_error("JUnit XML parsing failed: #{e}")
@@ -23,6 +28,12 @@ module Gitlab
private
+ def ensure_test_cases_limited!(total_parsed, limit)
+ return unless limit > 0 && total_parsed > limit
+
+ raise JunitParserError.new("number of test cases exceeded the limit of #{limit}")
+ end
+
def all_cases(root, parent = nil, &blk)
return unless root.present?
@@ -33,20 +44,24 @@ module Gitlab
all_cases(node['testsuites'], root, &blk) unless parent
# we require at least one level of testsuites or testsuite
- each_case(node['testcase'], &blk) if parent
+ each_case(node['testcase'], node['name'], &blk) if parent
# we allow multiple nested 'testsuite' (eg. PHPUnit)
all_cases(node['testsuite'], root, &blk)
end
end
- def each_case(testcase, &blk)
+ def each_case(testcase, testsuite_name, &blk)
return unless testcase.present?
- [testcase].flatten.compact.map(&blk)
+ [testcase].flatten.compact.each do |tc|
+ tc['suite_name'] = testsuite_name
+
+ yield(tc)
+ end
end
- def create_test_case(data, args)
+ def create_test_case(data, test_suite, job)
if data.key?('failure')
status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED
system_output = data['failure']
@@ -63,6 +78,7 @@ module Gitlab
end
::Gitlab::Ci::Reports::TestCase.new(
+ suite_name: data['suite_name'] || test_suite.name,
classname: data['classname'],
name: data['name'],
file: data['file'],
@@ -70,10 +86,14 @@ module Gitlab
status: status,
system_output: system_output,
attachment: attachment,
- job: args.fetch(:job)
+ job: job
)
end
+ def suite_name(parent, test_suite)
+ parent.dig('testsuite', 'name') || test_suite.name
+ end
+
def attachment_path(data)
return unless data
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index d1882059dd8..06096a33f27 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -16,7 +16,7 @@ module Gitlab
) do
include Gitlab::Utils::StrongMemoize
- def initialize(**params)
+ def initialize(params = {})
params.each do |key, value|
self[key] = value
end
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
index 8ccb33ffd34..c3fbd0c9e24 100644
--- a/lib/gitlab/ci/pipeline/chain/config/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -28,6 +28,8 @@ module Gitlab
error(result.errors.first, config_error: true)
end
+ @pipeline.merged_yaml = result.merged_yaml
+
rescue => ex
Gitlab::ErrorTracking.track_exception(ex,
project_id: project.id,
diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index 34649fe16f3..81ef3bb074d 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -8,7 +8,7 @@ module Gitlab
include Chain::Helpers
def perform!
- BulkInsertableAssociations.with_bulk_insert(enabled: ::Gitlab::Ci::Features.bulk_insert_on_create?(project)) do
+ BulkInsertableAssociations.with_bulk_insert do
pipeline.save!
end
rescue ActiveRecord::RecordInvalid => e
diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb
index a4127ea0be2..8d6fe13c3b9 100644
--- a/lib/gitlab/ci/pipeline/seed/build/cache.rb
+++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb
@@ -13,6 +13,7 @@ module Gitlab
@paths = local_cache.delete(:paths)
@policy = local_cache.delete(:policy)
@untracked = local_cache.delete(:untracked)
+ @when = local_cache.delete(:when)
raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any?
end
@@ -24,7 +25,8 @@ module Gitlab
key: key_string,
paths: @paths,
policy: @policy,
- untracked: @untracked
+ untracked: @untracked,
+ when: @when
}.compact.presence
}.compact
}
diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb
index 15a3c862c9e..8c70dbb6931 100644
--- a/lib/gitlab/ci/reports/test_case.rb
+++ b/lib/gitlab/ci/reports/test_case.rb
@@ -10,9 +10,10 @@ module Gitlab
STATUS_ERROR = 'error'
STATUS_TYPES = [STATUS_ERROR, STATUS_FAILED, STATUS_SUCCESS, STATUS_SKIPPED].freeze
- attr_reader :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job
+ attr_reader :suite_name, :name, :classname, :execution_time, :status, :file, :system_output, :stack_trace, :key, :attachment, :job
def initialize(params)
+ @suite_name = params.fetch(:suite_name)
@name = params.fetch(:name)
@classname = params.fetch(:classname)
@file = params.fetch(:file, nil)
@@ -23,7 +24,7 @@ module Gitlab
@attachment = params.fetch(:attachment, nil)
@job = params.fetch(:job, nil)
- @key = sanitize_key_name("#{classname}_#{name}")
+ @key = hash_key("#{suite_name}_#{classname}_#{name}")
end
def has_attachment?
@@ -42,8 +43,8 @@ module Gitlab
private
- def sanitize_key_name(key)
- key.gsub(/[^0-9A-Za-z]/, '-')
+ def hash_key(key)
+ Digest::SHA256.hexdigest(key)
end
end
end
diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb
index e9b78b841e4..00920dfbd54 100644
--- a/lib/gitlab/ci/reports/test_suite.rb
+++ b/lib/gitlab/ci/reports/test_suite.rb
@@ -12,18 +12,24 @@ module Gitlab
def initialize(name = nil)
@name = name
@test_cases = {}
+ @all_test_cases = []
@total_time = 0.0
- @duplicate_cases = []
end
def add_test_case(test_case)
- @duplicate_cases << test_case if existing_key?(test_case)
-
@test_cases[test_case.status] ||= {}
@test_cases[test_case.status][test_case.key] = test_case
@total_time += test_case.execution_time
end
+ def each_test_case
+ @test_cases.each do |status, test_cases|
+ test_cases.values.each do |test_case|
+ yield test_case
+ end
+ end
+ end
+
# rubocop: disable CodeReuse/ActiveRecord
def total_count
return 0 if suite_error
@@ -86,10 +92,6 @@ module Gitlab
private
- def existing_key?(test_case)
- @test_cases[test_case.status]&.key?(test_case.key)
- end
-
def sort_by_status
@test_cases = @test_cases.sort_by { |status, _| Gitlab::Ci::Reports::TestCase::STATUS_TYPES.index(status) }.to_h
end
diff --git a/lib/gitlab/ci/runner/backoff.rb b/lib/gitlab/ci/runner/backoff.rb
new file mode 100644
index 00000000000..95d7719e9cb
--- /dev/null
+++ b/lib/gitlab/ci/runner/backoff.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Runner
+ ##
+ # Runner Backoff class is an implementation of an exponential backoff
+ # used when a runner communicates with GitLab. We typically use it when a
+ # runner retries sending a build status after we created a build pending
+ # state.
+ #
+ # Backoff is calculated based on the backoff slot which is always a power
+ # of 2:
+ #
+ # 0s - 3s duration -> 1 second backoff
+ # 4s - 7s duration -> 2 seconds backoff
+ # 8s - 15s duration -> 4 seconds backoff
+ # 16s - 31s duration -> 8 seconds backoff
+ # 32s - 63s duration -> 16 seconds backoff
+ # 64s - 127s duration -> 32 seconds backoff
+ # 127s - 256s+ duration -> 64 seconds backoff
+ #
+ # It means that first 15 requests made by a runner will need to respect
+ # following backoffs:
+ #
+ # 0s -> 1 second backoff (backoff started, slot 0, 2^0 backoff)
+ # 1s -> 1 second backoff
+ # 2s -> 1 second backoff
+ # 3s -> 1 seconds backoff
+ # (slot 1 - 2^1 backoff)
+ # 4s -> 2 seconds backoff
+ # 6s -> 2 seconds backoff
+ # (slot 2 - 2^2 backoff)
+ # 8s -> 4 seconds backoff
+ # 12s -> 4 seconds backoff
+ # (slot 3 - 2^3 backoff)
+ # 16s -> 8 seconds backoff
+ # 24s -> 8 seconds backoff
+ # (slot 4 - 2^4 backoff)
+ # 32s -> 16 seconds backoff
+ # 48s -> 16 seconds backoff
+ # (slot 5 - 2^5 backoff)
+ # 64s -> 32 seconds backoff
+ # 96s -> 32 seconds backoff
+ # (slot 6 - 2^6 backoff)
+ # 128s -> 64 seconds backoff
+ #
+ # There is a cap on the backoff - it will never exceed 64 seconds.
+ #
+ class Backoff
+ def initialize(started)
+ @started = started
+
+ if duration < 0
+ raise ArgumentError, 'backoff duration negative'
+ end
+ end
+
+ def duration
+ (Time.current - @started).ceil
+ end
+
+ def slot
+ return 0 if duration < 2
+
+ Math.log(duration, 2).floor - 1
+ end
+
+ def to_seconds
+ 2**[slot, 6].min
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/status/bridge/action.rb b/lib/gitlab/ci/status/bridge/action.rb
new file mode 100644
index 00000000000..1ba4700d9b0
--- /dev/null
+++ b/lib/gitlab/ci/status/bridge/action.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Status
+ module Bridge
+ class Action < Status::Build::Action
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb
index b95565b5e09..d66d4b20bba 100644
--- a/lib/gitlab/ci/status/bridge/common.rb
+++ b/lib/gitlab/ci/status/bridge/common.rb
@@ -14,7 +14,6 @@ module Gitlab
end
def details_path
- return unless Feature.enabled?(:ci_bridge_pipeline_details, subject.project, default_enabled: true)
return unless can?(user, :read_pipeline, downstream_pipeline)
project_pipeline_path(downstream_project, downstream_pipeline)
diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb
index 5d397dba0de..b9bd66cee71 100644
--- a/lib/gitlab/ci/status/bridge/factory.rb
+++ b/lib/gitlab/ci/status/bridge/factory.rb
@@ -6,7 +6,10 @@ module Gitlab
module Bridge
class Factory < Status::Factory
def self.extended_statuses
- [Status::Bridge::Failed]
+ [[Status::Bridge::Failed],
+ [Status::Bridge::Manual],
+ [Status::Bridge::Play],
+ [Status::Bridge::Action]]
end
def self.common_helpers
diff --git a/lib/gitlab/ci/status/bridge/manual.rb b/lib/gitlab/ci/status/bridge/manual.rb
new file mode 100644
index 00000000000..e07e645a34d
--- /dev/null
+++ b/lib/gitlab/ci/status/bridge/manual.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Status
+ module Bridge
+ class Manual < Status::Build::Manual
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/status/bridge/play.rb b/lib/gitlab/ci/status/bridge/play.rb
new file mode 100644
index 00000000000..ae00ef6c2ad
--- /dev/null
+++ b/lib/gitlab/ci/status/bridge/play.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Status
+ module Bridge
+ class Play < Status::Build::Play
+ def has_action?
+ can?(user, :play_job, subject)
+ end
+
+ def self.matches?(bridge, user)
+ bridge.playable?
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb
index 07f37732023..f173964b36c 100644
--- a/lib/gitlab/ci/status/canceled.rb
+++ b/lib/gitlab/ci/status/canceled.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_canceled'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb
index fface4bb97b..33e67314d93 100644
--- a/lib/gitlab/ci/status/created.rb
+++ b/lib/gitlab/ci/status/created.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_created'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb
index 770ed7d4d5a..215d27734a7 100644
--- a/lib/gitlab/ci/status/failed.rb
+++ b/lib/gitlab/ci/status/failed.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_failed'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb
index 50c92add400..eb376df5f22 100644
--- a/lib/gitlab/ci/status/manual.rb
+++ b/lib/gitlab/ci/status/manual.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_manual'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb
index cea7e6ed938..4280ad84534 100644
--- a/lib/gitlab/ci/status/pending.rb
+++ b/lib/gitlab/ci/status/pending.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_pending'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/preparing.rb b/lib/gitlab/ci/status/preparing.rb
index 1ebdbc482b7..e59d1d2eed1 100644
--- a/lib/gitlab/ci/status/preparing.rb
+++ b/lib/gitlab/ci/status/preparing.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_preparing'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb
index ac7dd74cdce..eed1983e60e 100644
--- a/lib/gitlab/ci/status/running.rb
+++ b/lib/gitlab/ci/status/running.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_running'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb
index 16ad1da89e3..e9068c326cf 100644
--- a/lib/gitlab/ci/status/scheduled.rb
+++ b/lib/gitlab/ci/status/scheduled.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_scheduled'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb
index aaec1e1d201..238aa3ab4f9 100644
--- a/lib/gitlab/ci/status/skipped.rb
+++ b/lib/gitlab/ci/status/skipped.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_skipped'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb
index 020f2c5b89f..2a10e60414e 100644
--- a/lib/gitlab/ci/status/success.rb
+++ b/lib/gitlab/ci/status/success.rb
@@ -19,6 +19,10 @@ module Gitlab
def favicon
'favicon_status_success'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/waiting_for_resource.rb b/lib/gitlab/ci/status/waiting_for_resource.rb
index 4c9e706bc51..2026148f752 100644
--- a/lib/gitlab/ci/status/waiting_for_resource.rb
+++ b/lib/gitlab/ci/status/waiting_for_resource.rb
@@ -23,6 +23,10 @@ module Gitlab
def group
'waiting-for-resource'
end
+
+ def details_path
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml
new file mode 100644
index 00000000000..267027a1b8a
--- /dev/null
+++ b/lib/gitlab/ci/templates/AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml
@@ -0,0 +1,11 @@
+stages:
+ - provision
+ - review
+ - production
+
+variables:
+ AUTO_DEVOPS_PLATFORM_TARGET: EC2
+
+include:
+ - template: Jobs/CF-Provision.gitlab-ci.yml
+ - template: Jobs/Deploy/EC2.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index 6966ce88b30..cba13f374f4 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -20,6 +20,7 @@
# * dast: DAST_DISABLED
# * review: REVIEW_DISABLED
# * stop_review: REVIEW_DISABLED
+# * code_intelligence: CODE_INTELLIGENCE_DISABLED
#
# In order to deploy, you must have a Kubernetes cluster configured either
# via a project integration, or via group/project variables.
@@ -159,6 +160,7 @@ include:
- template: Jobs/Build.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
- template: Jobs/Test.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml
- template: Jobs/Code-Quality.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+ - template: Jobs/Code-Intelligence.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml
- template: Jobs/Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
- template: Jobs/Deploy/ECS.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
- template: Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
index 368069844ea..67e58d9ee99 100644
--- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
@@ -1,4 +1,4 @@
-# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options
+# see https://docs.gitlab.com/ee/ci/yaml/README.html for all available options
# you can delete this line if you're not using Docker
image: busybox:latest
diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml
index c3568c0d2c8..0c5850bdb8e 100644
--- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml
@@ -1,7 +1,7 @@
# Based on openjdk:8, already includes lein
image: clojure:lein-2.7.0
# If you need to configure a database, add a `services` section here
-# See https://docs.gitlab.com/ce/ci/services/postgres.html
+# See https://docs.gitlab.com/ee/ci/services/postgres.html
# Make sure you configure the connection as well
before_script:
diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml
index e9301a2638d..538f96c4084 100644
--- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml
@@ -4,7 +4,7 @@ image: "crystallang/crystal:latest"
# Pick zero or more services to be used on all builds.
# Only needed when using a docker container to run your tests in.
-# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
# services:
# - mysql:latest
# - redis:latest
diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
index d35fcb0f807..c657c7e8eb1 100644
--- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
@@ -4,7 +4,7 @@ image: python:latest
# Pick zero or more services to be used on all builds.
# Only needed when using a docker container to run your tests in.
-# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
services:
- mysql:latest
- postgres:latest
@@ -13,7 +13,7 @@ variables:
POSTGRES_DB: database_name
# This folder is cached between builds
-# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
+# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
cache:
paths:
- ~/.cache/pip/
diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml
index 4d4c6a64cd5..7271526ab1b 100644
--- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml
@@ -2,7 +2,7 @@ image: elixir:latest
# Pick zero or more services to be used on all builds.
# Only needed when using a docker container to run your tests in.
-# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
services:
- mysql:latest
- redis:latest
diff --git a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml
new file mode 100644
index 00000000000..31ca68c57d7
--- /dev/null
+++ b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml
@@ -0,0 +1,14 @@
+stages:
+ - provision
+
+cloud_formation:
+ image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-cloudformation:latest'
+ stage: provision
+ script:
+ - gl-cloudformation create-stack
+ rules:
+ - if: '($AUTO_DEVOPS_PLATFORM_TARGET != "EC2") || ($AUTO_DEVOPS_PLATFORM_TARGET != "ECS")'
+ when: never
+ - if: '$CI_KUBERNETES_ACTIVE'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml
new file mode 100644
index 00000000000..83bc5548614
--- /dev/null
+++ b/lib/gitlab/ci/templates/Jobs/Code-Intelligence.gitlab-ci.yml
@@ -0,0 +1,16 @@
+code_intelligence_go:
+ stage: test
+ needs: []
+ allow_failure: true
+ image: sourcegraph/lsif-go:v1
+ rules:
+ - if: $CODE_INTELLIGENCE_DISABLED
+ when: never
+ - if: $CI_COMMIT_BRANCH
+ exists:
+ - '**/*.go'
+ script:
+ - lsif-go
+ artifacts:
+ reports:
+ lsif: dump.lsif
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 568ceceeaa2..ec33020205b 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -9,9 +9,8 @@ code_quality:
DOCKER_TLS_CERTDIR: ""
CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1"
needs: []
- before_script:
- - export SOURCE_CODE=$PWD
script:
+ - export SOURCE_CODE=$PWD
- |
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index 86f946aafda..77216a6e404 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.dast-auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.5"
dast_environment_deploy:
extends: .dast-auto-deploy
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 829fd7a722f..32a207a85d1 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.5"
dependencies: []
review:
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
index 829fd7a722f..8b921305c11 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
@@ -1,5 +1,5 @@
.auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.3"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.0.0-beta.2"
dependencies: []
review:
@@ -91,7 +91,7 @@ canary:
- auto-deploy ensure_namespace
- auto-deploy initialize_tiller
- auto-deploy create_secret
- - auto-deploy deploy canary
+ - auto-deploy deploy canary 50
environment:
name: production
url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
@@ -114,7 +114,6 @@ canary:
- auto-deploy create_secret
- auto-deploy deploy
- auto-deploy delete canary
- - auto-deploy delete rollout
- auto-deploy persist_environment_url
environment:
name: production
@@ -163,9 +162,7 @@ production_manual:
- auto-deploy ensure_namespace
- auto-deploy initialize_tiller
- auto-deploy create_secret
- - auto-deploy deploy rollout $ROLLOUT_PERCENTAGE
- - auto-deploy scale stable $((100-ROLLOUT_PERCENTAGE))
- - auto-deploy delete canary
+ - auto-deploy deploy canary $ROLLOUT_PERCENTAGE
- auto-deploy persist_environment_url
environment:
name: production
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml
new file mode 100644
index 00000000000..ed2172ef7f5
--- /dev/null
+++ b/lib/gitlab/ci/templates/Jobs/Deploy/EC2.gitlab-ci.yml
@@ -0,0 +1,39 @@
+stages:
+ - review
+ - production
+
+.push-and-deploy:
+ image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ec2:latest'
+ script:
+ - gl-ec2 push-to-s3
+ - gl-ec2 deploy-to-ec2
+
+review_ec2:
+ extends: .push-and-deploy
+ stage: review
+ environment:
+ name: review/$CI_COMMIT_REF_NAME
+ rules:
+ - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"'
+ when: never
+ - if: '$CI_KUBERNETES_ACTIVE'
+ when: never
+ - if: '$REVIEW_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
+
+production_ec2:
+ extends: .push-and-deploy
+ stage: production
+ environment:
+ name: production
+ rules:
+ - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "EC2"'
+ when: never
+ - if: '$CI_KUBERNETES_ACTIVE'
+ when: never
+ - if: '$CI_COMMIT_BRANCH != "master"'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
index da474f8ac88..317e8bfab0e 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
@@ -10,6 +10,7 @@
.deploy_to_ecs:
image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest'
+ dependencies: []
script:
- ecs update-task-definition
diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
index 9bde04dff19..5d2c8024524 100644
--- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
@@ -4,7 +4,7 @@ image: php:latest
# Pick zero or more services to be used on all builds.
# Only needed when using a docker container to run your tests in.
-# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
services:
- mysql:latest
@@ -13,7 +13,7 @@ variables:
MYSQL_ROOT_PASSWORD: secret
# This folder is cached between builds
-# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
+# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
cache:
paths:
- vendor/
diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
index 7050b41e045..a9638f564f3 100644
--- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
@@ -1,6 +1,6 @@
apply:
stage: deploy
- image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.29.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.33.0"
environment:
name: production
variables:
diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
index b87178141a1..92379ded77c 100644
--- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
@@ -4,14 +4,14 @@ image: node:latest
# Pick zero or more services to be used on all builds.
# Only needed when using a docker container to run your tests in.
-# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
services:
- mysql:latest
- redis:latest
- postgres:latest
# This folder is cached between builds
-# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
+# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
cache:
paths:
- node_modules/
diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml
index 25ea20e454f..84e8223e69b 100644
--- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml
@@ -19,7 +19,7 @@ before_script:
- php composer.phar install
# Bring in any services we need http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
-# See http://docs.gitlab.com/ce/ci/services/README.html for examples.
+# See http://docs.gitlab.com/ee/ci/services/README.html for examples.
services:
- mysql:5.7
diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
index a683561a455..3a6eac63892 100644
--- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
@@ -1,7 +1,7 @@
image: node:latest
# This folder is cached between builds
-# http://docs.gitlab.com/ce/ci/yaml/README.html#cache
+# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
cache:
paths:
- node_modules/
diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml
index 0c8859dc779..f2f92fe0704 100644
--- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml
@@ -1,5 +1,5 @@
# Template project: https://gitlab.com/pages/jekyll
-# Docs: https://docs.gitlab.com/ce/pages/
+# Docs: https://docs.gitlab.com/ee/pages/
image: ruby:2.6
variables:
diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
index b3cad8b858a..275364afae4 100644
--- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
@@ -4,7 +4,7 @@ image: "ruby:2.5"
# Pick zero or more services to be used on all builds.
# Only needed when using a docker container to run your tests in.
-# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
services:
- mysql:latest
- redis:latest
diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml
index f35470367cc..94117a79d1c 100644
--- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml
@@ -4,7 +4,7 @@ image: "rust:latest"
# Optional: Pick zero or more services to be used on all builds.
# Only needed when using a docker container to run your tests in.
-# Check out: http://docs.gitlab.com/ce/ci/docker/using_docker_images.html#what-is-a-service
+# Check out: http://docs.gitlab.com/ee/ci/docker/using_docker_images.html#what-is-a-service
# services:
# - mysql:latest
# - redis:latest
diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
index 4b957a8f771..e268b48d133 100644
--- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
@@ -35,4 +35,3 @@ variables:
- if: $COVFUZZ_DISABLED
when: never
- if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/
- - if: $CI_RUNNER_EXECUTABLE_ARCH == "linux"
diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
index cc34d23decc..63237e41376 100644
--- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
@@ -1,7 +1,7 @@
# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/license_compliance/
#
# Configure the scanning tool through the environment variables.
-# List of the variables: https://gitlab.com/gitlab-org/security-products/license-management#settings
+# List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings
# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
variables:
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 77ea11d01d1..4418ff18d73 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -9,7 +9,7 @@ variables:
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
- SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec"
+ SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf"
SAST_EXCLUDED_PATHS: "spec, test, tests, tmp"
SAST_ANALYZER_IMAGE_TAG: 2
SCAN_KUBERNETES_MANIFESTS: "false"
@@ -125,6 +125,42 @@ gosec-sast:
exists:
- '**/*.go'
+mobsf-android-sast:
+ extends: .sast-analyzer
+ services:
+ - name: opensecurity/mobile-security-framework-mobsf:latest
+ alias: mobsf
+ image:
+ name: "$SAST_ANALYZER_IMAGE"
+ variables:
+ SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG"
+ rules:
+ - if: $SAST_DISABLED
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $SAST_DEFAULT_ANALYZERS =~ /mobsf/ &&
+ $SAST_EXPERIMENTAL_FEATURES == 'true'
+ exists:
+ - '**/AndroidManifest.xml'
+
+mobsf-ios-sast:
+ extends: .sast-analyzer
+ services:
+ - name: opensecurity/mobile-security-framework-mobsf:latest
+ alias: mobsf
+ image:
+ name: "$SAST_ANALYZER_IMAGE"
+ variables:
+ SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG"
+ rules:
+ - if: $SAST_DISABLED
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $SAST_DEFAULT_ANALYZERS =~ /mobsf/ &&
+ $SAST_EXPERIMENTAL_FEATURES == 'true'
+ exists:
+ - '**/*.xcodeproj/*'
+
nodejs-scan-sast:
extends: .sast-analyzer
image:
@@ -203,6 +239,11 @@ spotbugs-sast:
variables:
SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG"
rules:
+ - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ &&
+ $SAST_EXPERIMENTAL_FEATURES == 'true'
+ exists:
+ - '**/AndroidManifest.xml'
+ when: never
- if: $SAST_DISABLED
when: never
- if: $CI_COMMIT_BRANCH &&
diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
index bde6a0fbebb..6ebff102ccb 100644
--- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
@@ -34,8 +34,8 @@ secret_detection:
when: never
- if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
script:
- - git fetch origin $CI_DEFAULT_BRANCH $CI_BUILD_REF_NAME
- - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt
+ - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME
+ - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt
- export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt
- /analyzer run
- rm "$CI_COMMIT_SHA"_commit_list.txt
diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..b08ccf18b58
--- /dev/null
+++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
@@ -0,0 +1,22 @@
+include:
+ - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
+
+stages:
+ - init
+ - validate
+ - build
+ - deploy
+
+init:
+ extends: .init
+
+validate:
+ extends: .validate
+
+build:
+ extends: .build
+
+deploy:
+ extends: .deploy
+ dependencies:
+ - build
diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..000a1a7f580
--- /dev/null
+++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
@@ -0,0 +1,53 @@
+# Terraform/Base.latest
+#
+# The purpose of this template is to provide flexibility to the user so
+# they are able to only include the jobs that they find interesting.
+#
+# Therefore, this template is not supposed to run any jobs. The idea is to only
+# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs
+#
+# There is a more opinionated template which we suggest the users to abide,
+# which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
+
+image:
+ name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest
+
+before_script:
+ - cd ${TF_ROOT}
+
+variables:
+ TF_ROOT: ${CI_PROJECT_DIR}
+
+cache:
+ key: "${TF_ROOT}"
+ paths:
+ - ${TF_ROOT}/.terraform/
+
+.init: &init
+ stage: init
+ script:
+ - gitlab-terraform init
+
+.validate: &validate
+ stage: validate
+ script:
+ - gitlab-terraform validate
+
+.build: &build
+ stage: build
+ script:
+ - gitlab-terraform plan
+ - gitlab-terraform plan-json
+ artifacts:
+ paths:
+ - ${TF_ROOT}/plan.cache
+ reports:
+ terraform: ${TF_ROOT}/plan.json
+
+.deploy: &deploy
+ stage: deploy
+ script:
+ - gitlab-terraform apply
+ when: manual
+ only:
+ - master
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 348e5472cb4..0222ca021b7 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -16,6 +16,7 @@ module Gitlab
ArchiveError = Class.new(StandardError)
AlreadyArchivedError = Class.new(StandardError)
+ LockedError = Class.new(StandardError)
attr_reader :job
@@ -79,11 +80,9 @@ module Gitlab
job.trace_chunks.any? || current_path.present? || old_trace.present?
end
- def read(should_retry: true, &block)
+ def read(&block)
read_stream(&block)
- rescue Errno::ENOENT
- raise unless should_retry
-
+ rescue Errno::ENOENT, ChunkedIO::FailedToGetChunkError
job.reset
read_stream(&block)
end
@@ -130,6 +129,12 @@ module Gitlab
end
end
+ def lock(&block)
+ in_write_lock(&block)
+ rescue FailedToObtainLockError
+ raise LockedError, "build trace `#{job.id}` is locked"
+ end
+
private
def read_stream
diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb
new file mode 100644
index 00000000000..62532ef1cd2
--- /dev/null
+++ b/lib/gitlab/ci/trace/checksum.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Trace
+ ##
+ # Trace::Checksum class is responsible for calculating a CRC32 checksum
+ # of an entire build trace using partial build trace chunks stored in a
+ # database.
+ #
+ # CRC32 checksum can be easily calculated by combining partial checksums
+ # in a right order.
+ #
+ # Then we compare CRC32 checksum provided by a GitLab Runner and expect
+ # it to be the same as the CRC32 checksum derived from partial chunks.
+ #
+ class Checksum
+ include Gitlab::Utils::StrongMemoize
+
+ attr_reader :build
+
+ def initialize(build)
+ @build = build
+ end
+
+ def valid?
+ return false unless state_crc32.present?
+
+ state_crc32 == chunks_crc32
+ end
+
+ def state_crc32
+ strong_memoize(:state_crc32) { build.pending_state&.crc32 }
+ end
+
+ def chunks_crc32
+ strong_memoize(:chunks_crc32) do
+ trace_chunks.reduce(0) do |crc32, chunk|
+ Zlib.crc32_combine(crc32, chunk.crc32, chunk_size(chunk))
+ end
+ end
+ end
+
+ def last_chunk
+ strong_memoize(:last_chunk) { trace_chunks.max }
+ end
+
+ ##
+ # Trace chunks will be persisted in a database if an object store is
+ # not configured - in that case we do not want to load entire raw data
+ # of all the chunks into memory.
+ #
+ # We ignore `raw_data` attribute instead, and rely on internal build
+ # trace chunk database adapter to handle
+ # `ActiveModel::MissingAttributeError` exception.
+ #
+ # Alternative solution would be separating chunk data from chunk
+ # metadata on the database level too.
+ #
+ def trace_chunks
+ strong_memoize(:trace_chunks) do
+ build.trace_chunks.persisted
+ .select(::Ci::BuildTraceChunk.metadata_attributes)
+ end
+ end
+
+ def chunks_count
+ trace_chunks.to_a.size
+ end
+
+ private
+
+ def chunk_size(chunk)
+ if chunk == last_chunk
+ chunk.size
+ else
+ ::Ci::BuildTraceChunk::CHUNK_SIZE
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb
index 82a7d5fb83c..097436d84ea 100644
--- a/lib/gitlab/ci/trace/metrics.rb
+++ b/lib/gitlab/ci/trace/metrics.rb
@@ -6,8 +6,20 @@ module Gitlab
class Metrics
extend Gitlab::Utils::StrongMemoize
- OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite,
- :accepted, :finalized, :discarded, :conflict].freeze
+ OPERATIONS = [
+ :appended, # new trace data has been written to a chunk
+ :streamed, # new trace data has been sent by a runner
+ :chunked, # new trace chunk has been created
+ :mutated, # trace has been mutated when removing secrets
+ :overwrite, # runner requested overwritting a build trace
+ :accepted, # scheduled chunks for migration and responded with 202
+ :finalized, # all live build trace chunks have been persisted
+ :discarded, # failed to persist live chunks before timeout
+ :conflict, # runner has sent unrecognized build state details
+ :locked, # build trace has been locked by a different mechanism
+ :stalled, # failed to migrate chunk due to a worker duplication
+ :invalid # malformed build trace has been detected using CRC32
+ ].freeze
def increment_trace_operation(operation: :unknown)
unless OPERATIONS.include?(operation)
@@ -18,7 +30,11 @@ module Gitlab
end
def increment_trace_bytes(size)
- self.class.trace_bytes.increment(by: size.to_i)
+ self.class.trace_bytes.increment({}, size.to_i)
+ end
+
+ def observe_migration_duration(seconds)
+ self.class.finalize_histogram.observe({}, seconds.to_f)
end
def self.trace_operations
@@ -38,6 +54,17 @@ module Gitlab
Gitlab::Metrics.counter(name, comment)
end
end
+
+ def self.finalize_histogram
+ strong_memoize(:finalize_histogram) do
+ name = :gitlab_ci_trace_finalize_duration_seconds
+ comment = 'Duration of build trace chunks migration to object storage'
+ buckets = [0.1, 0.5, 1.0, 2.0, 3.0, 5.0, 7.0, 10.0, 20.0, 30.0, 60.0, 300.0]
+ labels = {}
+
+ ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+ end
+ end
end
end
end
diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb
index a072036daa8..84a9280e507 100644
--- a/lib/gitlab/ci/variables/collection/item.rb
+++ b/lib/gitlab/ci/variables/collection/item.rb
@@ -36,9 +36,9 @@ module Gitlab
def self.fabricate(resource)
case resource
when Hash
- self.new(resource.symbolize_keys)
+ self.new(**resource.symbolize_keys)
when ::Ci::HasVariable
- self.new(resource.to_runner_variable)
+ self.new(**resource.to_runner_variable)
when self
resource.dup
else
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index 68f61e52df7..52a00e41214 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -95,6 +95,14 @@ module Gitlab
}.compact }.compact
end
+ def merged_yaml
+ @ci_config&.to_hash&.to_yaml
+ end
+
+ def variables_with_data
+ @ci_config.variables_with_data
+ end
+
private
def variables
diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb
index 14eac474e27..a6638b2cbc8 100644
--- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb
+++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb
@@ -19,6 +19,11 @@ module Gitlab
def run!
log_info("Looking for orphan LFS files for project #{project.name_with_namespace}")
+ if project.lfs_objects.empty?
+ log_info("Project #{project.name_with_namespace} is linked to 0 LFS objects. Nothing to do")
+ return
+ end
+
remove_orphan_references
end
diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
index 92c799875b5..822012e0ed6 100644
--- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb
+++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb
@@ -5,8 +5,8 @@ module Gitlab
class PumaWorkerKillerInitializer
def self.start(
puma_options,
- puma_per_worker_max_memory_mb: 850,
- puma_master_max_memory_mb: 550,
+ puma_per_worker_max_memory_mb: 1024,
+ puma_master_max_memory_mb: 800,
additional_puma_dev_max_memory_mb: 200
)
require 'puma_worker_killer'
diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb
index 909d0536b5f..7d36f2f12cf 100644
--- a/lib/gitlab/code_navigation_path.rb
+++ b/lib/gitlab/code_navigation_path.rb
@@ -13,7 +13,6 @@ module Gitlab
end
def full_json_path_for(path)
- return unless Feature.enabled?(:code_navigation, project, default_enabled: true)
return unless build
raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json", file_type: :lsif)
diff --git a/lib/gitlab/config/entry/composable_array.rb b/lib/gitlab/config/entry/composable_array.rb
new file mode 100644
index 00000000000..e7ad259e826
--- /dev/null
+++ b/lib/gitlab/config/entry/composable_array.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Config
+ module Entry
+ ##
+ # Entry that represents a composable array definition
+ #
+ class ComposableArray < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+ include Gitlab::Utils::StrongMemoize
+
+ # TODO: Refactor `Validatable` code so that validations can apply to a child class
+ # See: https://gitlab.com/gitlab-org/gitlab/-/issues/263231
+ validations do
+ validates :config, type: Array
+ end
+
+ def compose!(deps = nil)
+ super do
+ @entries = Array(@entries)
+
+ # TODO: Isolate handling for a hash via: `[@config].flatten` to the `Needs` entry
+ # See: https://gitlab.com/gitlab-org/gitlab/-/issues/264376
+ [@config].flatten.each_with_index do |value, index|
+ raise ArgumentError, 'Missing Composable class' unless composable_class
+
+ composable_class_name = composable_class.name.demodulize.underscore
+
+ @entries << ::Gitlab::Config::Entry::Factory.new(composable_class)
+ .value(value)
+ .with(key: composable_class_name, parent: self, description: "#{composable_class_name} definition") # rubocop:disable CodeReuse/ActiveRecord
+ .create!
+ end
+
+ @entries.each do |entry|
+ entry.compose!(deps)
+ end
+ end
+ end
+
+ def value
+ @entries.map(&:value)
+ end
+
+ def descendants
+ @entries
+ end
+
+ def composable_class
+ strong_memoize(:composable_class) do
+ opt(:composable_class)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/config/entry/composable_hash.rb b/lib/gitlab/config/entry/composable_hash.rb
new file mode 100644
index 00000000000..9531b7e56fd
--- /dev/null
+++ b/lib/gitlab/config/entry/composable_hash.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Config
+ module Entry
+ ##
+ # Entry that represents a composable hash definition
+ # Where each hash key can be any value written by the user
+ #
+ class ComposableHash < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+
+ # TODO: Refactor `Validatable` code so that validations can apply to a child class
+ # See: https://gitlab.com/gitlab-org/gitlab/-/issues/263231
+ validations do
+ validates :config, type: Hash
+ end
+
+ def compose!(deps = nil)
+ super do
+ @config.each do |name, config|
+ entry_class = composable_class(name, config)
+ raise ArgumentError, 'Missing Composable class' unless entry_class
+
+ entry_class_name = entry_class.name.demodulize.underscore
+
+ factory = ::Gitlab::Config::Entry::Factory.new(entry_class)
+ .value(config || {})
+ .with(key: name, parent: self, description: "#{name} #{entry_class_name} definition") # rubocop:disable CodeReuse/ActiveRecord
+ .metadata(name: name)
+
+ @entries[name] = factory.create!
+ end
+
+ @entries.each_value do |entry|
+ entry.compose!(deps)
+ end
+ end
+ end
+
+ def composable_class(name, config)
+ opt(:composable_class)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb
index 7c5ffaa7621..f76c98f7cbf 100644
--- a/lib/gitlab/config/entry/factory.rb
+++ b/lib/gitlab/config/entry/factory.rb
@@ -79,7 +79,7 @@ module Gitlab
end
def fabricate(entry_class, value = nil)
- entry_class.new(value, @metadata) do |node|
+ entry_class.new(value, **@metadata) do |node|
node.key = @attributes[:key]
node.parent = @attributes[:parent]
node.default = @attributes[:default]
diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb
index 415f6f77214..be7d26fed4e 100644
--- a/lib/gitlab/config/entry/legacy_validation_helpers.rb
+++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb
@@ -50,6 +50,12 @@ module Gitlab
variables.values.flatten(1).all?(&method(:validate_alphanumeric))
end
+ def validate_string_or_hash_value_variables(variables, allowed_value_data)
+ variables.is_a?(Hash) &&
+ variables.keys.all?(&method(:validate_alphanumeric)) &&
+ variables.values.all? { |value| validate_string_or_hash_value_variable(value, allowed_value_data) }
+ end
+
def validate_alphanumeric(value)
validate_string(value) || validate_integer(value)
end
@@ -62,6 +68,14 @@ module Gitlab
value.is_a?(String) || value.is_a?(Symbol)
end
+ def validate_string_or_hash_value_variable(value, allowed_value_data)
+ if value.is_a?(Hash)
+ (value.keys - allowed_value_data).empty? && value.values.all?(&method(:validate_alphanumeric))
+ else
+ validate_alphanumeric(value)
+ end
+ end
+
def validate_regexp(value)
Gitlab::UntrustedRegexp::RubySyntax.valid?(value)
end
diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb
index 315f1947e2c..ee28891a174 100644
--- a/lib/gitlab/config/entry/simplifiable.rb
+++ b/lib/gitlab/config/entry/simplifiable.rb
@@ -19,7 +19,7 @@ module Gitlab
entry = self.class.entry_class(strategy)
- @subject = entry.new(config, metadata, &blk)
+ @subject = entry.new(config, **metadata, &blk)
super(@subject)
end
diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb
index a7ec98ace6e..2a386657e0b 100644
--- a/lib/gitlab/config/entry/validators.rb
+++ b/lib/gitlab/config/entry/validators.rb
@@ -274,6 +274,8 @@ module Gitlab
def validate_each(record, attribute, value)
if options[:array_values]
validate_key_array_values(record, attribute, value)
+ elsif options[:allowed_value_data]
+ validate_key_hash_values(record, attribute, value, options[:allowed_value_data])
else
validate_key_values(record, attribute, value)
end
@@ -290,6 +292,12 @@ module Gitlab
record.errors.add(attribute, 'should be a hash of key value pairs, value can be an array')
end
end
+
+ def validate_key_hash_values(record, attribute, value, allowed_value_data)
+ unless validate_string_or_hash_value_variables(value, allowed_value_data)
+ record.errors.add(attribute, 'should be a hash of key value pairs, value can be a hash')
+ end
+ end
end
class ExpressionValidator < ActiveModel::EachValidator
diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb
index 53406af2c4e..047600af267 100644
--- a/lib/gitlab/conflict/file_collection.rb
+++ b/lib/gitlab/conflict/file_collection.rb
@@ -21,11 +21,13 @@ module Gitlab
def resolve(user, commit_message, files)
msg = commit_message || default_commit_message
resolution = Gitlab::Git::Conflict::Resolution.new(user, files, msg)
- args = {
+
+ resolver.resolve_conflicts(
+ @source_repo,
+ resolution,
source_branch: merge_request.source_branch,
target_branch: merge_request.target_branch
- }
- resolver.resolve_conflicts(@source_repo, resolution, args)
+ )
ensure
@merge_request.clear_memoized_shas
end
diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb
index 954934518d7..7b01db125a9 100644
--- a/lib/gitlab/danger/commit_linter.rb
+++ b/lib/gitlab/danger/commit_linter.rb
@@ -10,7 +10,7 @@ module Gitlab
MAX_LINE_LENGTH = 72
MAX_CHANGED_FILES_IN_COMMIT = 3
MAX_CHANGED_LINES_IN_COMMIT = 30
- SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(#|!|&|%)\d+\b}.freeze
+ SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(?<!`)(#|!|&|%)\d+(?<!`)}.freeze
DEFAULT_SUBJECT_DESCRIPTION = 'commit subject'
WIP_PREFIX = 'WIP: '
PROBLEMS = {
@@ -118,7 +118,7 @@ module Gitlab
next unless line_too_long?(line)
- url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length } # rubocop:disable CodeReuse/ActiveRecord
+ url_size = line.scan(%r((https?://\S+))).sum { |(url)| url.length }
# If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but
# only if the line _without_ the URL does not exceed this limit.
diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb
index 3626ec5bf5b..783a5f1715c 100644
--- a/lib/gitlab/danger/helper.rb
+++ b/lib/gitlab/danger/helper.rb
@@ -123,7 +123,8 @@ module Gitlab
none: "",
qa: "~QA",
test: "~test ~Quality for `spec/features/*`",
- engineering_productivity: '~"Engineering Productivity" for CI, Danger'
+ engineering_productivity: '~"Engineering Productivity" for CI, Danger',
+ ci_template: '~"ci::templates"'
}.freeze
# First-match win, so be sure to put more specific regex at the top...
CATEGORIES = {
@@ -176,6 +177,8 @@ module Gitlab
%r{(CODEOWNERS)} => :engineering_productivity,
%r{(tests.yml)} => :engineering_productivity,
+ %r{\Alib/gitlab/ci/templates} => :ci_template,
+
%r{\A(ee/)?spec/features/} => :test,
%r{\A(ee/)?spec/support/shared_examples/features/} => :test,
%r{\A(ee/)?spec/support/shared_contexts/features/} => :test,
@@ -214,6 +217,12 @@ module Gitlab
title.gsub(DRAFT_REGEX, '').gsub(/`/, '\\\`')
end
+ def draft_mr?
+ return false unless gitlab_helper
+
+ DRAFT_REGEX.match?(gitlab_helper.mr_json['title'])
+ end
+
def security_mr?
return false unless gitlab_helper
diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb
index a6866868e6c..23f877b4e0f 100644
--- a/lib/gitlab/danger/roulette.rb
+++ b/lib/gitlab/danger/roulette.rb
@@ -52,6 +52,11 @@ module Gitlab
# Fetch an already picked backend maintainer, or pick one otherwise
spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer
end
+ when :ci_template
+ if spin.maintainer.nil?
+ # Fetch an already picked backend maintainer, or pick one otherwise
+ spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer
+ end
end
end
@@ -146,13 +151,19 @@ module Gitlab
%i[reviewer traintainer maintainer].map do |role|
spin_role_for_category(team, role, project, category)
end
+ hungry_reviewers = reviewers.select { |member| member.hungry }
+ hungry_traintainers = traintainers.select { |member| member.hungry }
# TODO: take CODEOWNERS into account?
# https://gitlab.com/gitlab-org/gitlab/issues/26723
- # Make traintainers have triple the chance to be picked as a reviewer
random = new_random(mr_source_branch)
- reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random, timezone_experiment: timezone_experiment)
+
+ # Make hungry traintainers have 4x the chance to be picked as a reviewer
+ # Make traintainers have 3x the chance to be picked as a reviewer
+ # Make hungry reviewers have 2x the chance to be picked as a reviewer
+ weighted_reviewers = reviewers + hungry_reviewers + traintainers + traintainers + traintainers + hungry_traintainers
+ reviewer = spin_for_person(weighted_reviewers, random: random, timezone_experiment: timezone_experiment)
maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment)
Spin.new(category, reviewer, maintainer, false, timezone_experiment)
diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb
index ebd96be40d7..4481977db15 100644
--- a/lib/gitlab/danger/teammate.rb
+++ b/lib/gitlab/danger/teammate.rb
@@ -3,7 +3,7 @@
module Gitlab
module Danger
class Teammate
- attr_reader :options, :username, :name, :role, :projects, :available, :tz_offset_hours
+ attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :tz_offset_hours
# The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb
def initialize(options = {})
@@ -14,6 +14,7 @@ module Gitlab
@role = options['role']
@projects = options['projects']
@available = options['available']
+ @hungry = options['hungry']
@tz_offset_hours = options['tz_offset_hours']
end
@@ -31,10 +32,8 @@ module Gitlab
projects&.has_key?(name)
end
- # Traintainers also count as reviewers
def reviewer?(project, category, labels)
- has_capability?(project, category, :reviewer, labels) ||
- traintainer?(project, category, labels)
+ has_capability?(project, category, :reviewer, labels)
end
def traintainer?(project, category, labels)
diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb
index 70587b3132a..87ebe832862 100644
--- a/lib/gitlab/data_builder/deployment.rb
+++ b/lib/gitlab/data_builder/deployment.rb
@@ -20,8 +20,8 @@ module Gitlab
environment: deployment.environment.name,
project: deployment.project.hook_attrs,
short_sha: deployment.short_sha,
- user: deployment.user.hook_attrs,
- user_url: Gitlab::UrlBuilder.build(deployment.user),
+ user: deployment.deployed_by.hook_attrs,
+ user_url: Gitlab::UrlBuilder.build(deployment.deployed_by),
commit_url: Gitlab::UrlBuilder.build(deployment.commit),
commit_title: deployment.commit.title
}
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index accc6330253..45d271a2fd4 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -92,10 +92,6 @@ module Gitlab
@version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1]
end
- def self.postgresql_9_or_less?
- version.to_f < 10
- end
-
def self.postgresql_minimum_supported_version?
version.to_f >= MINIMUM_POSTGRES_VERSION
end
@@ -127,28 +123,6 @@ module Gitlab
# ignore - happens when Rake tasks yet have to create a database, e.g. for testing
end
- # map some of the function names that changed between PostgreSQL 9 and 10
- # https://wiki.postgresql.org/wiki/New_in_postgres_10
- def self.pg_wal_lsn_diff
- Gitlab::Database.postgresql_9_or_less? ? 'pg_xlog_location_diff' : 'pg_wal_lsn_diff'
- end
-
- def self.pg_current_wal_insert_lsn
- Gitlab::Database.postgresql_9_or_less? ? 'pg_current_xlog_insert_location' : 'pg_current_wal_insert_lsn'
- end
-
- def self.pg_last_wal_receive_lsn
- Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_receive_location' : 'pg_last_wal_receive_lsn'
- end
-
- def self.pg_last_wal_replay_lsn
- Gitlab::Database.postgresql_9_or_less? ? 'pg_last_xlog_replay_location' : 'pg_last_wal_replay_lsn'
- end
-
- def self.pg_last_xact_replay_timestamp
- 'pg_last_xact_replay_timestamp'
- end
-
def self.nulls_last_order(field, direction = 'ASC')
Arel.sql("#{field} #{direction} NULLS LAST")
end
@@ -276,6 +250,20 @@ module Gitlab
false
end
+ def self.system_id
+ row = connection.execute('SELECT system_identifier FROM pg_control_system()').first
+
+ row['system_identifier']
+ end
+
+ def self.get_write_location(ar_connection)
+ row = ar_connection
+ .select_all("SELECT pg_current_wal_insert_lsn()::text AS location")
+ .first
+
+ row['location'] if row
+ end
+
private_class_method :database_version
def self.add_post_migrate_path_to_rails(force: false)
diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb
index 1762b81b7d8..11d9881aac2 100644
--- a/lib/gitlab/database/batch_count.rb
+++ b/lib/gitlab/database/batch_count.rb
@@ -8,15 +8,20 @@
# In order to not use a possible complex time consuming query when calculating min and max for batch_distinct_count
# the start and finish can be sent specifically
#
+# Grouped relations can be used as well. However, the preferred batch count should be around 10K because group by count is more expensive.
+#
# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705
#
# Examples:
# extend ::Gitlab::Database::BatchCount
# batch_count(User.active)
# batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id)
+# batch_count(Namespace.group(:type))
# batch_distinct_count(::Project, :creator_id)
# batch_distinct_count(::Project.with_active_services.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id))
+# batch_distinct_count(Project.group(:visibility_level), :creator_id)
# batch_sum(User, :sign_in_count)
+# batch_sum(Issue.group(:state_id), :weight))
module Gitlab
module Database
module BatchCount
@@ -77,34 +82,45 @@ module Gitlab
raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0
return FALLBACK if unwanted_configuration?(finish, batch_size, start)
- counter = 0
+ results = nil
batch_start = start
while batch_start <= finish
+ batch_relation = build_relation_batch(batch_start, batch_start + batch_size, mode)
begin
- counter += batch_fetch(batch_start, batch_start + batch_size, mode)
+ results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend
batch_start += batch_size
- rescue ActiveRecord::QueryCanceled
+ rescue ActiveRecord::QueryCanceled => error
# retry with a safe batch size & warmer cache
if batch_size >= 2 * MIN_REQUIRED_BATCH_SIZE
batch_size /= 2
else
+ log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error)
return FALLBACK
end
end
sleep(SLEEP_TIME_IN_SECONDS)
end
- counter
+ results
end
- def batch_fetch(start, finish, mode)
- # rubocop:disable GitlabSecurity/PublicSend
- @relation.select(@column).public_send(mode).where(between_condition(start, finish)).send(@operation, *@operation_args)
+ def merge_results(results, object)
+ return object unless results
+
+ if object.is_a?(Hash)
+ results.merge!(object) { |_, a, b| a + b }
+ else
+ results + object
+ end
end
private
+ def build_relation_batch(start, finish, mode)
+ @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend
+ end
+
def batch_size_for_mode_and_operation(mode, operation)
return DEFAULT_SUM_BATCH_SIZE if operation == :sum
@@ -118,11 +134,11 @@ module Gitlab
end
def actual_start(start)
- start || @relation.minimum(@column) || 0
+ start || @relation.unscope(:group, :having).minimum(@column) || 0
end
def actual_finish(finish)
- finish || @relation.maximum(@column) || 0
+ finish || @relation.unscope(:group, :having).maximum(@column) || 0
end
def check_mode!(mode)
@@ -130,6 +146,20 @@ module Gitlab
raise 'Use distinct count for optimized distinct counting' if @relation.limit(1).distinct_value.present? && mode != :distinct
raise 'Use distinct count only with non id fields' if @column == :id && mode == :distinct
end
+
+ def log_canceled_batch_fetch(batch_start, mode, query, error)
+ Gitlab::AppJsonLogger
+ .error(
+ event: 'batch_count',
+ relation: @relation.table_name,
+ operation: @operation,
+ operation_args: @operation_args,
+ start: batch_start,
+ mode: mode,
+ query: query,
+ message: "Query has been canceled with message: #{error.message}"
+ )
+ end
end
end
end
diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb
new file mode 100644
index 00000000000..1403d561890
--- /dev/null
+++ b/lib/gitlab/database/bulk_update.rb
@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ # Constructs queries of the form:
+ #
+ # with cte(a, b, c) as (
+ # select * from (values (:x, :y, :z), (:q, :r, :s)) as t
+ # )
+ # update table set b = cte.b, c = cte.c where a = cte.a
+ #
+ # Which is useful if you want to update a set of records in a single query
+ # but cannot express the update as a calculation (i.e. you have arbitrary
+ # updates to perform).
+ #
+ # The requirements are that the table must have an ID column used to
+ # identify the rows to be updated.
+ #
+ # Usage:
+ #
+ # mapping = {
+ # issue_a => { title: 'This title', relative_position: 100 },
+ # issue_b => { title: 'That title', relative_position: 173 }
+ # }
+ #
+ # ::Gitlab::Database::BulkUpdate.execute(%i[title relative_position], mapping)
+ #
+ # Note that this is a very low level tool, and operates on the raw column
+ # values. Enums/state fields must be translated into their underlying
+ # representations, for example, and no hooks will be called.
+ #
+ module BulkUpdate
+ LIST_SEPARATOR = ', '
+
+ class Setter
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(model, columns, mapping)
+ @table_name = model.table_name
+ @connection = model.connection
+ @columns = self.class.column_definitions(model, columns)
+ @mapping = self.class.value_mapping(mapping)
+ end
+
+ def update!
+ if without_prepared_statement?
+ # A workaround for https://github.com/rails/rails/issues/24893
+ # When prepared statements are prevented (such as when using the
+ # query counter or in omnibus by default), we cannot call
+ # `exec_update`, since that will discard the bindings.
+ connection.send(:exec_no_cache, sql, log_name, params) # rubocop: disable GitlabSecurity/PublicSend
+ else
+ connection.exec_update(sql, log_name, params)
+ end
+ end
+
+ def self.column_definitions(model, columns)
+ raise ArgumentError, 'invalid columns' if columns.blank? || columns.any? { |c| !c.is_a?(Symbol) }
+ raise ArgumentError, 'cannot set ID' if columns.include?(:id)
+
+ ([:id] | columns).map { |name| column_definition(model, name) }
+ end
+
+ def self.column_definition(model, name)
+ definition = model.column_for_attribute(name)
+ raise ArgumentError, "Unknown column: #{name}" unless definition.type
+
+ definition
+ end
+
+ def self.value_mapping(mapping)
+ raise ArgumentError, 'invalid mapping' if mapping.blank?
+ raise ArgumentError, 'invalid mapping value' if mapping.any? { |_k, v| !v.is_a?(Hash) }
+
+ mapping
+ end
+
+ private
+
+ attr_reader :table_name, :connection, :columns, :mapping
+
+ def log_name
+ strong_memoize(:log_name) do
+ "BulkUpdate #{table_name} #{columns.drop(1).map(&:name)}:#{mapping.size}"
+ end
+ end
+
+ def params
+ mapping.flat_map do |k, v|
+ obj_id = k.try(:id) || k
+ v = v.merge(id: obj_id)
+ columns.map { |c| query_attribute(c, k, v.with_indifferent_access) }
+ end
+ end
+
+ # A workaround for https://github.com/rails/rails/issues/24893
+ # We need to detect if prepared statements have been disabled.
+ def without_prepared_statement?
+ strong_memoize(:without_prepared_statement) do
+ connection.send(:without_prepared_statement?, [1]) # rubocop: disable GitlabSecurity/PublicSend
+ end
+ end
+
+ def query_attribute(column, key, values)
+ value = values[column.name]
+ key[column.name] = value if key.try(:id) # optimistic update
+ ActiveRecord::Relation::QueryAttribute.from_user(nil, value, ActiveModel::Type.lookup(column.type))
+ end
+
+ def values
+ counter = 0
+ typed = false
+
+ mapping.map do |k, v|
+ binds = columns.map do |c|
+ bind = "$#{counter += 1}"
+ # PG is not great at inferring types - help it for the first row.
+ bind += "::#{c.sql_type}" unless typed
+ bind
+ end
+ typed = true
+
+ "(#{list_of(binds)})"
+ end
+ end
+
+ def list_of(list)
+ list.join(LIST_SEPARATOR)
+ end
+
+ def sql
+ <<~SQL
+ WITH cte(#{list_of(cte_columns)}) AS (VALUES #{list_of(values)})
+ UPDATE #{table_name} SET #{list_of(updates)} FROM cte WHERE cte_id = id
+ SQL
+ end
+
+ def column_names
+ strong_memoize(:column_names) { columns.map(&:name) }
+ end
+
+ def cte_columns
+ strong_memoize(:cte_columns) do
+ column_names.map do |c|
+ connection.quote_column_name("cte_#{c}")
+ end
+ end
+ end
+
+ def updates
+ column_names.zip(cte_columns).drop(1).map do |dest, src|
+ "#{connection.quote_column_name(dest)} = cte.#{src}"
+ end
+ end
+ end
+
+ def self.execute(columns, mapping, &to_class)
+ raise ArgumentError if mapping.blank?
+
+ entries_by_class = mapping.group_by { |k, v| block_given? ? to_class.call(k) : k.class }
+
+ entries_by_class.each do |model, entries|
+ Setter.new(model, columns, entries).update!
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/concurrent_reindex.rb b/lib/gitlab/database/concurrent_reindex.rb
deleted file mode 100644
index 485ab35e55d..00000000000
--- a/lib/gitlab/database/concurrent_reindex.rb
+++ /dev/null
@@ -1,143 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- class ConcurrentReindex
- include Gitlab::Utils::StrongMemoize
- include MigrationHelpers
-
- ReindexError = Class.new(StandardError)
-
- PG_IDENTIFIER_LENGTH = 63
- TEMPORARY_INDEX_PREFIX = 'tmp_reindex_'
- REPLACED_INDEX_PREFIX = 'old_reindex_'
-
- attr_reader :index_name, :logger
-
- def initialize(index_name, logger:)
- @index_name = index_name
- @logger = logger
- end
-
- def execute
- raise ReindexError, "index #{index_name} does not exist" unless index_exists?
-
- raise ReindexError, 'UNIQUE indexes are currently not supported' if index_unique?
-
- logger.debug("dropping dangling index from previous run: #{replacement_index_name}")
- remove_replacement_index
-
- begin
- create_replacement_index
-
- unless replacement_index_valid?
- message = 'replacement index was created as INVALID'
- logger.error("#{message}, cleaning up")
- raise ReindexError, "failed to reindex #{index_name}: #{message}"
- end
-
- swap_replacement_index
- rescue Gitlab::Database::WithLockRetries::AttemptsExhaustedError => e
- logger.error('failed to obtain the required database locks to swap the indexes, cleaning up')
- raise ReindexError, e.message
- rescue ActiveRecord::ActiveRecordError, PG::Error => e
- logger.error("database error while attempting reindex of #{index_name}: #{e.message}")
- raise ReindexError, e.message
- ensure
- logger.info("dropping unneeded replacement index: #{replacement_index_name}")
- remove_replacement_index
- end
- end
-
- private
-
- def connection
- @connection ||= ActiveRecord::Base.connection
- end
-
- def replacement_index_name
- @replacement_index_name ||= constrained_index_name(TEMPORARY_INDEX_PREFIX)
- end
-
- def index
- strong_memoize(:index) do
- find_index(index_name)
- end
- end
-
- def index_exists?
- !index.nil?
- end
-
- def index_unique?
- index.indisunique
- end
-
- def constrained_index_name(prefix)
- "#{prefix}#{index_name}".slice(0, PG_IDENTIFIER_LENGTH)
- end
-
- def create_replacement_index
- create_replacement_index_statement = index.indexdef
- .sub(/CREATE INDEX/, 'CREATE INDEX CONCURRENTLY')
- .sub(/#{index_name}/, replacement_index_name)
-
- logger.info("creating replacement index #{replacement_index_name}")
- logger.debug("replacement index definition: #{create_replacement_index_statement}")
-
- disable_statement_timeout do
- connection.execute(create_replacement_index_statement)
- end
- end
-
- def replacement_index_valid?
- find_index(replacement_index_name).indisvalid
- end
-
- def find_index(index_name)
- record = connection.select_one(<<~SQL)
- SELECT
- pg_index.indisunique,
- pg_index.indisvalid,
- pg_indexes.indexdef
- FROM pg_index
- INNER JOIN pg_class ON pg_class.oid = pg_index.indexrelid
- INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid
- INNER JOIN pg_indexes ON pg_class.relname = pg_indexes.indexname
- WHERE pg_namespace.nspname = 'public'
- AND pg_class.relname = #{connection.quote(index_name)}
- SQL
-
- OpenStruct.new(record) if record
- end
-
- def swap_replacement_index
- replaced_index_name = constrained_index_name(REPLACED_INDEX_PREFIX)
-
- logger.info("swapping replacement index #{replacement_index_name} with #{index_name}")
-
- with_lock_retries do
- rename_index(index_name, replaced_index_name)
- rename_index(replacement_index_name, index_name)
- rename_index(replaced_index_name, replacement_index_name)
- end
- end
-
- def rename_index(old_index_name, new_index_name)
- connection.execute("ALTER INDEX #{old_index_name} RENAME TO #{new_index_name}")
- end
-
- def remove_replacement_index
- disable_statement_timeout do
- connection.execute("DROP INDEX CONCURRENTLY IF EXISTS #{replacement_index_name}")
- end
- end
-
- def with_lock_retries(&block)
- arguments = { klass: self.class, logger: logger }
-
- Gitlab::Database::WithLockRetries.new(arguments).run(raise_on_exhaustion: true, &block)
- end
- end
- end
-end
diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb
index e226ed7613a..89190320cf9 100644
--- a/lib/gitlab/database/count/reltuples_count_strategy.rb
+++ b/lib/gitlab/database/count/reltuples_count_strategy.rb
@@ -74,8 +74,9 @@ module Gitlab
def get_statistics(table_names, check_statistics: true)
time = 6.hours.ago
- query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)")
+ query = PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid")
.where(relname: table_names)
+ .where('schemaname = current_schema()')
.select('pg_class.relname AS table_name, reltuples::bigint AS estimate')
if check_statistics
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 723f0f6a308..66b6ce1ec55 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -176,7 +176,7 @@ module Gitlab
name: name.presence || concurrent_foreign_key_name(source, column)
}
- if foreign_key_exists?(source, target, options)
+ if foreign_key_exists?(source, target, **options)
warning_message = "Foreign key not created because it exists already " \
"(this may be due to an aborted migration or similar): " \
"source: #{source}, target: #{target}, column: #{options[:column]}, "\
@@ -330,13 +330,13 @@ module Gitlab
# * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION`
# * +logger+ - [Gitlab::JsonLogger]
# * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES`
- def with_lock_retries(**args, &block)
+ def with_lock_retries(*args, **kwargs, &block)
merged_args = {
klass: self.class,
logger: Gitlab::BackgroundMigration::Logger
- }.merge(args)
+ }.merge(kwargs)
- Gitlab::Database::WithLockRetries.new(merged_args).run(&block)
+ Gitlab::Database::WithLockRetries.new(**merged_args).run(&block)
end
def true_value
@@ -544,6 +544,16 @@ module Gitlab
rename_column_concurrently(table, column, temp_column, type: new_type, type_cast_function: type_cast_function, batch_column_name: batch_column_name)
end
+ # Reverses operations performed by change_column_type_concurrently.
+ #
+ # table - The table containing the column.
+ # column - The name of the column to change.
+ def undo_change_column_type_concurrently(table, column)
+ temp_column = "#{column}_for_type_change"
+
+ undo_rename_column_concurrently(table, column, temp_column)
+ end
+
# Performs cleanup of a concurrent type change.
#
# table - The table containing the column.
@@ -560,6 +570,65 @@ module Gitlab
end
end
+ # Reverses operations performed by cleanup_concurrent_column_type_change.
+ #
+ # table - The table containing the column.
+ # column - The name of the column to change.
+ # old_type - The type of the original column used with change_column_type_concurrently.
+ # type_cast_function - Required if the conversion back to the original type is not automatic
+ # batch_column_name - option for tables without a primary key, in this case
+ # another unique integer column can be used. Example: :user_id
+ def undo_cleanup_concurrent_column_type_change(table, column, old_type, type_cast_function: nil, batch_column_name: :id)
+ temp_column = "#{column}_for_type_change"
+
+ # Using a descriptive name that includes orinal column's name risks
+ # taking us above the 63 character limit, so we use a hash
+ identifier = "#{table}_#{column}_for_type_change"
+ hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10)
+ temp_undo_cleanup_column = "tmp_undo_cleanup_column_#{hashed_identifier}"
+
+ unless column_exists?(table, batch_column_name)
+ raise "Column #{batch_column_name} does not exist on #{table}"
+ end
+
+ if transaction_open?
+ raise 'undo_cleanup_concurrent_column_type_change can not be run inside a transaction'
+ end
+
+ check_trigger_permissions!(table)
+
+ begin
+ create_column_from(
+ table,
+ column,
+ temp_undo_cleanup_column,
+ type: old_type,
+ batch_column_name: batch_column_name,
+ type_cast_function: type_cast_function
+ )
+
+ transaction do
+ # This has to be performed in a transaction as otherwise we might
+ # have inconsistent data.
+ rename_column(table, column, temp_column)
+ rename_column(table, temp_undo_cleanup_column, column)
+
+ install_rename_triggers(table, column, temp_column)
+ end
+ rescue
+ # create_column_from can not run inside a transaction, which means
+ # that there is a risk that if any of the operations that follow it
+ # fail, we'll be left with an inconsistent schema
+ # For those reasons, we make sure that we drop temp_undo_cleanup_column
+ # if an error is caught
+ if column_exists?(table, temp_undo_cleanup_column)
+ remove_column(table, temp_undo_cleanup_column)
+ end
+
+ raise
+ end
+ end
+
# Cleans up a concurrent column name.
#
# This method takes care of removing previously installed triggers as well
@@ -882,7 +951,7 @@ module Gitlab
# column.
opclasses[new] = opclasses.delete(old) if opclasses[old]
- options[:opclasses] = opclasses
+ options[:opclass] = opclasses
end
add_concurrent_index(table, new_columns, options)
@@ -994,10 +1063,10 @@ into similar problems in the future (e.g. when new tables are created).
def postgres_exists_by_name?(table, name)
index_sql = <<~SQL
SELECT COUNT(*)
- FROM pg_index
- JOIN pg_class i ON (indexrelid=i.oid)
- JOIN pg_class t ON (indrelid=t.oid)
- WHERE i.relname = '#{name}' AND t.relname = '#{table}'
+ FROM pg_catalog.pg_indexes
+ WHERE schemaname = #{connection.quote(current_schema)}
+ AND tablename = #{connection.quote(table)}
+ AND indexname = #{connection.quote(name)}
SQL
connection.select_value(index_sql).to_i > 0
@@ -1053,11 +1122,15 @@ into similar problems in the future (e.g. when new tables are created).
# the table name in addition to using the constraint_name
check_sql = <<~SQL
SELECT COUNT(*)
- FROM pg_constraint
- JOIN pg_class ON pg_constraint.conrelid = pg_class.oid
- WHERE pg_constraint.contype = 'c'
- AND pg_constraint.conname = '#{constraint_name}'
- AND pg_class.relname = '#{table}'
+ FROM pg_catalog.pg_constraint con
+ INNER JOIN pg_catalog.pg_class rel
+ ON rel.oid = con.conrelid
+ INNER JOIN pg_catalog.pg_namespace nsp
+ ON nsp.oid = con.connamespace
+ WHERE con.contype = 'c'
+ AND con.conname = #{connection.quote(constraint_name)}
+ AND nsp.nspname = #{connection.quote(current_schema)}
+ AND rel.relname = #{connection.quote(table)}
SQL
connection.select_value(check_sql) > 0
@@ -1147,6 +1220,64 @@ into similar problems in the future (e.g. when new tables are created).
end
end
+ # Copies all check constraints for the old column to the new column.
+ #
+ # table - The table containing the columns.
+ # old - The old column.
+ # new - The new column.
+ # schema - The schema the table is defined for
+ # If it is not provided, then the current_schema is used
+ def copy_check_constraints(table, old, new, schema: nil)
+ if transaction_open?
+ raise 'copy_check_constraints can not be run inside a transaction'
+ end
+
+ unless column_exists?(table, old)
+ raise "Column #{old} does not exist on #{table}"
+ end
+
+ unless column_exists?(table, new)
+ raise "Column #{new} does not exist on #{table}"
+ end
+
+ table_with_schema = schema.present? ? "#{schema}.#{table}" : table
+
+ check_constraints_for(table, old, schema: schema).each do |check_c|
+ validate = !(check_c["constraint_def"].end_with? "NOT VALID")
+
+ # Normalize:
+ # - Old constraint definitions:
+ # '(char_length(entity_path) <= 5500)'
+ # - Definitionss from pg_get_constraintdef(oid):
+ # 'CHECK ((char_length(entity_path) <= 5500))'
+ # - Definitions from pg_get_constraintdef(oid, pretty_bool):
+ # 'CHECK (char_length(entity_path) <= 5500)'
+ # - Not valid constraints: 'CHECK (...) NOT VALID'
+ # to a single format that we can use:
+ # '(char_length(entity_path) <= 5500)'
+ check_definition = check_c["constraint_def"]
+ .sub(/^\s*(CHECK)?\s*\({0,2}/, '(')
+ .sub(/\){0,2}\s*(NOT VALID)?\s*$/, ')')
+
+ constraint_name = begin
+ if check_definition == "(#{old} IS NOT NULL)"
+ not_null_constraint_name(table_with_schema, new)
+ elsif check_definition.start_with? "(char_length(#{old}) <="
+ text_limit_name(table_with_schema, new)
+ else
+ check_constraint_name(table_with_schema, new, 'copy_check_constraint')
+ end
+ end
+
+ add_check_constraint(
+ table_with_schema,
+ check_definition.gsub(old.to_s, new.to_s),
+ constraint_name,
+ validate: validate
+ )
+ end
+ end
+
# Migration Helpers for adding limit to text columns
def add_text_limit(table, column, limit, constraint_name: nil, validate: true)
add_check_constraint(
@@ -1274,6 +1405,37 @@ into similar problems in the future (e.g. when new tables are created).
end
end
+ # Returns an ActiveRecord::Result containing the check constraints
+ # defined for the given column.
+ #
+ # If the schema is not provided, then the current_schema is used
+ def check_constraints_for(table, column, schema: nil)
+ check_sql = <<~SQL
+ SELECT
+ ccu.table_schema as schema_name,
+ ccu.table_name as table_name,
+ ccu.column_name as column_name,
+ con.conname as constraint_name,
+ pg_get_constraintdef(con.oid) as constraint_def
+ FROM pg_catalog.pg_constraint con
+ INNER JOIN pg_catalog.pg_class rel
+ ON rel.oid = con.conrelid
+ INNER JOIN pg_catalog.pg_namespace nsp
+ ON nsp.oid = con.connamespace
+ INNER JOIN information_schema.constraint_column_usage ccu
+ ON con.conname = ccu.constraint_name
+ AND nsp.nspname = ccu.constraint_schema
+ AND rel.relname = ccu.table_name
+ WHERE nsp.nspname = #{connection.quote(schema.presence || current_schema)}
+ AND rel.relname = #{connection.quote(table)}
+ AND ccu.column_name = #{connection.quote(column)}
+ AND con.contype = 'c'
+ ORDER BY constraint_name
+ SQL
+
+ connection.exec_query(check_sql)
+ end
+
def statement_timeout_disabled?
# This is a string of the form "100ms" or "0" when disabled
connection.select_value('SHOW statement_timeout') == "0"
@@ -1284,8 +1446,9 @@ into similar problems in the future (e.g. when new tables are created).
check_sql = <<~SQL
SELECT c.is_nullable
FROM information_schema.columns c
- WHERE c.table_name = '#{table}'
- AND c.column_name = '#{column}'
+ WHERE c.table_schema = #{connection.quote(current_schema)}
+ AND c.table_name = #{connection.quote(table)}
+ AND c.column_name = #{connection.quote(column)}
SQL
connection.select_value(check_sql) == 'YES'
@@ -1352,6 +1515,7 @@ into similar problems in the future (e.g. when new tables are created).
copy_indexes(table, old, new)
copy_foreign_keys(table, old, new)
+ copy_check_constraints(table, old, new)
end
def validate_timestamp_column_name!(column_name)
diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_creator.rb
index 4c1b13fe3b5..547e0b9b957 100644
--- a/lib/gitlab/database/partitioning/partition_creator.rb
+++ b/lib/gitlab/database/partitioning/partition_creator.rb
@@ -72,10 +72,10 @@ module Gitlab
end
def with_lock_retries(&block)
- Gitlab::Database::WithLockRetries.new({
+ Gitlab::Database::WithLockRetries.new(
klass: self.class,
logger: Gitlab::AppLogger
- }).run(&block)
+ ).run(&block)
end
def connection
diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb
index f9ad1e60776..17a42d997e6 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb
@@ -11,8 +11,6 @@ module Gitlab
PAUSE_SECONDS = 0.25
def perform(start_id, stop_id, source_table, partitioned_table, source_column)
- return unless Feature.enabled?(:backfill_partitioned_audit_events, default_enabled: true)
-
if transaction_open?
raise "Aborting job to backfill partitioned #{source_table} table! Do not run this job in a transaction block!"
end
diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb
new file mode 100644
index 00000000000..2a9f23f0098
--- /dev/null
+++ b/lib/gitlab/database/postgres_index.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class PostgresIndex < ActiveRecord::Base
+ self.table_name = 'postgres_indexes'
+ self.primary_key = 'identifier'
+
+ scope :by_identifier, ->(identifier) do
+ raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/
+
+ find(identifier)
+ end
+
+ # A 'regular' index is a non-unique index,
+ # that does not serve an exclusion constraint and
+ # is defined on a table that is not partitioned.
+ scope :regular, -> { where(unique: false, partitioned: false, exclusion: false)}
+
+ scope :random_few, ->(how_many) do
+ limit(how_many).order(Arel.sql('RANDOM()'))
+ end
+
+ scope :not_match, ->(regex) { where("name !~ ?", regex)}
+
+ def to_s
+ name
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb
new file mode 100644
index 00000000000..074752fe75b
--- /dev/null
+++ b/lib/gitlab/database/reindexing.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Reindexing
+ def self.perform(index_selector)
+ Coordinator.new(index_selector).perform
+ end
+
+ def self.candidate_indexes
+ Gitlab::Database::PostgresIndex
+ .regular
+ .not_match("^#{ConcurrentReindex::TEMPORARY_INDEX_PREFIX}")
+ .not_match("^#{ConcurrentReindex::REPLACED_INDEX_PREFIX}")
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/reindexing/concurrent_reindex.rb b/lib/gitlab/database/reindexing/concurrent_reindex.rb
new file mode 100644
index 00000000000..fd3dca88567
--- /dev/null
+++ b/lib/gitlab/database/reindexing/concurrent_reindex.rb
@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Reindexing
+ class ConcurrentReindex
+ include Gitlab::Utils::StrongMemoize
+
+ ReindexError = Class.new(StandardError)
+
+ PG_IDENTIFIER_LENGTH = 63
+ TEMPORARY_INDEX_PREFIX = 'tmp_reindex_'
+ REPLACED_INDEX_PREFIX = 'old_reindex_'
+ STATEMENT_TIMEOUT = 6.hours
+
+ attr_reader :index, :logger
+
+ def initialize(index, logger: Gitlab::AppLogger)
+ @index = index
+ @logger = logger
+ end
+
+ def perform
+ raise ReindexError, 'UNIQUE indexes are currently not supported' if index.unique?
+ raise ReindexError, 'partitioned indexes are currently not supported' if index.partitioned?
+ raise ReindexError, 'indexes serving an exclusion constraint are currently not supported' if index.exclusion?
+ raise ReindexError, 'index is a left-over temporary index from a previous reindexing run' if index.name.start_with?(TEMPORARY_INDEX_PREFIX, REPLACED_INDEX_PREFIX)
+
+ logger.info "Starting reindex of #{index}"
+
+ with_rebuilt_index do |replacement_index|
+ swap_index(replacement_index)
+ end
+ end
+
+ private
+
+ def with_rebuilt_index
+ if Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name)
+ logger.debug("dropping dangling index from previous run (if it exists): #{replacement_index_name}")
+ remove_index(index.schema, replacement_index_name)
+ end
+
+ create_replacement_index_statement = index.definition
+ .sub(/CREATE INDEX #{index.name}/, "CREATE INDEX CONCURRENTLY #{replacement_index_name}")
+
+ logger.info("creating replacement index #{replacement_index_name}")
+ logger.debug("replacement index definition: #{create_replacement_index_statement}")
+
+ set_statement_timeout do
+ connection.execute(create_replacement_index_statement)
+ end
+
+ replacement_index = Gitlab::Database::PostgresIndex.find_by(schema: index.schema, name: replacement_index_name)
+
+ unless replacement_index.valid_index?
+ message = 'replacement index was created as INVALID'
+ logger.error("#{message}, cleaning up")
+ raise ReindexError, "failed to reindex #{index}: #{message}"
+ end
+
+ yield replacement_index
+ ensure
+ begin
+ remove_index(index.schema, replacement_index_name)
+ rescue => e
+ logger.error(e)
+ end
+ end
+
+ def swap_index(replacement_index)
+ logger.info("swapping replacement index #{replacement_index} with #{index}")
+
+ with_lock_retries do
+ rename_index(index.schema, index.name, replaced_index_name)
+ rename_index(replacement_index.schema, replacement_index.name, index.name)
+ rename_index(index.schema, replaced_index_name, replacement_index.name)
+ end
+ end
+
+ def rename_index(schema, old_index_name, new_index_name)
+ connection.execute(<<~SQL)
+ ALTER INDEX #{quote_table_name(schema)}.#{quote_table_name(old_index_name)}
+ RENAME TO #{quote_table_name(new_index_name)}
+ SQL
+ end
+
+ def remove_index(schema, name)
+ logger.info("Removing index #{schema}.#{name}")
+
+ set_statement_timeout do
+ connection.execute(<<~SQL)
+ DROP INDEX CONCURRENTLY
+ IF EXISTS #{quote_table_name(schema)}.#{quote_table_name(name)}
+ SQL
+ end
+ end
+
+ def replacement_index_name
+ @replacement_index_name ||= "#{TEMPORARY_INDEX_PREFIX}#{index.indexrelid}"
+ end
+
+ def replaced_index_name
+ @replaced_index_name ||= "#{REPLACED_INDEX_PREFIX}#{index.indexrelid}"
+ end
+
+ def with_lock_retries(&block)
+ arguments = { klass: self.class, logger: logger }
+
+ Gitlab::Database::WithLockRetries.new(**arguments).run(raise_on_exhaustion: true, &block)
+ end
+
+ def set_statement_timeout
+ execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT)
+ yield
+ ensure
+ execute('RESET statement_timeout')
+ end
+
+ delegate :execute, :quote_table_name, to: :connection
+ def connection
+ @connection ||= ActiveRecord::Base.connection
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb
new file mode 100644
index 00000000000..0957f43e166
--- /dev/null
+++ b/lib/gitlab/database/reindexing/coordinator.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Reindexing
+ class Coordinator
+ include ExclusiveLeaseGuard
+
+ # Maximum lease time for the global Redis lease
+ # This should be higher than the maximum time for any
+ # long running step in the reindexing process (compare with
+ # statement timeouts).
+ TIMEOUT_PER_ACTION = 1.day
+
+ attr_reader :indexes
+
+ def initialize(indexes)
+ @indexes = indexes
+ end
+
+ def perform
+ indexes.each do |index|
+ # This obtains a global lease such that there's
+ # only one live reindexing process at a time.
+ try_obtain_lease do
+ ReindexAction.keep_track_of(index) do
+ ConcurrentReindex.new(index).perform
+ end
+ end
+ end
+ end
+
+ private
+
+ def lease_timeout
+ TIMEOUT_PER_ACTION
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb
new file mode 100644
index 00000000000..0928ef90e5d
--- /dev/null
+++ b/lib/gitlab/database/reindexing/reindex_action.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module Reindexing
+ class ReindexAction < ActiveRecord::Base
+ self.table_name = 'postgres_reindex_actions'
+
+ enum state: { started: 0, finished: 1, failed: 2 }
+
+ def self.keep_track_of(index, &block)
+ action = create!(
+ index_identifier: index.identifier,
+ action_start: Time.zone.now,
+ ondisk_size_bytes_start: index.ondisk_size_bytes
+ )
+
+ yield
+
+ action.state = :finished
+ rescue
+ action.state = :failed
+ raise
+ ensure
+ index.reload # rubocop:disable Cop/ActiveRecordAssociationReload
+
+ action.action_end = Time.zone.now
+ action.ondisk_size_bytes_end = index.ondisk_size_bytes
+
+ action.save!
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb
index dda4d8eecdb..3d929c62933 100644
--- a/lib/gitlab/database/schema_helpers.rb
+++ b/lib/gitlab/database/schema_helpers.rb
@@ -32,11 +32,14 @@ module Gitlab
def trigger_exists?(table_name, name)
connection.select_value(<<~SQL)
SELECT 1
- FROM pg_trigger
- INNER JOIN pg_class
- ON pg_trigger.tgrelid = pg_class.oid
- WHERE pg_class.relname = '#{table_name}'
- AND pg_trigger.tgname = '#{name}'
+ FROM pg_catalog.pg_trigger trgr
+ INNER JOIN pg_catalog.pg_class rel
+ ON trgr.tgrelid = rel.oid
+ INNER JOIN pg_catalog.pg_namespace nsp
+ ON nsp.oid = rel.relnamespace
+ WHERE nsp.nspname = #{connection.quote(current_schema)}
+ AND rel.relname = #{connection.quote(table_name)}
+ AND trgr.tgname = #{connection.quote(name)}
SQL
end
@@ -68,10 +71,10 @@ module Gitlab
end
def with_lock_retries(&block)
- Gitlab::Database::WithLockRetries.new({
+ Gitlab::Database::WithLockRetries.new(
klass: self.class,
logger: Gitlab::BackgroundMigration::Logger
- }).run(&block)
+ ).run(&block)
end
def assert_not_in_transaction_block(scope:)
diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb
index 2633c29438a..ff78fd0218c 100644
--- a/lib/gitlab/database/similarity_score.rb
+++ b/lib/gitlab/database/similarity_score.rb
@@ -6,6 +6,11 @@ module Gitlab
EMPTY_STRING = Arel.sql("''").freeze
EXPRESSION_ON_INVALID_INPUT = Arel::Nodes::NamedFunction.new('CAST', [Arel.sql('0').as('integer')]).freeze
DEFAULT_MULTIPLIER = 1
+ DISPLAY_NAME = self.name.underscore.freeze
+
+ # Adds a "magic" comment in the generated SQL expression in order to be able to tell if we're sorting by similarity.
+ # Example: /* gitlab/database/similarity_score */ SIMILARITY(COALESCE...
+ SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY".freeze
# This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity.
#
@@ -74,6 +79,10 @@ module Gitlab
end
end
+ def self.order_by_similarity?(arel_query)
+ arel_query.to_sql.include?(SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION)
+ end
+
# (SIMILARITY(COALESCE(column, ''), 'search_string') * CAST(multiplier AS numeric))
def self.rule_to_arel(search, rule)
Arel::Nodes::Grouping.new(
@@ -91,7 +100,7 @@ module Gitlab
# SIMILARITY(COALESCE(column, ''), 'search_string')
def self.similarity_function_call(search, column)
- Arel::Nodes::NamedFunction.new('SIMILARITY', [column, Arel.sql(search)])
+ Arel::Nodes::NamedFunction.new(SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION, [column, Arel.sql(search)])
end
# CAST(multiplier AS numeric)
diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb
index a9c86e4e267..3fb52d786ad 100644
--- a/lib/gitlab/database/with_lock_retries.rb
+++ b/lib/gitlab/database/with_lock_retries.rb
@@ -95,7 +95,7 @@ module Gitlab
run_block_with_transaction
rescue ActiveRecord::LockWaitTimeout
if retry_with_lock_timeout?
- disable_idle_in_transaction_timeout
+ disable_idle_in_transaction_timeout if ActiveRecord::Base.connection.transaction_open?
wait_until_next_retry
reset_db_settings
@@ -149,7 +149,7 @@ module Gitlab
log(message: "Couldn't acquire lock to perform the migration", current_iteration: current_iteration)
log(message: "Executing the migration without lock timeout", current_iteration: current_iteration)
- execute("SET LOCAL lock_timeout TO '0'")
+ disable_lock_timeout if ActiveRecord::Base.connection.transaction_open?
run_block
@@ -184,6 +184,10 @@ module Gitlab
execute("SET LOCAL idle_in_transaction_session_timeout TO '0'")
end
+ def disable_lock_timeout
+ execute("SET LOCAL lock_timeout TO '0'")
+ end
+
def reset_db_settings
execute('RESET idle_in_transaction_session_timeout; RESET lock_timeout')
end
diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml
new file mode 100644
index 00000000000..1d341e6520e
--- /dev/null
+++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml
@@ -0,0 +1,43 @@
+# This file exists to lock the attributes of Design Management models
+# that get copied in `DesignManagement::CopyDesignCollection::CopyService`
+# to specific schemas.
+#
+# This allows us to perform sanity checks and alert when there are changes
+# to the schema by running expectations against the lists in this file
+# and the actual schema of the models in `copy_designs_service_spec.rb`.
+#
+# If you are here because you received a failed test in
+# `copy_designs_service_spec.rb`, you need to decide how to handle the
+# changes and whether the new attribute(s) should be included in the copy
+# or ignored.
+
+# COPY.
+# Add attributes that should be copied to the `{model}_attributes` lists:
+design_attributes:
+ - filename
+ - relative_position
+
+version_attributes:
+ - author_id
+ - created_at
+
+action_attributes: # (None)
+
+# IGNORE.
+# Add attributes that should not be copied to the `ignore_{model}_attributes` lists:
+ignore_design_attributes:
+ - id
+ - issue_id
+ - project_id
+
+ignore_version_attributes:
+ - id
+ - issue_id
+ - sha
+
+ignore_action_attributes:
+ - id
+ - design_id
+ - event
+ - image_v432x230
+ - version_id
diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
index d54e1aad19a..16257bb5ff5 100644
--- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
+++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb
@@ -11,7 +11,7 @@ module Gitlab
super(merge_request_diff,
project: merge_request_diff.project,
- diff_options: diff_options,
+ diff_options: merged_diff_options(diff_options),
diff_refs: merge_request_diff.diff_refs,
fallback_diff_refs: merge_request_diff.fallback_diff_refs)
end
@@ -64,6 +64,13 @@ module Gitlab
diff_stats_cache.read || super
end
end
+
+ def merged_diff_options(diff_options)
+ project = @merge_request_diff.project
+ max_diff_options = ::Commit.max_diff_options(project: project).merge(project: project)
+
+ diff_options.present? ? diff_options.merge(max_diff_options) : max_diff_options
+ end
end
end
end
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index 0eb22e6b3cb..90cb9c8638a 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -20,10 +20,27 @@ module Gitlab
# - Assigns DiffFile#highlighted_diff_lines for cached files
#
def decorate(diff_file)
- if content = read_file(diff_file)
- diff_file.highlighted_diff_lines = content.map do |line|
- Gitlab::Diff::Line.safe_init_from_hash(line)
- end
+ content = read_file(diff_file)
+
+ return [] unless content
+
+ # TODO: We could add some kind of flag to #initialize that would allow
+ # us to force re-caching
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/263508
+ #
+ if content.empty? && recache_due_to_size?(diff_file)
+ # If the file is missing from the cache and there's reason to believe
+ # it is uncached due to a size issue around changing the values for
+ # max patch size, manually populate the hash and then set the value.
+ #
+ new_cache_content = {}
+ new_cache_content[diff_file.file_path] = diff_file.highlighted_diff_lines.map(&:to_hash)
+
+ write_to_redis_hash(new_cache_content)
+
+ set_highlighted_diff_lines(diff_file, read_file(diff_file))
+ else
+ set_highlighted_diff_lines(diff_file, content)
end
end
@@ -58,6 +75,28 @@ module Gitlab
private
+ def set_highlighted_diff_lines(diff_file, content)
+ diff_file.highlighted_diff_lines = content.map do |line|
+ Gitlab::Diff::Line.safe_init_from_hash(line)
+ end
+ end
+
+ def recache_due_to_size?(diff_file)
+ diff_file_class = diff_file.diff.class
+
+ current_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes
+ default_patch_safe_limit_bytes = diff_file_class.patch_safe_limit_bytes(diff_file_class::DEFAULT_MAX_PATCH_BYTES)
+
+ # If the diff is >= than the default limit, but less than the current
+ # limit, it is likely uncached due to having hit the default limit,
+ # making it eligible for recalculating.
+ #
+ diff_file.diff.diff_bytesize.between?(
+ default_patch_safe_limit_bytes,
+ current_patch_safe_limit_bytes
+ )
+ end
+
def cacheable_files
strong_memoize(:cacheable_files) do
diff_files.select { |file| cacheable?(file) && read_file(file).nil? }
diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb
index 10762d83588..da5b0afad38 100644
--- a/lib/gitlab/exclusive_lease_helpers.rb
+++ b/lib/gitlab/exclusive_lease_helpers.rb
@@ -35,7 +35,7 @@ module Gitlab
lease.obtain(1 + retries)
- yield(lease.retried?)
+ yield(lease.retried?, lease)
ensure
lease&.cancel
end
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
index dca60c93fb2..1ce3ffe4c86 100644
--- a/lib/gitlab/experimentation.rb
+++ b/lib/gitlab/experimentation.rb
@@ -48,12 +48,12 @@ module Gitlab
invite_members_version_a: {
tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA'
},
+ invite_members_version_b: {
+ tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB'
+ },
new_create_project_ui: {
tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi'
},
- terms_opt_in: {
- tracking_category: 'Growth::Acquisition::Experiment::TermsOptIn'
- },
contact_sales_btn_in_app: {
tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp'
},
@@ -62,6 +62,15 @@ module Gitlab
},
invite_email: {
tracking_category: 'Growth::Acquisition::Experiment::InviteEmail'
+ },
+ invitation_reminders: {
+ tracking_category: 'Growth::Acquisition::Experiment::InvitationReminders'
+ },
+ group_only_trials: {
+ tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials'
+ },
+ default_to_issues_board: {
+ tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard'
}
}.freeze
@@ -91,28 +100,40 @@ module Gitlab
}
end
+ def push_frontend_experiment(experiment_key)
+ var_name = experiment_key.to_s.camelize(:lower)
+ enabled = experiment_enabled?(experiment_key)
+
+ gon.push({ experiments: { var_name => enabled } }, true)
+ end
+
def experiment_enabled?(experiment_key)
return false if dnt_enabled?
- return true if Experimentation.enabled_for_user?(experiment_key, experimentation_subject_index)
+ return true if Experimentation.enabled_for_value?(experiment_key, experimentation_subject_index)
return true if forced_enabled?(experiment_key)
false
end
def track_experiment_event(experiment_key, action, value = nil)
+ return if dnt_enabled?
+
track_experiment_event_for(experiment_key, action, value) do |tracking_data|
- ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), tracking_data)
+ ::Gitlab::Tracking.event(tracking_data.delete(:category), tracking_data.delete(:action), **tracking_data)
end
end
def frontend_experimentation_tracking_data(experiment_key, action, value = nil)
+ return if dnt_enabled?
+
track_experiment_event_for(experiment_key, action, value) do |tracking_data|
gon.push(tracking_data: tracking_data)
end
end
def record_experiment_user(experiment_key)
+ return if dnt_enabled?
return unless Experimentation.enabled?(experiment_key) && current_user
::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user)
@@ -183,9 +204,14 @@ module Gitlab
experiment.enabled? && experiment.enabled_for_environment?
end
- def enabled_for_user?(experiment_key, experimentation_subject_index)
+ def enabled_for_attribute?(experiment_key, attribute)
+ index = Digest::SHA1.hexdigest(attribute).hex % 100
+ enabled_for_value?(experiment_key, index)
+ end
+
+ def enabled_for_value?(experiment_key, experimentation_subject_index)
enabled?(experiment_key) &&
- experiment(experiment_key).enabled_for_experimentation_subject?(experimentation_subject_index)
+ experiment(experiment_key).enabled_for_index?(experimentation_subject_index)
end
end
@@ -200,10 +226,10 @@ module Gitlab
environment
end
- def enabled_for_experimentation_subject?(experimentation_subject_index)
- return false if experimentation_subject_index.blank?
+ def enabled_for_index?(index)
+ return false if index.blank?
- experimentation_subject_index <= experiment_percentage
+ index <= experiment_percentage
end
private
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index 09a49b6c1ca..78c47023c08 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -120,8 +120,8 @@ module Gitlab
# default.
#
# Patches surpassing this limit should still be persisted in the database.
- def patch_safe_limit_bytes
- patch_hard_limit_bytes / 10
+ def patch_safe_limit_bytes(limit = patch_hard_limit_bytes)
+ limit / 10
end
# Returns the limit for a single diff file (patch).
@@ -174,9 +174,13 @@ module Gitlab
@line_count ||= Util.count_lines(@diff)
end
+ def diff_bytesize
+ @diff_bytesize ||= @diff.bytesize
+ end
+
def too_large?
if @too_large.nil?
- @too_large = @diff.bytesize >= self.class.patch_hard_limit_bytes
+ @too_large = diff_bytesize >= self.class.patch_hard_limit_bytes
else
@too_large
end
@@ -194,7 +198,7 @@ module Gitlab
def collapsed?
return @collapsed if defined?(@collapsed)
- @collapsed = !expanded && @diff.bytesize >= self.class.patch_safe_limit_bytes
+ @collapsed = !expanded && diff_bytesize >= self.class.patch_safe_limit_bytes
end
def collapse!
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index e6121d688ba..6090d1b9f69 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -7,19 +7,27 @@ module Gitlab
class DiffCollection
include Enumerable
- DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze
-
attr_reader :limits
delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits
+ def self.default_limits(project: nil)
+ if Feature.enabled?(:increased_diff_limits, project)
+ { max_files: 200, max_lines: 7500 }
+ else
+ { max_files: 100, max_lines: 5000 }
+ end
+ end
+
def self.limits(options = {})
limits = {}
- limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
- limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
+ defaults = default_limits(project: options[:project])
+ limits[:max_files] = options.fetch(:max_files, defaults[:max_files])
+ limits[:max_lines] = options.fetch(:max_lines, defaults[:max_lines])
limits[:max_bytes] = limits[:max_files] * 5.kilobytes # Average 5 KB per file
- limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min
- limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min
+
+ limits[:safe_max_files] = [limits[:max_files], defaults[:max_files]].min
+ limits[:safe_max_lines] = [limits[:max_lines], defaults[:max_lines]].min
limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file
limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes
@@ -110,11 +118,17 @@ module Gitlab
files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes
end
+ def expand_diff?
+ # Force single-entry diff collections to always present as expanded
+ #
+ @iterator.size == 1 || !@enforce_limits || @expanded
+ end
+
def each_gitaly_patch
i = @array.length
@iterator.each do |raw|
- diff = Gitlab::Git::Diff.new(raw, expanded: !@enforce_limits || @expanded)
+ diff = Gitlab::Git::Diff.new(raw, expanded: expand_diff?)
if raw.overflow_marker
@overflow = true
@@ -137,11 +151,9 @@ module Gitlab
break
end
- expanded = !@enforce_limits || @expanded
-
- diff = Gitlab::Git::Diff.new(raw, expanded: expanded)
+ diff = Gitlab::Git::Diff.new(raw, expanded: expand_diff?)
- if !expanded && over_safe_limits?(i) && diff.line_count > 0
+ if !expand_diff? && over_safe_limits?(i) && diff.line_count > 0
diff.collapse!
end
diff --git a/lib/gitlab/git/diff_stats_collection.rb b/lib/gitlab/git/diff_stats_collection.rb
index 7e49d79676e..e30ec836a49 100644
--- a/lib/gitlab/git/diff_stats_collection.rb
+++ b/lib/gitlab/git/diff_stats_collection.rb
@@ -22,8 +22,8 @@ module Gitlab
@collection.map(&:path)
end
- def real_size
- max_files = ::Commit.max_diff_options[:max_files]
+ def real_size(project: nil)
+ max_files = ::Commit.max_diff_options(project: project)[:max_files]
if paths.size > max_files
"#{max_files}+"
else
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 8ace4157ad7..1a3409c1f84 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -297,10 +297,17 @@ module Gitlab
end
file_name = "#{name}.#{extension}"
- File.join(storage_path, self.gl_repository, sha, file_name)
+ File.join(storage_path, self.gl_repository, sha, archive_version_path, file_name)
end
private :archive_file_path
+ def archive_version_path
+ return '' unless Feature.enabled?(:include_lfs_blobs_in_archive)
+
+ '@v2'
+ end
+ private :archive_version_path
+
# Return repo size in megabytes
def size
size = gitaly_repository_client.repository_size
@@ -580,9 +587,9 @@ module Gitlab
tags.find { |tag| tag.name == name }
end
- def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref)
+ def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts)
wrapped_gitaly_errors do
- gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref)
+ gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts)
end
end
@@ -610,7 +617,7 @@ module Gitlab
}
wrapped_gitaly_errors do
- gitaly_operation_client.user_revert(args)
+ gitaly_operation_client.user_revert(**args)
end
end
@@ -626,7 +633,7 @@ module Gitlab
}
wrapped_gitaly_errors do
- gitaly_operation_client.user_cherry_pick(args)
+ gitaly_operation_client.user_cherry_pick(**args)
end
end
@@ -640,7 +647,7 @@ module Gitlab
}
wrapped_gitaly_errors do
- gitaly_operation_client.user_update_submodule(args)
+ gitaly_operation_client.user_update_submodule(**args)
end
end
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index da2d015ca4a..11919be594d 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -100,10 +100,6 @@ module Gitlab
wrapped_gitaly_errors do
gitaly_find_page(title: title, version: version, dir: dir)
end
- rescue Gitlab::Git::CommandError
- # Return nil for invalid versions.
- # This can be removed with https://gitlab.com/gitlab-org/gitaly/-/merge_requests/2323 in place.
- nil
end
def file(name, version)
@@ -159,6 +155,8 @@ module Gitlab
return unless wiki_page
Gitlab::Git::WikiPage.new(wiki_page, version)
+ rescue GRPC::InvalidArgument
+ nil
end
def gitaly_find_file(name, version)
@@ -173,9 +171,9 @@ module Gitlab
gitaly_pages =
if load_content
- gitaly_wiki_client.load_all_pages(params)
+ gitaly_wiki_client.load_all_pages(**params)
else
- gitaly_wiki_client.list_all_pages(params)
+ gitaly_wiki_client.list_all_pages(**params)
end
gitaly_pages.map do |wiki_page, version|
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index b67b3a37440..0576d1dd9db 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -47,6 +47,16 @@ module Gitlab
:cmd, :changes
attr_accessor :container
+ def self.error_message(key)
+ self.ancestors.each do |cls|
+ return cls.const_get('ERROR_MESSAGES', false).fetch(key)
+ rescue NameError, KeyError
+ next
+ end
+
+ raise ArgumentError, "No error message defined for #{key}"
+ end
+
def initialize(actor, container, protocol, authentication_abilities:, namespace_path: nil, repository_path: nil, redirected_path: nil, auth_result_type: nil)
@actor = actor
@container = container
@@ -137,6 +147,10 @@ module Gitlab
private
def check_container!
+ # Strict nil check, to avoid any surprises with Object#present?
+ # which can delegate to #empty?
+ raise NotFoundError, not_found_message if container.nil?
+
check_project! if project?
end
@@ -204,9 +218,7 @@ module Gitlab
end
def check_project_accessibility!
- if project.blank? || !can_read_project?
- raise NotFoundError, not_found_message
- end
+ raise NotFoundError, not_found_message unless can_read_project?
end
def not_found_message
@@ -279,10 +291,10 @@ module Gitlab
error_message(:download)
end
- # We assume that all git-access classes are in project context by default.
- # Override this method to be more specific.
def project?
- true
+ # Strict nil check, to avoid any surprises with Object#present?
+ # which can delegate to #empty?
+ !project.nil?
end
def project
@@ -290,7 +302,7 @@ module Gitlab
end
def check_push_access!
- if container.repository_read_only?
+ if project&.repository_read_only?
raise ForbiddenError, error_message(:read_only)
end
@@ -411,13 +423,7 @@ module Gitlab
protected
def error_message(key)
- self.class.ancestors.each do |cls|
- return cls.const_get('ERROR_MESSAGES', false).fetch(key)
- rescue NameError, KeyError
- next
- end
-
- raise ArgumentError, "No error message defined for #{key}"
+ self.class.error_message(key)
end
def success_result
@@ -504,7 +510,7 @@ module Gitlab
changes_size = 0
changes_list.each do |change|
- changes_size += repository.new_blobs(change[:newrev]).sum(&:size) # rubocop: disable CodeReuse/ActiveRecord
+ changes_size += repository.new_blobs(change[:newrev]).sum(&:size)
check_size_against_limit(changes_size)
end
diff --git a/lib/gitlab/git_access_result/custom_action.rb b/lib/gitlab/git_access_result/custom_action.rb
index e03459ea7a1..224c6b9eb52 100644
--- a/lib/gitlab/git_access_result/custom_action.rb
+++ b/lib/gitlab/git_access_result/custom_action.rb
@@ -12,7 +12,7 @@ module Gitlab
# 'data' => {
# 'api_endpoints' => %w{geo/proxy_git_ssh/info_refs_receive_pack geo/proxy_git_ssh/receive_pack},
# 'gl_username' => user.username,
- # 'primary_repo' => geo_primary_http_url_to_repo(project_or_wiki)
+ # 'primary_repo' => geo_primary_http_url_to_repo(container)
# }
# }
#
diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb
index ae83e45f2b3..710e2ce90ec 100644
--- a/lib/gitlab/git_access_snippet.rb
+++ b/lib/gitlab/git_access_snippet.rb
@@ -21,6 +21,11 @@ module Gitlab
@authentication_abilities &= [:download_code, :push_code]
end
+ override :project
+ def project
+ container.project if container.is_a?(ProjectSnippet)
+ end
+
override :check
def check(cmd, changes)
check_snippet_accessibility!
@@ -46,29 +51,19 @@ module Gitlab
# snippets never return custom actions, such as geo replication.
end
- override :project?
- def project?
- project_snippet?
- end
-
- override :project
- def project
- snippet&.project
- end
-
override :check_valid_actor!
def check_valid_actor!
# TODO: Investigate if expanding actor/authentication types are needed.
# https://gitlab.com/gitlab-org/gitlab/issues/202190
- if actor && !actor.is_a?(User) && !actor.instance_of?(Key)
+ if actor && !allowed_actor?
raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism]
end
super
end
- def project_snippet?
- snippet.is_a?(ProjectSnippet)
+ def allowed_actor?
+ actor.is_a?(User) || actor.instance_of?(Key)
end
override :check_push_access!
@@ -120,7 +115,7 @@ module Gitlab
override :check_single_change_access
def check_single_change_access(change, _skip_lfs_integrity_check: false)
Checks::SnippetCheck.new(change, default_branch: snippet.default_branch, logger: logger).validate!
- Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit(user), logger: logger).validate!
+ Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit, logger: logger).validate!
rescue Checks::TimedLogger::TimeoutError
raise TimeoutError, logger.full_message
end
@@ -138,3 +133,5 @@ module Gitlab
end
end
end
+
+Gitlab::GitAccessSnippet.prepend_if_ee('EE::Gitlab::GitAccessSnippet')
diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb
index a941282e713..3011b794b8f 100644
--- a/lib/gitlab/git_access_wiki.rb
+++ b/lib/gitlab/git_access_wiki.rb
@@ -12,6 +12,11 @@ module Gitlab
write_to_wiki: "You are not allowed to write to this project's wiki."
}.freeze
+ override :project
+ def project
+ container.project if container.is_a?(ProjectWiki)
+ end
+
override :download_ability
def download_ability
:download_wiki_code
@@ -40,11 +45,6 @@ module Gitlab
def not_found_message
error_message(:not_found)
end
-
- override :repository
- def repository
- container.wiki.repository
- end
end
end
diff --git a/lib/gitlab/gitaly_client/diff_stitcher.rb b/lib/gitlab/gitaly_client/diff_stitcher.rb
index 98d327a7329..e98ae75590d 100644
--- a/lib/gitlab/gitaly_client/diff_stitcher.rb
+++ b/lib/gitlab/gitaly_client/diff_stitcher.rb
@@ -5,8 +5,10 @@ module Gitlab
class DiffStitcher
include Enumerable
- def initialize(rpc_response)
- @rpc_response = rpc_response
+ delegate :size, to: :rpc_response
+
+ def initialize(rpc_response_param)
+ @rpc_response = rpc_response_param
end
def each
@@ -31,6 +33,10 @@ module Gitlab
end
end
end
+
+ private
+
+ attr_reader :rpc_response
end
end
end
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 786eb3ca4ae..4850d646de4 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -102,7 +102,7 @@ module Gitlab
end
end
- def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref)
+ def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts)
request = Gitaly::UserMergeToRefRequest.new(
repository: @gitaly_repo,
source_sha: source_sha,
@@ -110,7 +110,8 @@ module Gitlab
target_ref: encode_binary(target_ref),
user: Gitlab::Git::User.from_gitlab(user).to_gitaly,
message: encode_binary(message),
- first_parent_ref: encode_binary(first_parent_ref)
+ first_parent_ref: encode_binary(first_parent_ref),
+ allow_conflicts: allow_conflicts
)
response = GitalyClient.call(@repository.storage, :operation_service,
diff --git a/lib/gitlab/gitpod.rb b/lib/gitlab/gitpod.rb
index 11b54db72ea..e35fb8fed02 100644
--- a/lib/gitlab/gitpod.rb
+++ b/lib/gitlab/gitpod.rb
@@ -3,17 +3,13 @@
module Gitlab
class Gitpod
class << self
- def feature_conditional?
- feature.conditional?
- end
-
def feature_available?
# The gitpod_bundle feature could be conditionally applied, so check if `!off?`
- !feature.off?
+ !feature.off? || feature_enabled?
end
def feature_enabled?(actor = nil)
- feature.enabled?(actor)
+ Feature.enabled?(:gitpod, actor, default_enabled: true)
end
def feature_and_settings_enabled?(actor = nil)
diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb
index 54dca93a891..352a93817be 100644
--- a/lib/gitlab/gl_repository.rb
+++ b/lib/gitlab/gl_repository.rb
@@ -4,6 +4,8 @@ module Gitlab
class GlRepository
include Singleton
+ # TODO: Refactor these constants into proper classes
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/259008
PROJECT = RepoType.new(
name: :project,
access_checker_class: Gitlab::GitAccessProject,
@@ -12,8 +14,12 @@ module Gitlab
WIKI = RepoType.new(
name: :wiki,
access_checker_class: Gitlab::GitAccessWiki,
- repository_resolver: -> (container) { ::Repository.new(container.wiki.full_path, container, shard: container.wiki.repository_storage, disk_path: container.wiki.disk_path, repo_type: WIKI) },
- project_resolver: -> (container) { container.is_a?(Project) ? container : nil },
+ repository_resolver: -> (container) do
+ wiki = container.is_a?(Wiki) ? container : container.wiki # Also allow passing a Project, Group, or Geo::DeletedProject
+ ::Repository.new(wiki.full_path, wiki, shard: wiki.repository_storage, disk_path: wiki.disk_path, repo_type: WIKI)
+ end,
+ container_class: ProjectWiki,
+ project_resolver: -> (wiki) { wiki.try(:project) },
suffix: :wiki
).freeze
SNIPPET = RepoType.new(
diff --git a/lib/gitlab/gl_repository/identifier.rb b/lib/gitlab/gl_repository/identifier.rb
index 57350b1edb0..f521a14ea19 100644
--- a/lib/gitlab/gl_repository/identifier.rb
+++ b/lib/gitlab/gl_repository/identifier.rb
@@ -53,12 +53,13 @@ module Gitlab
private
def container_class
- case @container_type
- when 'project'
- Project
- when 'group'
- Group
- end
+ # NOTE: This is currently only used and supported for group wikis
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/219192
+ return unless @repo_type_name == 'wiki'
+
+ "#{@container_type}_#{@repo_type_name}".classify.constantize
+ rescue NameError
+ nil
end
end
diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb
index 346f6be0d98..4b1f4fcc2a2 100644
--- a/lib/gitlab/gl_repository/repo_type.rb
+++ b/lib/gitlab/gl_repository/repo_type.rb
@@ -29,10 +29,6 @@ module Gitlab
end
def identifier_for_container(container)
- if container.is_a?(Group)
- return "#{container.class.name.underscore}-#{container.id}-#{name}"
- end
-
"#{name}-#{container.id}"
end
@@ -84,3 +80,5 @@ module Gitlab
end
end
end
+
+Gitlab::GlRepository::RepoType.prepend_if_ee('EE::Gitlab::GlRepository::RepoType')
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 66517ecd743..10660649623 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -43,12 +43,10 @@ module Gitlab
# Initialize gon.features with any flags that should be
# made globally available to the frontend
- push_frontend_feature_flag(:snippets_vue, default_enabled: true)
- push_frontend_feature_flag(:monaco_blobs, default_enabled: true)
- push_frontend_feature_flag(:monaco_ci, default_enabled: false)
- push_frontend_feature_flag(:snippets_edit_vue, default_enabled: true)
push_frontend_feature_flag(:webperf_experiment, default_enabled: false)
push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false)
+ push_frontend_feature_flag(:usage_data_api, default_enabled: true)
+ push_frontend_feature_flag(:security_auto_fix, default_enabled: false)
# Startup CSS feature is a special one as it can be enabled by means of cookies and params
gon.push({ features: { 'startupCss' => use_startup_css? } }, true)
@@ -59,9 +57,9 @@ module Gitlab
# name - The name of the feature flag, e.g. `my_feature`.
# args - Any additional arguments to pass to `Feature.enabled?`. This allows
# you to check if a flag is enabled for a particular user.
- def push_frontend_feature_flag(name, *args)
+ def push_frontend_feature_flag(name, *args, **kwargs)
var_name = name.to_s.camelize(:lower)
- enabled = Feature.enabled?(name, *args)
+ enabled = Feature.enabled?(name, *args, **kwargs)
# Here the `true` argument signals gon that the value should be merged
# into any existing ones, instead of overwriting them. This allows you to
diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb
index 27673e5c27a..c70127553fd 100644
--- a/lib/gitlab/graphql/authorize/authorize_resource.rb
+++ b/lib/gitlab/graphql/authorize/authorize_resource.rb
@@ -29,8 +29,8 @@ module Gitlab
raise NotImplementedError, "Implement #find_object in #{self.class.name}"
end
- def authorized_find!(*args)
- object = Graphql::Lazy.force(find_object(*args))
+ def authorized_find!(*args, **kwargs)
+ object = Graphql::Lazy.force(find_object(*args, **kwargs))
authorize!(object)
diff --git a/lib/gitlab/graphql/global_id_compatibility.rb b/lib/gitlab/graphql/global_id_compatibility.rb
new file mode 100644
index 00000000000..a96e4c4b976
--- /dev/null
+++ b/lib/gitlab/graphql/global_id_compatibility.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module GlobalIDCompatibility
+ # TODO: remove this module once the compatibility layer is no longer needed.
+ # See: https://gitlab.com/gitlab-org/gitlab/-/issues/257883
+ def coerce_global_id_arguments!(args)
+ global_id_arguments = self.class.arguments.values.select do |arg|
+ arg.type.is_a?(Class) && arg.type <= ::Types::GlobalIDType
+ end
+
+ global_id_arguments.each do |arg|
+ k = arg.keyword
+ args[k] &&= arg.type.coerce_isolated_input(args[k])
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb
index 7be6810f7ba..0b5bde8d8d9 100644
--- a/lib/gitlab/graphql/markdown_field.rb
+++ b/lib/gitlab/graphql/markdown_field.rb
@@ -12,13 +12,19 @@ module Gitlab
end
method_name = kwargs.delete(:method) || name.to_s.sub(/_html$/, '')
- kwargs[:resolve] = Gitlab::Graphql::MarkdownField::Resolver.new(method_name.to_sym).proc
+ resolver_method = "#{name}_resolver".to_sym
+ kwargs[:resolver_method] = resolver_method
kwargs[:description] ||= "The GitLab Flavored Markdown rendering of `#{method_name}`"
# Adding complexity to rendered notes since that could cause queries.
kwargs[:complexity] ||= 5
field name, GraphQL::STRING_TYPE, **kwargs
+
+ define_method resolver_method do
+ # We need to `dup` the context so the MarkdownHelper doesn't modify it
+ ::MarkupHelper.markdown_field(object, method_name.to_sym, context.to_h.dup)
+ end
end
end
end
diff --git a/lib/gitlab/graphql/markdown_field/resolver.rb b/lib/gitlab/graphql/markdown_field/resolver.rb
deleted file mode 100644
index 11a01b95ad1..00000000000
--- a/lib/gitlab/graphql/markdown_field/resolver.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Graphql
- module MarkdownField
- class Resolver
- attr_reader :method_name
-
- def initialize(method_name)
- @method_name = method_name
- end
-
- def proc
- -> (object, _args, ctx) do
- # We need to `dup` the context so the MarkdownHelper doesn't modify it
- ::MarkupHelper.markdown_field(object, method_name, ctx.to_h.dup)
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb
index 17cd22d38ad..252f6371765 100644
--- a/lib/gitlab/graphql/pagination/keyset/connection.rb
+++ b/lib/gitlab/graphql/pagination/keyset/connection.rb
@@ -110,8 +110,7 @@ module Gitlab
end
if last
- # grab one more than we need
- paginated_nodes = sliced_nodes.last(limit_value + 1)
+ paginated_nodes = LastItems.take_items(sliced_nodes, limit_value + 1)
# there is an extra node, so there is a previous page
@has_previous_page = paginated_nodes.count > limit_value
diff --git a/lib/gitlab/graphql/pagination/keyset/last_items.rb b/lib/gitlab/graphql/pagination/keyset/last_items.rb
new file mode 100644
index 00000000000..45bf15236c1
--- /dev/null
+++ b/lib/gitlab/graphql/pagination/keyset/last_items.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module Pagination
+ module Keyset
+ # This class handles the last(N) ActiveRecord call even if a special ORDER BY configuration is present.
+ # For the last(N) call, ActiveRecord calls reverse_order, however for some cases it raises
+ # ActiveRecord::IrreversibleOrderError error.
+ class LastItems
+ # rubocop: disable CodeReuse/ActiveRecord
+ def self.take_items(scope, count)
+ if custom_order = lookup_custom_reverse_order(scope.order_values)
+ items = scope.reorder(*custom_order).first(count) # returns a single record when count is nil
+ items.is_a?(Array) ? items.reverse : items
+ else
+ scope.last(count)
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ # Detect special ordering and provide the reversed order
+ def self.lookup_custom_reverse_order(order_values)
+ if ordering_by_merged_at_and_mr_id_desc?(order_values)
+ [
+ Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', 'ASC'), # reversing the order
+ MergeRequest.arel_table[:id].asc
+ ]
+ elsif ordering_by_merged_at_and_mr_id_asc?(order_values)
+ [
+ Gitlab::Database.nulls_first_order('merge_request_metrics.merged_at', 'DESC'),
+ MergeRequest.arel_table[:id].asc
+ ]
+ end
+ end
+
+ def self.ordering_by_merged_at_and_mr_id_desc?(order_values)
+ order_values.size == 2 &&
+ order_values.first.to_s == Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', 'DESC') &&
+ order_values.last.is_a?(Arel::Nodes::Descending) &&
+ order_values.last.to_sql == MergeRequest.arel_table[:id].desc.to_sql
+ end
+
+ def self.ordering_by_merged_at_and_mr_id_asc?(order_values)
+ order_values.size == 2 &&
+ order_values.first.to_s == Gitlab::Database.nulls_last_order('merge_request_metrics.merged_at', 'ASC') &&
+ order_values.last.is_a?(Arel::Nodes::Descending) &&
+ order_values.last.to_sql == MergeRequest.arel_table[:id].desc.to_sql
+ end
+
+ private_class_method :ordering_by_merged_at_and_mr_id_desc?
+ private_class_method :ordering_by_merged_at_and_mr_id_asc?
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb
index f54695ddb9a..f3ce3a10703 100644
--- a/lib/gitlab/graphql/pagination/keyset/order_info.rb
+++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb
@@ -94,6 +94,10 @@ module Gitlab
[order_value.expr.expressions[0].name.to_s, order_value.direction, order_value.expr]
elsif ordering_by_similarity?(order_value)
['similarity', order_value.direction, order_value.expr]
+ elsif ordering_by_case?(order_value)
+ ['case_order_value', order_value.direction, order_value.expr]
+ elsif ordering_by_array_position?(order_value)
+ ['array_position', order_value.direction, order_value.expr]
else
[order_value.expr.name, order_value.direction, nil]
end
@@ -104,9 +108,19 @@ module Gitlab
order_value.expr.is_a?(Arel::Nodes::NamedFunction) && order_value.expr&.name&.downcase == 'lower'
end
+ # determine if ordering using ARRAY_POSITION, eg. "ORDER BY ARRAY_POSITION(Array[4,3,1,2]::smallint, state)"
+ def ordering_by_array_position?(order_value)
+ order_value.expr.is_a?(Arel::Nodes::NamedFunction) && order_value.expr&.name&.downcase == 'array_position'
+ end
+
# determine if ordering using SIMILARITY scoring based on Gitlab::Database::SimilarityScore
def ordering_by_similarity?(order_value)
- order_value.to_sql.match?(/SIMILARITY\(.+\*/)
+ Gitlab::Database::SimilarityScore.order_by_similarity?(order_value)
+ end
+
+ # determine if ordering using CASE
+ def ordering_by_case?(order_value)
+ order_value.expr.is_a?(Arel::Nodes::Case)
end
end
end
diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
index 6b6bb72eb31..1285365376f 100644
--- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
+++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
@@ -6,6 +6,8 @@ module Gitlab
class LoggerAnalyzer
COMPLEXITY_ANALYZER = GraphQL::Analysis::QueryComplexity.new { |query, complexity_value| complexity_value }
DEPTH_ANALYZER = GraphQL::Analysis::QueryDepth.new { |query, depth_value| depth_value }
+ FIELD_USAGE_ANALYZER = GraphQL::Analysis::FieldUsage.new { |query, used_fields, used_deprecated_fields| [used_fields, used_deprecated_fields] }
+ ALL_ANALYZERS = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER, FIELD_USAGE_ANALYZER].freeze
def analyze?(query)
Feature.enabled?(:graphql_logging, default_enabled: true)
@@ -23,18 +25,21 @@ module Gitlab
end
def call(memo, visit_type, irep_node)
- memo
+ RequestStore.store[:graphql_logs] = memo
end
def final_value(memo)
return if memo.nil?
- analyzers = [COMPLEXITY_ANALYZER, DEPTH_ANALYZER]
- complexity, depth = GraphQL::Analysis.analyze_query(memo[:query], analyzers)
+ complexity, depth, field_usages = GraphQL::Analysis.analyze_query(memo[:query], ALL_ANALYZERS)
memo[:depth] = depth
memo[:complexity] = complexity
+ # This duration is not the execution time of the
+ # query but the execution time of the analyzer.
memo[:duration_s] = duration(memo[:time_started]).round(1)
+ memo[:used_fields] = field_usages.first
+ memo[:used_deprecated_fields] = field_usages.second
GraphqlLogger.info(memo.except!(:time_started, :query))
rescue => e
diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb
index 0cc3de297ba..5fec50eecd2 100644
--- a/lib/gitlab/group_search_results.rb
+++ b/lib/gitlab/group_search_results.rb
@@ -4,10 +4,10 @@ module Gitlab
class GroupSearchResults < SearchResults
attr_reader :group
- def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, filters: {})
+ def initialize(current_user, query, limit_projects = nil, group:, default_project_filter: false, sort: nil, filters: {})
@group = group
- super(current_user, query, limit_projects, default_project_filter: default_project_filter, filters: filters)
+ super(current_user, query, limit_projects, default_project_filter: default_project_filter, sort: sort, filters: filters)
end
# rubocop:disable CodeReuse/ActiveRecord
@@ -38,3 +38,5 @@ module Gitlab
end
end
end
+
+Gitlab::GroupSearchResults.prepend_if_ee('EE::Gitlab::GroupSearchResults')
diff --git a/lib/gitlab/health_checks/unicorn_check.rb b/lib/gitlab/health_checks/unicorn_check.rb
index cdc6d2a7519..f0c6fdab600 100644
--- a/lib/gitlab/health_checks/unicorn_check.rb
+++ b/lib/gitlab/health_checks/unicorn_check.rb
@@ -22,7 +22,7 @@ module Gitlab
def check
return unless http_servers
- http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord
+ http_servers.sum(&:worker_processes)
end
# Traversal of ObjectSpace is expensive, on fully loaded application
diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb
index 018cb36fc58..379a734b19c 100644
--- a/lib/gitlab/import_export/attribute_cleaner.rb
+++ b/lib/gitlab/import_export/attribute_cleaner.rb
@@ -20,8 +20,8 @@ module Gitlab
/\Aremote_\w+_(url|urls|request_header)\Z/ # carrierwave automatically creates these attribute methods for uploads
).freeze
- def self.clean(*args)
- new(*args).clean
+ def self.clean(*args, **kwargs)
+ new(*args, **kwargs).clean
end
def initialize(relation_hash:, relation_class:, excluded_keys: [])
diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb
index 05b69362976..d60bc79df4c 100644
--- a/lib/gitlab/import_export/base/relation_factory.rb
+++ b/lib/gitlab/import_export/base/relation_factory.rb
@@ -31,8 +31,8 @@ module Gitlab
TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
- def self.create(*args)
- new(*args).create
+ def self.create(*args, **kwargs)
+ new(*args, **kwargs).create
end
def self.relation_class(relation_name)
@@ -53,6 +53,7 @@ module Gitlab
@importable = importable
@imported_object_retries = 0
@relation_hash[importable_column_name] = @importable.id
+ @original_user = {}
# Remove excluded keys from relation_hash
# We don't do this in the parsed_relation_hash because of the 'transformed attributes'
@@ -112,6 +113,7 @@ module Gitlab
def update_user_references
self.class::USER_REFERENCES.each do |reference|
if @relation_hash[reference]
+ @original_user[reference] = @relation_hash[reference]
@relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
end
end
@@ -243,28 +245,20 @@ module Gitlab
# will be used. Otherwise, a note stating the original author name
# is left.
def set_note_author
- old_author_id = @relation_hash['author_id']
+ old_author_id = @original_user['author_id']
author = @relation_hash.delete('author')
- update_note_for_missing_author(author['name']) unless has_author?(old_author_id)
- end
-
- def has_author?(old_author_id)
- admin_user? && @members_mapper.include?(old_author_id)
+ unless @members_mapper.include?(old_author_id)
+ @relation_hash['note'] = "%{note}\n\n %{missing_author_note}" % {
+ note: @relation_hash['note'].presence || '*Blank note*',
+ missing_author_note: missing_author_note(@relation_hash['updated_at'], author['name'])
+ }
+ end
end
def missing_author_note(updated_at, author_name)
timestamp = updated_at.split('.').first
- "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
- end
-
- def update_note_for_missing_author(author_name)
- @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
- @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}"
- end
-
- def admin_user?
- @user.admin?
+ "*By #{author_name} on #{timestamp} (imported from GitLab)*"
end
def existing_object?
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 081745a49f4..5a6f6e017d2 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -10,8 +10,8 @@ module Gitlab
MAX_RETRIES = 8
IGNORED_FILENAMES = %w(. ..).freeze
- def self.import(*args)
- new(*args).import
+ def self.import(*args, **kwargs)
+ new(*args, **kwargs).import
end
def initialize(importable:, archive_file:, shared:)
diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb
index e9b05afc7d4..0d9839b86cf 100644
--- a/lib/gitlab/import_export/json/ndjson_reader.rb
+++ b/lib/gitlab/import_export/json/ndjson_reader.rb
@@ -35,6 +35,7 @@ module Gitlab
# This reads from `tree/project/merge_requests.ndjson`
path = file_path(importable_path, "#{key}.ndjson")
+
next unless File.exist?(path)
File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num|
@@ -43,6 +44,11 @@ module Gitlab
end
end
+ # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330)
+ def clear_consumed_relations
+ @consumed_relations.clear
+ end
+
private
def json_decode(string)
diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb
index 515fd98630c..4964b8b16f4 100644
--- a/lib/gitlab/import_export/lfs_saver.rb
+++ b/lib/gitlab/import_export/lfs_saver.rb
@@ -21,10 +21,10 @@ module Gitlab
save_lfs_object(lfs_object)
end
- append_lfs_json_for_batch(batch) if write_lfs_json_enabled?
+ append_lfs_json_for_batch(batch)
end
- write_lfs_json if write_lfs_json_enabled?
+ write_lfs_json
true
rescue => e
@@ -35,10 +35,6 @@ module Gitlab
private
- def write_lfs_json_enabled?
- ::Feature.enabled?(:export_lfs_objects_projects, default_enabled: true)
- end
-
def save_lfs_object(lfs_object)
if lfs_object.local_store?
copy_file_for_lfs_object(lfs_object)
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index 31d1f7b48bd..6b37683ea68 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -34,8 +34,8 @@ module Gitlab
@user.id
end
- def include?(old_author_id)
- map.has_key?(old_author_id) && map[old_author_id] != default_user_id
+ def include?(old_user_id)
+ map.has_key?(old_user_id)
end
private
@@ -63,6 +63,8 @@ module Gitlab
end
def add_team_member(member, existing_user = nil)
+ return true if existing_user && @importable.members.exists?(user_id: existing_user.id)
+
member['user'] = existing_user
member_hash = member_hash(member)
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 9ec5df8cde9..a0526ba0414 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -403,9 +403,15 @@ ee:
- issues:
- epic_issue:
- :epic
+ - :issuable_sla
- protected_branches:
- :unprotect_access_levels
- protected_environments:
- :deploy_access_levels
- :service_desk_setting
- :security_setting
+
+ included_attributes:
+ issuable_sla:
+ - :issue
+ - :due_at
diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb
new file mode 100644
index 00000000000..2d989d21166
--- /dev/null
+++ b/lib/gitlab/import_export/project/sample/date_calculator.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ module Sample
+ class DateCalculator
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(dates)
+ @dates = dates.dup
+ @dates.flatten!
+ @dates.compact!
+ @dates.sort!
+ @dates.map! { |date| date.to_time.to_f }
+ end
+
+ def closest_date_to_average
+ strong_memoize(:closest_date_to_average) do
+ next if @dates.empty?
+
+ average_date = (@dates.first + @dates.last) / 2.0
+ closest_date = @dates.min_by { |date| (date - average_date).abs }
+ Time.zone.at(closest_date)
+ end
+ end
+
+ def calculate_by_closest_date_to_average(date)
+ return date unless closest_date_to_average && closest_date_to_average < Time.current
+
+ date + (Time.current - closest_date_to_average).seconds
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb
new file mode 100644
index 00000000000..b0c3940b5f9
--- /dev/null
+++ b/lib/gitlab/import_export/project/sample/sample_data_relation_tree_restorer.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ module Sample
+ class SampleDataRelationTreeRestorer < RelationTreeRestorer
+ DATE_MODELS = %i[issues milestones].freeze
+
+ def initialize(*args)
+ super
+
+ date_calculator
+ end
+
+ private
+
+ def build_relation(relation_key, relation_definition, data_hash)
+ # Override due date attributes in data hash for Sample Data templates
+ # Dates are moved by taking the closest one to average and moving that (and rest around it) to the date of import
+ # TODO: To move this logic to RelationFactory (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465333)
+ override_date_attributes!(relation_key, data_hash)
+ super
+ end
+
+ def override_date_attributes!(relation_key, data_hash)
+ return unless DATE_MODELS.include?(relation_key.to_sym)
+
+ data_hash['start_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['start_date'].to_time) unless data_hash['start_date'].nil?
+ data_hash['due_date'] = date_calculator.calculate_by_closest_date_to_average(data_hash['due_date'].to_time) unless data_hash['due_date'].nil?
+ end
+
+ # TODO: Move clear logic into main comsume_relation method (see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430465330)
+ def dates
+ unless relation_reader.legacy?
+ DATE_MODELS.map do |tag|
+ relation_reader.consume_relation(@importable_path, tag).map { |model| model.first['due_date'] }.tap do
+ relation_reader.clear_consumed_relations
+ end
+ end
+ end
+ end
+
+ def date_calculator
+ @date_calculator ||= Gitlab::ImportExport::Project::Sample::DateCalculator.new(dates)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
index a16ffe36054..b1d647281ab 100644
--- a/lib/gitlab/import_export/project/tree_restorer.rb
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -70,7 +70,7 @@ module Gitlab
end
def relation_tree_restorer
- @relation_tree_restorer ||= RelationTreeRestorer.new(
+ @relation_tree_restorer ||= relation_tree_restorer_class.new(
user: @user,
shared: @shared,
relation_reader: relation_reader,
@@ -84,6 +84,14 @@ module Gitlab
)
end
+ def relation_tree_restorer_class
+ sample_data_template? ? Sample::SampleDataRelationTreeRestorer : RelationTreeRestorer
+ end
+
+ def sample_data_template?
+ @project&.import_data&.data&.dig('sample_data')
+ end
+
def members_mapper
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
user: @user,
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 7cca3596da6..80dacf2eb20 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -36,7 +36,7 @@ module Gitlab
end
def exportable
- @project.present(exportable_params)
+ @project.present(**exportable_params)
end
def exportable_params
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index 9e10e7aea13..f808e30bd6e 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -14,10 +14,10 @@ module Gitlab
def restore
return true unless File.exist?(path_to_bundle)
+ ensure_repository_does_not_exist!
+
repository.create_from_bundle(path_to_bundle)
rescue => e
- Repositories::DestroyService.new(repository).execute
-
shared.error(e)
false
end
@@ -25,6 +25,16 @@ module Gitlab
private
attr_accessor :repository, :path_to_bundle, :shared
+
+ def ensure_repository_does_not_exist!
+ if repository.exists?
+ shared.logger.info(
+ message: %Q{Deleting existing "#{repository.path}" to re-import it.}
+ )
+
+ Repositories::DestroyService.new(repository).execute
+ end
+ end
end
end
end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index e4724659eff..045ba2495bf 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -5,8 +5,8 @@ module Gitlab
class Saver
include Gitlab::ImportExport::CommandLineUtil
- def self.save(*args)
- new(*args).save
+ def self.save(*args, **kwargs)
+ new(*args, **kwargs).save
end
def initialize(exportable:, shared:)
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index dca8e3a7449..26e7d2cf765 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -40,7 +40,7 @@ module Gitlab
def add_upload(upload)
uploader_context = FileUploader.extract_dynamic_path(upload).named_captures.symbolize_keys
- UploadService.new(@project, File.open(upload, 'r'), FileUploader, uploader_context).execute.to_h
+ UploadService.new(@project, File.open(upload, 'r'), FileUploader, **uploader_context).execute.to_h
end
def copy_project_uploads
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
index 4154d4fe775..48f5b558e52 100644
--- a/lib/gitlab/import_export/version_checker.rb
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -3,8 +3,8 @@
module Gitlab
module ImportExport
class VersionChecker
- def self.check!(*args)
- new(*args).check!
+ def self.check!(*args, **kwargs)
+ new(*args, **kwargs).check!
end
def initialize(shared:)
diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb
index 4a85a313fd7..d1ac6a55fb7 100644
--- a/lib/gitlab/instrumentation/redis.rb
+++ b/lib/gitlab/instrumentation/redis.rb
@@ -37,7 +37,7 @@ module Gitlab
%i[get_request_count query_time read_bytes write_bytes].each do |method|
define_method method do
- STORAGES.sum(&method) # rubocop:disable CodeReuse/ActiveRecord
+ STORAGES.sum(&method)
end
end
end
diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb
index 7be54a214dd..945ab7f40c2 100644
--- a/lib/gitlab/issuables_count_for_state.rb
+++ b/lib/gitlab/issuables_count_for_state.rb
@@ -16,9 +16,12 @@ module Gitlab
end
# finder - The finder class to use for retrieving the issuables.
- def initialize(finder, project = nil)
+ # fast_fail - restrict counting to a shorter period, degrading gracefully on
+ # failure
+ def initialize(finder, project = nil, fast_fail: false)
@finder = finder
@project = project
+ @fast_fail = fast_fail
@cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache
end
@@ -26,6 +29,10 @@ module Gitlab
self[state || :opened]
end
+ def fast_fail?
+ !!@fast_fail
+ end
+
# Define method for each state
STATES.each do |state|
define_method(state) { self[state] }
@@ -53,7 +60,53 @@ module Gitlab
end
def initialize_cache
- Hash.new { |hash, finder| hash[finder] = finder.count_by_state }
+ Hash.new { |hash, finder| hash[finder] = perform_count(finder) }
+ end
+
+ def perform_count(finder)
+ return finder.count_by_state unless fast_fail?
+
+ fast_count_by_state_attempt!
+
+ # Determining counts when referring to issuable titles or descriptions can
+ # be very expensive, and involve the database reading gigabytes of data
+ # for a relatively minor piece of functionality. This may slow index pages
+ # by seconds in the best case, or lead to a statement timeout in the worst
+ # case.
+ #
+ # In time, we may be able to use elasticsearch or postgresql tsv columns
+ # to perform the calculation more efficiently. Until then, use a shorter
+ # timeout and return -1 as a sentinel value if it is triggered
+ begin
+ ApplicationRecord.with_fast_statement_timeout do
+ finder.count_by_state
+ end
+ rescue ActiveRecord::QueryCanceled => err
+ fast_count_by_state_failure!
+
+ Gitlab::ErrorTracking.track_exception(
+ err,
+ params: finder.params,
+ current_user_id: finder.current_user&.id,
+ issue_url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/249180'
+ )
+
+ Hash.new(-1)
+ end
+ end
+
+ def fast_count_by_state_attempt!
+ Gitlab::Metrics.counter(
+ :gitlab_issuable_fast_count_by_state_total,
+ "Count of total calls to IssuableFinder#count_by_state with fast failure"
+ ).increment
+ end
+
+ def fast_count_by_state_failure!
+ Gitlab::Metrics.counter(
+ :gitlab_issuable_fast_count_by_state_failures_total,
+ "Count of failed calls to IssuableFinder#count_by_state with fast failure"
+ ).increment
end
end
end
diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb
index e7a8cc6305a..2cede524cac 100644
--- a/lib/gitlab/job_waiter.rb
+++ b/lib/gitlab/job_waiter.rb
@@ -23,7 +23,15 @@ module Gitlab
TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total
def self.notify(key, jid)
- Gitlab::Redis::SharedState.with { |redis| redis.lpush(key, jid) }
+ Gitlab::Redis::SharedState.with do |redis|
+ # Use a Redis MULTI transaction to ensure we always set an expiry
+ redis.multi do |multi|
+ multi.lpush(key, jid)
+ # This TTL needs to be long enough to allow whichever Sidekiq job calls
+ # JobWaiter#wait to reach BLPOP.
+ multi.expire(key, 6.hours.to_i)
+ end
+ end
end
def self.key?(key)
@@ -52,10 +60,6 @@ module Gitlab
increment_counter(STARTED_METRIC)
Gitlab::Redis::SharedState.with do |redis|
- # Fallback key expiry: allow a long grace period to reduce the chance of
- # a job pushing to an expired key and recreating it
- redis.expire(key, [timeout * 2, 10.minutes.to_i].max)
-
while jobs_remaining > 0
# Redis will not take fractional seconds. Prefer waiting too long over
# not waiting long enough
@@ -75,9 +79,6 @@ module Gitlab
@finished << jid
@jobs_remaining -= 1
end
-
- # All jobs have finished, so expire the key immediately
- redis.expire(key, 0) if jobs_remaining == 0
end
finished
diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb
index fa68afd39f5..13cd6dcad3f 100644
--- a/lib/gitlab/kubernetes/kube_client.rb
+++ b/lib/gitlab/kubernetes/kube_client.rb
@@ -167,6 +167,21 @@ module Gitlab
end
end
+ # Ingresses resource is currently on the apis/extensions api group
+ # until Kubernetes 1.21. Kubernetest 1.22+ has ingresses resources in
+ # the networking.k8s.io/v1 api group.
+ #
+ # As we still support Kubernetes 1.12+, we will need to support both.
+ def get_ingresses(**args)
+ extensions_client.discover unless extensions_client.discovered
+
+ if extensions_client.respond_to?(:get_ingresses)
+ extensions_client.get_ingresses(**args)
+ else
+ networking_client.get_ingresses(**args)
+ end
+ end
+
def create_or_update_cluster_role_binding(resource)
update_cluster_role_binding(resource)
end
diff --git a/lib/gitlab/kubernetes/pod.rb b/lib/gitlab/kubernetes/pod.rb
index d247662dc3b..a5651f2f184 100644
--- a/lib/gitlab/kubernetes/pod.rb
+++ b/lib/gitlab/kubernetes/pod.rb
@@ -2,13 +2,47 @@
module Gitlab
module Kubernetes
- module Pod
+ class Pod
PENDING = 'Pending'
RUNNING = 'Running'
SUCCEEDED = 'Succeeded'
FAILED = 'Failed'
UNKNOWN = 'Unknown'
PHASES = [PENDING, RUNNING, SUCCEEDED, FAILED, UNKNOWN].freeze
+
+ STABLE_TRACK_VALUE = 'stable'
+
+ def initialize(attributes = {})
+ @attributes = attributes
+ end
+
+ def track
+ attributes.dig('metadata', 'labels', 'track') || STABLE_TRACK_VALUE
+ end
+
+ def name
+ metadata['name'] || metadata['generateName']
+ end
+
+ def stable?
+ track == STABLE_TRACK_VALUE
+ end
+
+ def status
+ attributes.dig('status', 'phase')
+ end
+
+ def order
+ stable? ? 1 : 0
+ end
+
+ private
+
+ attr_reader :attributes
+
+ def metadata
+ attributes.fetch('metadata', {})
+ end
end
end
end
diff --git a/lib/gitlab/lfs/client.rb b/lib/gitlab/lfs/client.rb
index e4d600694c2..825d7399190 100644
--- a/lib/gitlab/lfs/client.rb
+++ b/lib/gitlab/lfs/client.rb
@@ -6,6 +6,14 @@ module Gitlab
# * https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
# * https://github.com/git-lfs/git-lfs/blob/master/docs/api/basic-transfers.md
class Client
+ GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs+json'
+ GIT_LFS_USER_AGENT = "GitLab #{Gitlab::VERSION} LFS client"
+ DEFAULT_HEADERS = {
+ 'Accept' => GIT_LFS_CONTENT_TYPE,
+ 'Content-Type' => GIT_LFS_CONTENT_TYPE,
+ 'User-Agent' => GIT_LFS_USER_AGENT
+ }.freeze
+
attr_reader :base_url
def initialize(base_url, credentials:)
@@ -13,19 +21,19 @@ module Gitlab
@credentials = credentials
end
- def batch(operation, objects)
+ def batch!(operation, objects)
body = {
operation: operation,
transfers: ['basic'],
# We don't know `ref`, so can't send it
- objects: objects.map { |object| { oid: object.oid, size: object.size } }
+ objects: objects.as_json(only: [:oid, :size])
}
rsp = Gitlab::HTTP.post(
batch_url,
basic_auth: basic_auth,
body: body.to_json,
- headers: { 'Content-Type' => 'application/vnd.git-lfs+json' }
+ headers: build_request_headers
)
raise BatchSubmitError unless rsp.success?
@@ -40,17 +48,19 @@ module Gitlab
body
end
- def upload(object, upload_action, authenticated:)
+ def upload!(object, upload_action, authenticated:)
file = object.file.open
params = {
body_stream: file,
headers: {
'Content-Length' => object.size.to_s,
- 'Content-Type' => 'application/octet-stream'
+ 'Content-Type' => 'application/octet-stream',
+ 'User-Agent' => GIT_LFS_USER_AGENT
}.merge(upload_action['header'] || {})
}
+ authenticated = true if params[:headers].key?('Authorization')
params[:basic_auth] = basic_auth unless authenticated
rsp = Gitlab::HTTP.put(upload_action['href'], params)
@@ -60,8 +70,26 @@ module Gitlab
file&.close
end
+ def verify!(object, verify_action, authenticated:)
+ params = {
+ body: object.to_json(only: [:oid, :size]),
+ headers: build_request_headers(verify_action['header'])
+ }
+
+ authenticated = true if params[:headers].key?('Authorization')
+ params[:basic_auth] = basic_auth unless authenticated
+
+ rsp = Gitlab::HTTP.post(verify_action['href'], params)
+
+ raise ObjectVerifyError unless rsp.success?
+ end
+
private
+ def build_request_headers(extra_headers = nil)
+ DEFAULT_HEADERS.merge(extra_headers || {})
+ end
+
attr_reader :credentials
def batch_url
@@ -96,6 +124,12 @@ module Gitlab
"Failed to upload object"
end
end
+
+ class ObjectVerifyError < StandardError
+ def message
+ "Failed to verify object"
+ end
+ end
end
end
end
diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb
new file mode 100644
index 00000000000..80dff075391
--- /dev/null
+++ b/lib/gitlab/manifest_import/metadata.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ManifestImport
+ class Metadata
+ EXPIRY_TIME = 1.week
+
+ attr_reader :user, :fallback
+
+ def initialize(user, fallback: {})
+ @user = user
+ @fallback = fallback
+ end
+
+ def save(repositories, group_id)
+ Gitlab::Redis::SharedState.with do |redis|
+ redis.multi do
+ redis.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME)
+ redis.set(key_for('group_id'), group_id, ex: EXPIRY_TIME)
+ end
+ end
+ end
+
+ def repositories
+ redis_get('repositories').then do |repositories|
+ next unless repositories
+
+ Gitlab::Json.parse(repositories).map(&:symbolize_keys)
+ end || fallback[:manifest_import_repositories]
+ end
+
+ def group_id
+ redis_get('group_id')&.to_i || fallback[:manifest_import_group_id]
+ end
+
+ private
+
+ def key_for(field)
+ "manifest_import:metadata:user:#{user.id}:#{field}"
+ end
+
+ def redis_get(field)
+ Gitlab::Redis::SharedState.with do |redis|
+ redis.get(key_for(field))
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/marginalia.rb b/lib/gitlab/marginalia.rb
index 24e21a1d512..325a8c5c325 100644
--- a/lib/gitlab/marginalia.rb
+++ b/lib/gitlab/marginalia.rb
@@ -4,8 +4,6 @@ module Gitlab
module Marginalia
cattr_accessor :enabled, default: false
- MARGINALIA_FEATURE_FLAG = :marginalia
-
def self.set_application_name
::Marginalia.application_name = Gitlab.process_name
end
@@ -16,15 +14,11 @@ module Gitlab
end
end
- def self.cached_feature_enabled?
- enabled
- end
-
- def self.set_feature_cache
+ def self.set_enabled_from_feature_flag
# During db:create and db:bootstrap skip feature query as DB is not available yet.
return false unless Gitlab::Database.cached_table_exists?('features')
- self.enabled = Feature.enabled?(MARGINALIA_FEATURE_FLAG)
+ self.enabled = Feature.enabled?(:marginalia, type: :ops)
end
end
end
diff --git a/lib/gitlab/marginalia/active_record_instrumentation.rb b/lib/gitlab/marginalia/active_record_instrumentation.rb
index 3266b9f8336..452f472bf6a 100644
--- a/lib/gitlab/marginalia/active_record_instrumentation.rb
+++ b/lib/gitlab/marginalia/active_record_instrumentation.rb
@@ -5,7 +5,7 @@ module Gitlab
module Marginalia
module ActiveRecordInstrumentation
def annotate_sql(sql)
- Gitlab::Marginalia.cached_feature_enabled? ? super(sql) : sql
+ Gitlab::Marginalia.enabled ? super(sql) : sql
end
end
end
diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb
index da3b597a74e..36e9a6ccef6 100644
--- a/lib/gitlab/markdown_cache.rb
+++ b/lib/gitlab/markdown_cache.rb
@@ -3,7 +3,7 @@
module Gitlab
module MarkdownCache
# Increment this number every time the renderer changes its output
- CACHE_COMMONMARK_VERSION = 25
+ CACHE_COMMONMARK_VERSION = 26
CACHE_COMMONMARK_VERSION_START = 10
BaseError = Class.new(StandardError)
diff --git a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
index d1490d5d9b6..8a176be30a2 100644
--- a/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
+++ b/lib/gitlab/metrics/dashboard/importers/prometheus_metrics.rb
@@ -13,11 +13,12 @@ module Gitlab
@dashboard_hash = dashboard_hash
@project = project
@dashboard_path = dashboard_path
+ @affected_environment_ids = []
end
def execute
import
- rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError
+ rescue ActiveRecord::RecordInvalid, Dashboard::Transformers::Errors::BaseError
false
end
@@ -32,28 +33,51 @@ module Gitlab
def import
delete_stale_metrics
create_or_update_metrics
+ update_prometheus_environments
end
# rubocop: disable CodeReuse/ActiveRecord
def create_or_update_metrics
# TODO: use upsert and worker for callbacks?
+
+ affected_metric_ids = []
prometheus_metrics_attributes.each do |attributes|
- prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project))
+ prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:dashboard_path, :identifier, :project))
prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES))
+
+ affected_metric_ids << prometheus_metric.id
end
+
+ @affected_environment_ids += find_alerts(affected_metric_ids).get_environment_id
end
# rubocop: enable CodeReuse/ActiveRecord
def delete_stale_metrics
- identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] }
+ identifiers_from_yml = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] }
stale_metrics = PrometheusMetric.for_project(project)
.for_dashboard_path(dashboard_path)
.for_group(Enums::PrometheusMetric.groups[:custom])
- .not_identifier(identifiers)
+ .not_identifier(identifiers_from_yml)
+
+ return unless stale_metrics.exists?
+
+ delete_stale_alerts(stale_metrics)
+ stale_metrics.each_batch { |batch| batch.delete_all }
+ end
+
+ def delete_stale_alerts(stale_metrics)
+ stale_alerts = find_alerts(stale_metrics)
+
+ affected_environment_ids = stale_alerts.get_environment_id
+ return unless affected_environment_ids.present?
- # TODO: use destroy_all and worker for callbacks?
- stale_metrics.each(&:destroy)
+ @affected_environment_ids += affected_environment_ids
+ stale_alerts.each_batch { |batch| batch.delete_all }
+ end
+
+ def find_alerts(metrics)
+ Projects::Prometheus::AlertsFinder.new(project: project, metric: metrics).execute
end
def prometheus_metrics_attributes
@@ -65,6 +89,19 @@ module Gitlab
).execute
end
end
+
+ def update_prometheus_environments
+ affected_environments = ::Environment.for_id(@affected_environment_ids.flatten.uniq).for_project(project)
+
+ return unless affected_environments.exists?
+
+ affected_environments.each do |affected_environment|
+ ::Clusters::Applications::ScheduleUpdateService.new(
+ affected_environment.cluster_prometheus_adapter,
+ project
+ ).execute
+ end
+ end
end
end
end
diff --git a/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb
new file mode 100644
index 00000000000..5ed4466f440
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/custom_dashboard_metrics_inserter.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Dashboard
+ module Stages
+ # Acts on metrics which have been ingested from source controlled dashboards
+ class CustomDashboardMetricsInserter < BaseStage
+ # For each metric in the dashboard config, attempts to
+ # find a corresponding database record. If found, includes
+ # the record's id in the dashboard config.
+ def transform!
+ database_metrics = ::PrometheusMetricsFinder.new(common: false, group: :custom, project: project).execute
+
+ for_metrics do |metric|
+ metric_record = database_metrics.find { |m| m.identifier == metric[:id] }
+ metric[:metric_id] = metric_record.id if metric_record
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
index 9e2bb0d1a70..ad9d78133af 100644
--- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb
+++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
@@ -46,7 +46,7 @@ module Gitlab
links&.each do |link|
next unless link.is_a? Hash
- Gitlab::UrlBlocker.validate!(link[:url], blocker_args)
+ Gitlab::UrlBlocker.validate!(link[:url], **blocker_args)
rescue Gitlab::UrlBlocker::BlockedUrlError
link[:url] = ''
end
diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb
index 4d94ab098ae..bc85dc4e131 100644
--- a/lib/gitlab/metrics/dashboard/transformers/errors.rb
+++ b/lib/gitlab/metrics/dashboard/transformers/errors.rb
@@ -4,10 +4,10 @@ module Gitlab
module Metrics
module Dashboard
module Transformers
- TransformerError = Class.new(StandardError)
-
module Errors
- class MissingAttribute < TransformerError
+ BaseError = Class.new(StandardError)
+
+ class MissingAttribute < BaseError
def initialize(attribute_name)
super("Missing attribute: '#{attribute_name}'")
end
diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb
index 15db3999fa4..f6bda0dbea4 100644
--- a/lib/gitlab/metrics/requests_rack_middleware.rb
+++ b/lib/gitlab/metrics/requests_rack_middleware.rb
@@ -3,20 +3,13 @@
module Gitlab
module Metrics
class RequestsRackMiddleware
- HTTP_METHODS = {
- "delete" => %w(200 202 204 303 400 401 403 404 410 422 500 503),
- "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 412 422 429 500 503),
- "head" => %w(200 204 301 302 303 304 400 401 403 404 410 429 500 503),
- "options" => %w(200 404),
- "patch" => %w(200 202 204 400 403 404 409 416 422 500),
- "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 413 415 422 429 500 503),
- "propfind" => %w(404),
- "put" => %w(200 202 204 400 401 403 404 405 406 409 410 415 422 500),
- "report" => %w(404)
- }.freeze
+ HTTP_METHODS = %w(delete get head options patch post put).to_set.freeze
HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze
+ FEATURE_CATEGORY_HEADER = 'X-Gitlab-Feature-Category'
+ FEATURE_CATEGORY_DEFAULT = 'unknown'
+
def initialize(app)
@app = app
end
@@ -39,33 +32,39 @@ module Gitlab
end
def self.initialize_http_request_duration_seconds
- HTTP_METHODS.each do |method, statuses|
- statuses.each do |status|
- http_request_duration_seconds.get({ method: method, status: status.to_s })
- end
+ HTTP_METHODS.each do |method|
+ http_request_duration_seconds.get({ method: method })
end
end
def call(env)
method = env['REQUEST_METHOD'].downcase
+ method = 'INVALID' unless HTTP_METHODS.include?(method)
started = Time.now.to_f
+ health_endpoint = health_endpoint?(env['PATH_INFO'])
+ status = 'undefined'
+ feature_category = nil
begin
- if health_endpoint?(env['PATH_INFO'])
- RequestsRackMiddleware.http_health_requests_total.increment(method: method)
- else
- RequestsRackMiddleware.http_request_total.increment(method: method)
- end
-
status, headers, body = @app.call(env)
elapsed = Time.now.to_f - started
- RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method, status: status.to_s }, elapsed)
+ feature_category = headers&.fetch(FEATURE_CATEGORY_HEADER, nil)
+
+ unless health_endpoint
+ RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method }, elapsed)
+ end
[status, headers, body]
rescue
RequestsRackMiddleware.rack_uncaught_errors_count.increment
raise
+ ensure
+ if health_endpoint
+ RequestsRackMiddleware.http_health_requests_total.increment(status: status, method: method)
+ else
+ RequestsRackMiddleware.http_request_total.increment(status: status, method: method, feature_category: feature_category || FEATURE_CATEGORY_DEFAULT)
+ end
end
end
diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
index 8c4d150adad..d7935d65e12 100644
--- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb
+++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
@@ -54,7 +54,7 @@ module Gitlab
end
def unicorn_workers_count
- http_servers.sum(&:worker_processes) # rubocop: disable CodeReuse/ActiveRecord
+ http_servers.sum(&:worker_processes)
end
# Traversal of ObjectSpace is expensive, on fully loaded application
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index e53ac00e77f..f9ba0a69b0e 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -61,7 +61,7 @@ module Gitlab
end
def current_transaction
- Transaction.current
+ ::Gitlab::Metrics::Transaction.current
end
end
end
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index 47d0b9ba8cb..4b65bbcc791 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -18,6 +18,15 @@ module Gitlab
request = ActionDispatch::Request.new(env)
render_go_doc(request) || @app.call(env)
+ rescue Gitlab::Auth::IpBlacklisted
+ Gitlab::AuthLogger.error(
+ message: 'Rack_Attack',
+ env: :blocklist,
+ remote_ip: request.ip,
+ request_method: request.request_method,
+ path: request.fullpath
+ )
+ Rack::Response.new('', 403).finish
end
private
diff --git a/lib/gitlab/middleware/handle_null_bytes.rb b/lib/gitlab/middleware/handle_null_bytes.rb
new file mode 100644
index 00000000000..c88dfb6ee0b
--- /dev/null
+++ b/lib/gitlab/middleware/handle_null_bytes.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Middleware
+ # There is no valid reason for a request to contain a null byte (U+0000)
+ # so just return HTTP 400 (Bad Request) if we receive one
+ class HandleNullBytes
+ NULL_BYTE_REGEX = Regexp.new(Regexp.escape("\u0000")).freeze
+
+ attr_reader :app
+
+ def initialize(app)
+ @app = app
+ end
+
+ def call(env)
+ return [400, {}, ["Bad Request"]] if request_has_null_byte?(env)
+
+ app.call(env)
+ end
+
+ private
+
+ def request_has_null_byte?(request)
+ return false if ENV['REJECT_NULL_BYTES'] == "1"
+
+ request = Rack::Request.new(request)
+
+ request.params.values.any? do |value|
+ param_has_null_byte?(value)
+ end
+ end
+
+ def param_has_null_byte?(value, depth = 0)
+ # Guard against possible attack sending large amounts of nested params
+ # Should be safe as deeply nested params are highly uncommon.
+ return false if depth > 2
+
+ depth += 1
+
+ if value.respond_to?(:match)
+ string_contains_null_byte?(value)
+ elsif value.respond_to?(:values)
+ value.values.any? do |hash_value|
+ param_has_null_byte?(hash_value, depth)
+ end
+ elsif value.is_a?(Array)
+ value.any? do |array_value|
+ param_has_null_byte?(array_value, depth)
+ end
+ else
+ false
+ end
+ end
+
+ def string_contains_null_byte?(string)
+ string.match?(NULL_BYTE_REGEX)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index 8e6ac7610f2..7e98f1fc1f7 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -137,6 +137,7 @@ module Gitlab
# TODO this class is meant to replace Handler when the feature flag
# upload_middleware_jwt_params_handler is removed
+ # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps
class HandlerForJWTParams < Handler
def with_open_files
@rewritten_fields.keys.each do |field|
@@ -228,7 +229,7 @@ module Gitlab
private
def handler_class
- if Feature.enabled?(:upload_middleware_jwt_params_handler)
+ if Feature.enabled?(:upload_middleware_jwt_params_handler, default_enabled: true)
::Gitlab::Middleware::Multipart::HandlerForJWTParams
else
::Gitlab::Middleware::Multipart::Handler
diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb
index 8796dd4d7ec..46c74b8fe3c 100644
--- a/lib/gitlab/pagination/offset_pagination.rb
+++ b/lib/gitlab/pagination/offset_pagination.rb
@@ -10,9 +10,9 @@ module Gitlab
@request_context = request_context
end
- def paginate(relation)
+ def paginate(relation, exclude_total_headers: false)
paginate_with_limit_optimization(add_default_order(relation)).tap do |data|
- add_pagination_headers(data)
+ add_pagination_headers(data, exclude_total_headers)
end
end
@@ -27,7 +27,7 @@ module Gitlab
end
return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation)
- return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit)
+ return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops)
limited_total_count = pagination_data.total_count_with_limit
if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT
@@ -47,14 +47,14 @@ module Gitlab
relation
end
- def add_pagination_headers(paginated_data)
+ def add_pagination_headers(paginated_data, exclude_total_headers)
header 'X-Per-Page', paginated_data.limit_value.to_s
header 'X-Page', paginated_data.current_page.to_s
header 'X-Next-Page', paginated_data.next_page.to_s
header 'X-Prev-Page', paginated_data.prev_page.to_s
header 'Link', pagination_links(paginated_data)
- return if data_without_counts?(paginated_data)
+ return if exclude_total_headers || data_without_counts?(paginated_data)
header 'X-Total', paginated_data.total_count.to_s
header 'X-Total-Pages', total_pages(paginated_data).to_s
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index ded8d4ade3f..333564bee01 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -4,11 +4,11 @@ module Gitlab
class ProjectSearchResults < SearchResults
attr_reader :project, :repository_ref
- def initialize(current_user, query, project:, repository_ref: nil, filters: {})
+ def initialize(current_user, query, project:, repository_ref: nil, sort: nil, filters: {})
@project = project
@repository_ref = repository_ref.presence
- super(current_user, query, [project], filters: filters)
+ super(current_user, query, [project], sort: sort, filters: filters)
end
def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil)
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
index e6e599e079d..a830f949b21 100644
--- a/lib/gitlab/project_template.rb
+++ b/lib/gitlab/project_template.rb
@@ -36,35 +36,37 @@ module Gitlab
name == other.name && title == other.title
end
- def self.localized_templates_table
- [
- ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'),
- ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'),
- ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'),
- ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift', 'illustrations/logos/swift.svg'),
- ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'),
- ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps.'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'),
- ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'),
- ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby.'), 'https://gitlab.com/pages/gatsby'),
- ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'),
- ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'),
- ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'),
- ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'),
- ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'),
- ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman'),
- ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'),
- ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'),
- ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'),
- ProjectTemplate.new('nfgitbook', 'Netlify/GitBook', _('A GitBook site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfgitbook', 'illustrations/logos/netlify.svg'),
- ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'),
- ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools.'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'),
- ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'),
- ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'),
- ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab.'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management')
- ].freeze
- end
-
class << self
+ # TODO: Review child inheritance of this table (see: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41699#note_430928221)
+ def localized_templates_table
+ [
+ ProjectTemplate.new('rails', 'Ruby on Rails', _('Includes an MVC structure, Gemfile, Rakefile, along with many others, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/rails', 'illustrations/logos/rails.svg'),
+ ProjectTemplate.new('spring', 'Spring', _('Includes an MVC structure, mvnw and pom.xml to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/spring', 'illustrations/logos/spring.svg'),
+ ProjectTemplate.new('express', 'NodeJS Express', _('Includes an MVC structure to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/express', 'illustrations/logos/express.svg'),
+ ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift', 'illustrations/logos/swift.svg'),
+ ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'),
+ ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps.'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'),
+ ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'),
+ ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby.'), 'https://gitlab.com/pages/gatsby'),
+ ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'),
+ ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'),
+ ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'),
+ ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'),
+ ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'),
+ ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman', 'illustrations/logos/middleman.svg'),
+ ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'),
+ ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'),
+ ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'),
+ ProjectTemplate.new('nfplainhtml', 'Netlify/Plain HTML', _('A plain HTML site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfplain-html', 'illustrations/logos/netlify.svg'),
+ ProjectTemplate.new('nfgitbook', 'Netlify/GitBook', _('A GitBook site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfgitbook', 'illustrations/logos/netlify.svg'),
+ ProjectTemplate.new('nfhexo', 'Netlify/Hexo', _('A Hexo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhexo', 'illustrations/logos/netlify.svg'),
+ ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools.'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'),
+ ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'),
+ ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'),
+ ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab.'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management')
+ ].freeze
+ end
+
def all
localized_templates_table
end
diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb
index 6aa3f515ef0..efe07aa8ab2 100644
--- a/lib/gitlab/quick_actions/issuable_actions.rb
+++ b/lib/gitlab/quick_actions/issuable_actions.rb
@@ -145,9 +145,9 @@ module Gitlab
run_label_command(labels: find_labels(labels_param), command: :relabel, updates_key: :label_ids)
end
- desc _('Add a To Do')
- explanation _('Adds a To Do.')
- execution_message _('Added a To Do.')
+ desc _('Add a to do')
+ explanation _('Adds a to do.')
+ execution_message _('Added a to do.')
types Issuable
condition do
quick_action_target.persisted? &&
@@ -157,9 +157,9 @@ module Gitlab
@updates[:todo_event] = 'add'
end
- desc _('Mark To Do as done')
- explanation _('Marks To Do as done.')
- execution_message _('Marked To Do as done.')
+ desc _('Mark to do as done')
+ explanation _('Marks to do as done.')
+ execution_message _('Marked to do as done.')
types Issuable
condition do
quick_action_target.persisted? &&
diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb
index 98db8ff761e..c8c949a9363 100644
--- a/lib/gitlab/quick_actions/merge_request_actions.rb
+++ b/lib/gitlab/quick_actions/merge_request_actions.rb
@@ -121,6 +121,20 @@ module Gitlab
result[:message]
end
end
+
+ desc _('Approve a merge request')
+ explanation _('Approve the current merge request.')
+ types MergeRequest
+ condition do
+ quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user)
+ end
+ command :approve do
+ success = MergeRequests::ApprovalService.new(quick_action_target.project, current_user).execute(quick_action_target)
+
+ next unless success
+
+ @execution_message[:approve] = _('Approved the current merge request.')
+ end
end
def merge_orchestration_service
diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb
index ff5754675e2..010a6b59da5 100644
--- a/lib/gitlab/redis/hll.rb
+++ b/lib/gitlab/redis/hll.rb
@@ -3,15 +3,16 @@
module Gitlab
module Redis
class HLL
+ BATCH_SIZE = 300
KEY_REGEX = %r{\A(\w|-|:)*\{\w*\}(\w|-|:)*\z}.freeze
KeyFormatError = Class.new(StandardError)
def self.count(params)
- self.new.count(params)
+ self.new.count(**params)
end
def self.add(params)
- self.new.add(params)
+ self.new.add(**params)
end
def count(keys:)
@@ -29,17 +30,24 @@ module Gitlab
# 2020-216-{project_action}
# i_{analytics}_dev_ops_score-2020-32
def add(key:, value:, expiry:)
- unless KEY_REGEX.match?(key)
- raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands")
- end
+ validate_key!(key)
Gitlab::Redis::SharedState.with do |redis|
redis.multi do |multi|
- multi.pfadd(key, value)
+ Array.wrap(value).each_slice(BATCH_SIZE) { |batch| multi.pfadd(key, batch) }
+
multi.expire(key, expiry)
end
end
end
+
+ private
+
+ def validate_key!(key)
+ return if KEY_REGEX.match?(key)
+
+ raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands")
+ end
end
end
end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 8e23ac6aca5..693a10a9de3 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -46,6 +46,12 @@ module Gitlab
maven_app_name_regex
end
+ def nuget_version_regex
+ @nuget_version_regex ||= /
+ \A#{_semver_major_minor_patch_regex}(\.\d*)?#{_semver_prerelease_build_regex}\z
+ /x.freeze
+ end
+
def pypi_version_regex
# See the official regex: https://github.com/pypa/packaging/blob/16.7/packaging/version.py#L159
@@ -61,6 +67,39 @@ module Gitlab
)\z}xi.freeze
end
+ def debian_package_name_regex
+ # See official parser
+ # https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n122
+ # @debian_package_name_regex ||= %r{\A[a-z0-9][-+\._a-z0-9]*\z}i.freeze
+ # But we prefer a more strict version from Lintian
+ # https://salsa.debian.org/lintian/lintian/-/blob/5080c0068ffc4a9ddee92022a91d0c2ff53e56d1/lib/Lintian/Util.pm#L116
+ @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z}.freeze
+ end
+
+ def debian_version_regex
+ # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/parsehelp.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n205
+ @debian_version_regex ||= %r{
+ \A(?:
+ (?:([0-9]{1,9}):)? (?# epoch)
+ ([0-9][0-9a-z\.+~-]*) (?# version)
+ (?:(-[0-0a-z\.+~]+))? (?# revision)
+ )\z}xi.freeze
+ end
+
+ def debian_architecture_regex
+ # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43
+ # But we limit to lower case
+ @debian_architecture_regex ||= %r{\A[a-z0-9][-a-z0-9]*\z}.freeze
+ end
+
+ def debian_distribution_regex
+ @debian_distribution_regex ||= %r{\A[a-z0-9][a-z0-9\.-]*\z}i.freeze
+ end
+
+ def debian_component_regex
+ @debian_component_regex ||= %r{#{debian_distribution_regex}}.freeze
+ end
+
def unbounded_semver_regex
# See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
@@ -68,16 +107,34 @@ module Gitlab
# reordered to be greedy. Without this change, the unbounded regex would
# only partially match "v0.0.0-20201230123456-abcdefabcdef".
@unbounded_semver_regex ||= /
+ #{_semver_major_minor_patch_regex}#{_semver_prerelease_build_regex}
+ /x.freeze
+ end
+
+ def semver_regex
+ @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options)
+ end
+
+ # These partial semver regexes are intended for use in composing other
+ # regexes rather than being used alone.
+ def _semver_major_minor_patch_regex
+ @_semver_major_minor_patch_regex ||= /
(?<major>0|[1-9]\d*)
\.(?<minor>0|[1-9]\d*)
\.(?<patch>0|[1-9]\d*)
+ /x.freeze
+ end
+
+ def _semver_prerelease_build_regex
+ @_semver_prerelease_build_regex ||= /
(?:-(?<prerelease>(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0)(?:\.(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0))*))?
(?:\+(?<build>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?
/x.freeze
end
- def semver_regex
- @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options)
+ def prefixed_semver_regex
+ # identical to semver_regex, except starting with 'v'
+ @prefixed_semver_regex ||= Regexp.new("\\Av#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options)
end
def go_package_regex
@@ -103,6 +160,14 @@ module Gitlab
def generic_package_version_regex
/\A\d+\.\d+\.\d+\z/
end
+
+ def generic_package_name_regex
+ maven_file_name_regex
+ end
+
+ def generic_package_file_name_regex
+ generic_package_name_regex
+ end
end
extend self
@@ -211,8 +276,27 @@ module Gitlab
"Must start with a letter, and cannot end with '-'"
end
+ # The section start, e.g. section_start:12345678:NAME
+ def logs_section_prefix_regex
+ /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)/
+ end
+
+ # The optional section options, e.g. [collapsed=true]
+ def logs_section_options_regex
+ /(\[(?:\w+=\w+)(?:, ?(?:\w+=\w+))*\])?/
+ end
+
+ # The region end, always: \r\e\[0K
+ def logs_section_suffix_regex
+ /\r\033\[0K/
+ end
+
def build_trace_section_regex
- @build_trace_section_regexp ||= /section_((?:start)|(?:end)):(\d+):([a-zA-Z0-9_.-]+)\r\033\[0K/.freeze
+ @build_trace_section_regexp ||= %r{
+ #{logs_section_prefix_regex}
+ #{logs_section_options_regex}
+ #{logs_section_suffix_regex}
+ }x.freeze
end
def markdown_code_or_html_blocks
diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb
index cd03a347355..8f5495ece5e 100644
--- a/lib/gitlab/relative_positioning/item_context.rb
+++ b/lib/gitlab/relative_positioning/item_context.rb
@@ -131,12 +131,12 @@ module Gitlab
def shift_left
move_sequence_before(true)
- object.reset
+ object.reset_relative_position
end
def shift_right
move_sequence_after(true)
- object.reset
+ object.reset_relative_position
end
def create_space_left
diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb
index 67e23624045..9ee6f67e455 100644
--- a/lib/gitlab/repo_path.rb
+++ b/lib/gitlab/repo_path.rb
@@ -35,6 +35,10 @@ module Gitlab
snippet, redirected_path = find_snippet(full_path)
[snippet, snippet&.project, redirected_path]
+ elsif type.wiki?
+ wiki, redirected_path = find_wiki(full_path)
+
+ [wiki, wiki.try(:project), redirected_path]
else
project, redirected_path = find_project(full_path)
@@ -67,6 +71,17 @@ module Gitlab
[Snippet.find_by_id_and_project(id: snippet_id, project: project), redirected_path]
end
+ # Wiki path can be either:
+ # - namespace/project
+ # - group/subgroup/project
+ def self.find_wiki(wiki_path)
+ return [nil, nil] if wiki_path.blank?
+
+ project, redirected_path = find_project(wiki_path)
+
+ [project&.wiki, redirected_path]
+ end
+
def self.extract_snippet_info(snippet_path)
path_segments = snippet_path.split('/')
snippet_id = path_segments.pop
diff --git a/lib/gitlab/repository_size_checker.rb b/lib/gitlab/repository_size_checker.rb
index 82c57f7a07d..03d9f961dd9 100644
--- a/lib/gitlab/repository_size_checker.rb
+++ b/lib/gitlab/repository_size_checker.rb
@@ -6,9 +6,10 @@ module Gitlab
attr_reader :limit
# @param current_size_proc [Proc] returns repository size in bytes
- def initialize(current_size_proc:, limit:, enabled: true)
+ def initialize(current_size_proc:, limit:, namespace:, enabled: true)
@current_size_proc = current_size_proc
@limit = limit
+ @namespace = namespace
@enabled = enabled && limit != 0
end
@@ -40,7 +41,13 @@ module Gitlab
end
def error_message
- @error_message_object ||= Gitlab::RepositorySizeErrorMessage.new(self)
+ @error_message_object ||= ::Gitlab::RepositorySizeErrorMessage.new(self)
end
+
+ private
+
+ attr_reader :namespace
end
end
+
+Gitlab::RepositorySizeChecker.prepend_if_ee('EE::Gitlab::RepositorySizeChecker')
diff --git a/lib/gitlab/sample_data_template.rb b/lib/gitlab/sample_data_template.rb
new file mode 100644
index 00000000000..ae74dc710b7
--- /dev/null
+++ b/lib/gitlab/sample_data_template.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+ class SampleDataTemplate < ProjectTemplate
+ class << self
+ def localized_templates_table
+ [
+ SampleDataTemplate.new('basic', 'Basic', _('Basic Sample Data template with Issues, Merge Requests and Milestones.'), 'https://gitlab.com/gitlab-org/sample-data-templates/basic'),
+ SampleDataTemplate.new('serenity_valley', 'Serenity Valley', _('Serenity Valley Sample Data template.'), 'https://gitlab.com/gitlab-org/sample-data-templates/serenity-valley')
+ ].freeze
+ end
+
+ def all
+ localized_templates_table
+ end
+
+ def archive_directory
+ Rails.root.join("vendor/sample_data_templates")
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/search/recent_items.rb b/lib/gitlab/search/recent_items.rb
index 40d96ded275..593148025e1 100644
--- a/lib/gitlab/search/recent_items.rb
+++ b/lib/gitlab/search/recent_items.rb
@@ -6,9 +6,12 @@ module Gitlab
# items. The #type and #finder methods are the only ones needed to be
# implemented by classes inheriting from this.
class RecentItems
- ITEMS_LIMIT = 100
+ ITEMS_LIMIT = 100 # How much history to remember from the user
+ SEARCH_LIMIT = 5 # How many matching items to return from search
EXPIRES_AFTER = 7.days
+ attr_reader :user
+
def initialize(user:, items_limit: ITEMS_LIMIT, expires_after: EXPIRES_AFTER)
@user = user
@items_limit = items_limit
@@ -30,21 +33,25 @@ module Gitlab
end
def search(term)
- ids = with_redis do |redis|
- redis.zrevrange(key, 0, @items_limit - 1)
- end.map(&:to_i)
-
- finder.new(@user, search: term, in: 'title').execute.reorder(nil).id_in_ordered(ids) # rubocop: disable CodeReuse/ActiveRecord
+ finder.new(user, search: term, in: 'title')
+ .execute
+ .limit(SEARCH_LIMIT).reorder(nil).id_in_ordered(latest_ids) # rubocop: disable CodeReuse/ActiveRecord
end
private
+ def latest_ids
+ with_redis do |redis|
+ redis.zrevrange(key, 0, @items_limit - 1)
+ end.map(&:to_i)
+ end
+
def with_redis(&blk)
Gitlab::Redis::SharedState.with(&blk) # rubocop: disable CodeReuse/ActiveRecord
end
def key
- "recent_items:#{type.name.downcase}:#{@user.id}"
+ "recent_items:#{type.name.downcase}:#{user.id}"
end
def type
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 06d8dca2f70..b81264c5d0c 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -7,7 +7,7 @@ module Gitlab
DEFAULT_PAGE = 1
DEFAULT_PER_PAGE = 20
- attr_reader :current_user, :query, :filters
+ attr_reader :current_user, :query, :sort, :filters
# Limit search results by passed projects
# It allows us to search only for projects user has access to
@@ -19,31 +19,23 @@ module Gitlab
# query
attr_reader :default_project_filter
- def initialize(current_user, query, limit_projects = nil, default_project_filter: false, filters: {})
+ def initialize(current_user, query, limit_projects = nil, sort: nil, default_project_filter: false, filters: {})
@current_user = current_user
@query = query
@limit_projects = limit_projects || Project.all
@default_project_filter = default_project_filter
+ @sort = sort
@filters = filters
end
def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true, preload_method: nil)
should_preload = preload_method.present?
- collection = case scope
- when 'projects'
- projects
- when 'issues'
- issues
- when 'merge_requests'
- merge_requests
- when 'milestones'
- milestones
- when 'users'
- users
- else
- should_preload = false
- Kaminari.paginate_array([])
- end
+ collection = collection_for(scope)
+
+ if collection.nil?
+ should_preload = false
+ collection = Kaminari.paginate_array([])
+ end
collection = collection.public_send(preload_method) if should_preload # rubocop:disable GitlabSecurity/PublicSend
collection = collection.page(page).per(per_page)
@@ -116,8 +108,41 @@ module Gitlab
UsersFinder.new(current_user, search: query).execute
end
+ # highlighting is only performed by Elasticsearch backed results
+ def highlight_map(scope)
+ {}
+ end
+
private
+ def collection_for(scope)
+ case scope
+ when 'projects'
+ projects
+ when 'issues'
+ issues
+ when 'merge_requests'
+ merge_requests
+ when 'milestones'
+ milestones
+ when 'users'
+ users
+ end
+ end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def apply_sort(scope)
+ case sort
+ when 'oldest'
+ scope.reorder('created_at ASC')
+ when 'newest'
+ scope.reorder('created_at DESC')
+ else
+ scope
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
def projects
limit_projects.search(query)
end
@@ -129,7 +154,7 @@ module Gitlab
issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord
end
- issues
+ apply_sort(issues)
end
# rubocop: disable CodeReuse/ActiveRecord
@@ -149,7 +174,7 @@ module Gitlab
merge_requests = merge_requests.in_projects(project_ids_relation)
end
- merge_requests
+ apply_sort(merge_requests)
end
def default_scope
@@ -193,6 +218,10 @@ module Gitlab
end
params[:state] = filters[:state] if filters.key?(:state)
+
+ if [true, false].include?(filters[:confidential]) && Feature.enabled?(:search_filter_by_confidential)
+ params[:confidential] = filters[:confidential]
+ end
end
end
diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb
index e26d45e1b33..4158cec9b09 100644
--- a/lib/gitlab/seeder.rb
+++ b/lib/gitlab/seeder.rb
@@ -87,7 +87,9 @@ module Gitlab
SeedFu.quiet = true
- yield
+ without_statement_timeout do
+ yield
+ end
SeedFu.quiet = false
ActiveRecord::Base.logger = old_logger
@@ -114,6 +116,13 @@ module Gitlab
def self.mute_mailer
ActionMailer::MessageDelivery.prepend(DeliverNever)
end
+
+ def self.without_statement_timeout
+ ActiveRecord::Base.connection.execute('SET statement_timeout=0')
+ yield
+ ensure
+ ActiveRecord::Base.connection.execute('RESET statement_timeout')
+ end
end
end
# :nocov:
diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb
index e1a87a77f04..8793a672693 100644
--- a/lib/gitlab/sidekiq_daemon/memory_killer.rb
+++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb
@@ -231,7 +231,7 @@ module Gitlab
def rss_increase_by_jobs
Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do
- Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord
+ Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job|
rss_increase_by_job(job)
end
end
diff --git a/lib/gitlab/sidekiq_middleware/worker_context/server.rb b/lib/gitlab/sidekiq_middleware/worker_context/server.rb
index d2d84742c17..2d8fd8002d2 100644
--- a/lib/gitlab/sidekiq_middleware/worker_context/server.rb
+++ b/lib/gitlab/sidekiq_middleware/worker_context/server.rb
@@ -12,8 +12,10 @@ module Gitlab
# This is not a worker we know about, perhaps from a gem
return yield unless worker_class.respond_to?(:get_worker_context)
- # Use the context defined on the class level as a base context
- wrap_in_optional_context(worker_class.get_worker_context, &block)
+ Gitlab::ApplicationContext.with_context(feature_category: worker_class.get_feature_category.to_s) do
+ # Use the context defined on the class level as the more specific context
+ wrap_in_optional_context(worker_class.get_worker_context, &block)
+ end
end
end
end
diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb
index 2d8df2ca204..714ca77c3e5 100644
--- a/lib/gitlab/slash_commands/presenters/help.rb
+++ b/lib/gitlab/slash_commands/presenters/help.rb
@@ -48,7 +48,7 @@ module Gitlab
*Documentation*
For more information about GitLab chatops, refer to its
- documentation: https://docs.gitlab.com/ce/ci/chatops/README.html.
+ documentation: https://docs.gitlab.com/ee/ci/chatops/README.html.
MESSAGE
message
diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb
index f647c85e1c8..315c603c1dd 100644
--- a/lib/gitlab/static_site_editor/config/file_config.rb
+++ b/lib/gitlab/static_site_editor/config/file_config.rb
@@ -3,11 +3,38 @@
module Gitlab
module StaticSiteEditor
module Config
+ #
+ # Base GitLab Static Site Editor Configuration facade
+ #
class FileConfig
- def data
- {
- static_site_generator: 'middleman'
- }
+ ConfigError = Class.new(StandardError)
+
+ def initialize(yaml)
+ content_hash = content_hash(yaml)
+ @global = Entry::Global.new(content_hash)
+ @global.compose!
+ rescue Gitlab::Config::Loader::FormatError => e
+ raise FileConfig::ConfigError, e.message
+ end
+
+ def valid?
+ @global.valid?
+ end
+
+ def errors
+ @global.errors
+ end
+
+ def to_hash_with_defaults
+ # NOTE: The current approach of simply mapping all the descendents' keys and values ('config')
+ # into a flat hash may need to be enhanced as we add more complex, non-scalar entries.
+ @global.descendants.map { |descendant| [descendant.key, descendant.config] }.to_h
+ end
+
+ private
+
+ def content_hash(yaml)
+ Gitlab::Config::Loader::Yaml.new(yaml).load!
end
end
end
diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/global.rb b/lib/gitlab/static_site_editor/config/file_config/entry/global.rb
new file mode 100644
index 00000000000..c295ccf1d11
--- /dev/null
+++ b/lib/gitlab/static_site_editor/config/file_config/entry/global.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module StaticSiteEditor
+ module Config
+ class FileConfig
+ module Entry
+ ##
+ # This class represents a global entry - root Entry for entire
+ # GitLab StaticSiteEditor Configuration file.
+ #
+ class Global < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Configurable
+ include ::Gitlab::Config::Entry::Attributable
+
+ ALLOWED_KEYS = %i[
+ image_upload_path
+ mounts
+ static_site_generator
+ ].freeze
+
+ attributes ALLOWED_KEYS
+
+ validations do
+ validates :config, allowed_keys: ALLOWED_KEYS
+ end
+
+ entry :image_upload_path, Entry::ImageUploadPath,
+ description: 'Configuration of the Static Site Editor image upload path.'
+ entry :mounts, Entry::Mounts,
+ description: 'Configuration of the Static Site Editor mounts.'
+ entry :static_site_generator, Entry::StaticSiteGenerator,
+ description: 'Configuration of the Static Site Editor static site generator.'
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb b/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb
new file mode 100644
index 00000000000..6a2b9e10d33
--- /dev/null
+++ b/lib/gitlab/static_site_editor/config/file_config/entry/image_upload_path.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module StaticSiteEditor
+ module Config
+ class FileConfig
+ module Entry
+ ##
+ # Entry that represents the path to which images will be uploaded
+ #
+ class ImageUploadPath < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+
+ validations do
+ validates :config, type: String
+ end
+
+ def self.default
+ 'source/images'
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb
new file mode 100644
index 00000000000..b10956e17a5
--- /dev/null
+++ b/lib/gitlab/static_site_editor/config/file_config/entry/mount.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module StaticSiteEditor
+ module Config
+ class FileConfig
+ module Entry
+ ##
+ # Entry that represents the mappings of mounted source directories to target paths
+ #
+ class Mount < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+ include ::Gitlab::Config::Entry::Attributable
+
+ ALLOWED_KEYS = %i[source target].freeze
+
+ attributes ALLOWED_KEYS
+
+ validations do
+ validates :config, allowed_keys: ALLOWED_KEYS
+
+ validates :source, type: String, presence: true
+ validates :target, type: String, presence: true, allow_blank: true
+ end
+
+ def self.default
+ # NOTE: This is the default for middleman projects. Ideally, this would be determined
+ # based on the defaults for whatever `static_site_generator` is configured.
+ {
+ source: 'source',
+ target: ''
+ }
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb b/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb
new file mode 100644
index 00000000000..10bd377e419
--- /dev/null
+++ b/lib/gitlab/static_site_editor/config/file_config/entry/mounts.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module StaticSiteEditor
+ module Config
+ class FileConfig
+ module Entry
+ ##
+ # Entry that represents the mappings of mounted source directories to target paths
+ #
+ class Mounts < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Configurable
+ include ::Gitlab::Config::Entry::Validatable
+
+ entry :mount, Entry::Mount, description: 'Configuration of a Static Site Editor mount.'
+
+ validations do
+ validates :config, type: Array, presence: true
+ end
+
+ def skip_config_hash_validation?
+ true
+ end
+
+ def self.default
+ [Entry::Mount.default]
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb b/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb
new file mode 100644
index 00000000000..593c0951f93
--- /dev/null
+++ b/lib/gitlab/static_site_editor/config/file_config/entry/static_site_generator.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module StaticSiteEditor
+ module Config
+ class FileConfig
+ module Entry
+ ##
+ # Entry that represents the static site generator tool/framework.
+ #
+ class StaticSiteGenerator < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+
+ validations do
+ validates :config, type: String, inclusion: { in: %w[middleman], message: "should be 'middleman'" }
+ end
+
+ def self.default
+ 'middleman'
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/static_site_editor/config/generated_config.rb b/lib/gitlab/static_site_editor/config/generated_config.rb
index f3dce74a32f..ff24ec69ab0 100644
--- a/lib/gitlab/static_site_editor/config/generated_config.rb
+++ b/lib/gitlab/static_site_editor/config/generated_config.rb
@@ -4,8 +4,6 @@ module Gitlab
module StaticSiteEditor
module Config
class GeneratedConfig
- SUPPORTED_EXTENSIONS = %w[.md].freeze
-
def initialize(repository, ref, path, return_url)
@repository = repository
@ref = ref
@@ -23,7 +21,7 @@ module Gitlab
project: project.path,
namespace: project.namespace.full_path,
return_url: sanitize_url(return_url),
- is_supported_content: supported_content?.to_s,
+ is_supported_content: supported_content?,
base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path),
merge_requests_illustration_path: merge_requests_illustration_path
}
@@ -35,8 +33,12 @@ module Gitlab
delegate :project, to: :repository
+ def supported_extensions
+ %w[.md].freeze
+ end
+
def commit_id
- repository.commit(ref)&.id if ref
+ repository.commit(ref)&.id
end
def supported_content?
@@ -50,7 +52,7 @@ module Gitlab
def extension_supported?
return true if path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project)
- SUPPORTED_EXTENSIONS.any? { |ext| path.end_with?(ext) }
+ supported_extensions.any? { |ext| path.end_with?(ext) }
end
def file_exists?
diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb
new file mode 100644
index 00000000000..a918e7bec80
--- /dev/null
+++ b/lib/gitlab/subscription_portal.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SubscriptionPortal
+ def self.default_subscriptions_url
+ ::Gitlab.dev_or_test_env? ? 'https://customers.stg.gitlab.com' : 'https://customers.gitlab.com'
+ end
+
+ SUBSCRIPTIONS_URL = ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url).freeze
+ end
+end
diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb
index bb1e9db55fa..e12af6bf0a4 100644
--- a/lib/gitlab/template/gitlab_ci_yml_template.rb
+++ b/lib/gitlab/template/gitlab_ci_yml_template.rb
@@ -3,7 +3,7 @@
module Gitlab
module Template
class GitlabCiYmlTemplate < BaseTemplate
- BASE_EXCLUDED_PATTERNS = [%r{\.latest$}].freeze
+ BASE_EXCLUDED_PATTERNS = [%r{\.latest\.}].freeze
def content
explanation = "# This file is a template, and might need editing before it works on your project."
diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 72783a2d682..16e7b8a7eca 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -10,21 +10,21 @@ module Gitlab
APPLICATION_DEFAULT = 1
# Struct class representing a single Theme
- Theme = Struct.new(:id, :name, :css_class)
+ Theme = Struct.new(:id, :name, :css_class, :css_filename)
# All available Themes
THEMES = [
- Theme.new(1, 'Indigo', 'ui-indigo'),
- Theme.new(6, 'Light Indigo', 'ui-light-indigo'),
- Theme.new(4, 'Blue', 'ui-blue'),
- Theme.new(7, 'Light Blue', 'ui-light-blue'),
- Theme.new(5, 'Green', 'ui-green'),
- Theme.new(8, 'Light Green', 'ui-light-green'),
- Theme.new(9, 'Red', 'ui-red'),
- Theme.new(10, 'Light Red', 'ui-light-red'),
- Theme.new(2, 'Dark', 'ui-dark'),
- Theme.new(3, 'Light', 'ui-light'),
- Theme.new(11, 'Dark Mode (alpha)', 'gl-dark')
+ Theme.new(1, 'Indigo', 'ui-indigo', 'theme_indigo'),
+ Theme.new(6, 'Light Indigo', 'ui-light-indigo', 'theme_light_indigo'),
+ Theme.new(4, 'Blue', 'ui-blue', 'theme_blue'),
+ Theme.new(7, 'Light Blue', 'ui-light-blue', 'theme_light_blue'),
+ Theme.new(5, 'Green', 'ui-green', 'theme_green'),
+ Theme.new(8, 'Light Green', 'ui-light-green', 'theme_light_green'),
+ Theme.new(9, 'Red', 'ui-red', 'theme_red'),
+ Theme.new(10, 'Light Red', 'ui-light-red', 'theme_light_red'),
+ Theme.new(2, 'Dark', 'ui-dark', 'theme_dark'),
+ Theme.new(3, 'Light', 'ui-light', 'theme_light'),
+ Theme.new(11, 'Dark Mode (alpha)', 'gl-dark', nil)
].freeze
# Convenience method to get a space-separated String of all the theme
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 89605ce5d07..68f24559b1f 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -12,6 +12,19 @@
# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
module Gitlab
class UsageData
+ CE_MEMOIZED_VALUES = %i(
+ issue_minimum_id
+ issue_maximum_id
+ project_minimum_id
+ project_maximum_id
+ user_minimum_id
+ user_maximum_id
+ unique_visit_service
+ deployment_minimum_id
+ deployment_maximum_id
+ auth_providers
+ ).freeze
+
class << self
include Gitlab::Utils::UsageData
include Gitlab::Utils::StrongMemoize
@@ -138,8 +151,10 @@ module Gitlab
pages_domains: count(PagesDomain),
pool_repositories: count(PoolRepository),
projects: count(Project),
+ projects_creating_incidents: distinct_count(Issue.incident, :project_id),
projects_imported_from_github: count(Project.where(import_type: 'github')),
projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)),
+ projects_with_tracing_enabled: count(ProjectTracingSetting),
projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
projects_with_alerts_service_enabled: count(AlertsService.active),
projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id),
@@ -166,8 +181,7 @@ module Gitlab
user_preferences_usage,
ingress_modsecurity_usage,
container_expiration_policies_usage,
- service_desk_counts,
- snowplow_event_counts
+ service_desk_counts
).tap do |data|
data[:snippets] = data[:personal_snippets] + data[:project_snippets]
end
@@ -175,7 +189,7 @@ module Gitlab
end
# rubocop: enable Metrics/AbcSize
- def snowplow_event_counts(time_period: {})
+ def snowplow_event_counts(time_period)
return {} unless report_snowplow_events?
{
@@ -200,7 +214,7 @@ module Gitlab
personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)),
project_snippets: count(ProjectSnippet.where(last_28_days_time_period))
}.merge(
- snowplow_event_counts(time_period: last_28_days_time_period(column: :collector_tstamp))
+ snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp))
).tap do |data|
data[:snippets] = data[:personal_snippets] + data[:project_snippets]
end
@@ -242,7 +256,8 @@ module Gitlab
signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? },
web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? },
ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity),
- grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }
+ grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? },
+ gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? }
}
end
@@ -264,7 +279,8 @@ module Gitlab
Gitlab::UsageDataCounters::SourceCodeCounter,
Gitlab::UsageDataCounters::MergeRequestCounter,
Gitlab::UsageDataCounters::DesignsCounter,
- Gitlab::UsageDataCounters::KubernetesAgentCounter
+ Gitlab::UsageDataCounters::KubernetesAgentCounter,
+ Gitlab::UsageDataCounters::StaticSiteEditorCounter
]
end
@@ -287,7 +303,8 @@ module Gitlab
},
database: {
adapter: alt_usage_data { Gitlab::Database.adapter_name },
- version: alt_usage_data { Gitlab::Database.version }
+ version: alt_usage_data { Gitlab::Database.version },
+ pg_system_id: alt_usage_data { Gitlab::Database.system_id }
},
mail: {
smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] }
@@ -385,10 +402,12 @@ module Gitlab
def services_usage
# rubocop: disable UsageData/LargeTable:
Service.available_services_names.each_with_object({}) do |service_name, response|
- response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, instance: false, type: "#{service_name}_service".camelize))
+ response["projects_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where(type: "#{service_name}_service".camelize))
+ response["groups_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where(type: "#{service_name}_service".camelize))
response["templates_#{service_name}_active".to_sym] = count(Service.active.where(template: true, type: "#{service_name}_service".camelize))
response["instances_#{service_name}_active".to_sym] = count(Service.active.where(instance: true, type: "#{service_name}_service".camelize))
- response["projects_inheriting_instance_#{service_name}_active".to_sym] = count(Service.active.where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize))
+ response["projects_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize))
+ response["groups_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize))
end.merge(jira_usage, jira_import_usage)
# rubocop: enable UsageData/LargeTable:
end
@@ -444,8 +463,11 @@ module Gitlab
# rubocop: enable UsageData/LargeTable
# rubocop: enable CodeReuse/ActiveRecord
+ # augmented in EE
def user_preferences_usage
- {} # augmented in EE
+ {
+ user_preferences_user_gitpod_enabled: count(UserPreference.with_user.gitpod_enabled.merge(User.active))
+ }
end
def merge_requests_users(time_period)
@@ -469,7 +491,7 @@ module Gitlab
end
def last_28_days_time_period(column: :created_at)
- { column => 28.days.ago..Time.current }
+ { column => 30.days.ago..2.days.ago }
end
# Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv
@@ -541,6 +563,7 @@ module Gitlab
groups: distinct_count(::GroupMember.where(time_period), :user_id),
users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id),
omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
+ user_auth_by_provider: distinct_count_user_auth_by_provider(time_period),
projects_imported: {
gitlab_project: projects_imported_count('gitlab_project', time_period),
gitlab: projects_imported_count('gitlab', time_period),
@@ -555,7 +578,8 @@ module Gitlab
jira: distinct_count(::JiraImportState.where(time_period), :user_id),
fogbugz: projects_imported_count('fogbugz', time_period),
phabricator: projects_imported_count('phabricator', time_period)
- }
+ },
+ groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id)
}
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -567,7 +591,8 @@ module Gitlab
clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period),
operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
start: user_minimum_id,
- finish: user_maximum_id)
+ finish: user_maximum_id),
+ projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id)
}
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -696,10 +721,10 @@ module Gitlab
counter = Gitlab::UsageDataCounters::EditorUniqueCounter
{
- action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(date_range) },
- action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(date_range) },
- action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(date_range) },
- action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(date_range) }
+ action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) },
+ action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) },
+ action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) },
+ action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) }
}
end
@@ -801,17 +826,7 @@ module Gitlab
end
def clear_memoized
- clear_memoization(:issue_minimum_id)
- clear_memoization(:issue_maximum_id)
- clear_memoization(:user_minimum_id)
- clear_memoization(:user_maximum_id)
- clear_memoization(:unique_visit_service)
- clear_memoization(:deployment_minimum_id)
- clear_memoization(:deployment_maximum_id)
- clear_memoization(:approval_merge_request_rule_minimum_id)
- clear_memoization(:approval_merge_request_rule_maximum_id)
- clear_memoization(:project_minimum_id)
- clear_memoization(:project_maximum_id)
+ CE_MEMOIZED_VALUES.each { |v| clear_memoization(v) }
end
# rubocop: disable CodeReuse/ActiveRecord
@@ -843,6 +858,39 @@ module Gitlab
def projects_imported_count(from, time_period)
distinct_count(::Project.imported_from(from).where(time_period), :creator_id) # rubocop: disable CodeReuse/ActiveRecord
end
+
+ # rubocop:disable CodeReuse/ActiveRecord
+ def distinct_count_user_auth_by_provider(time_period)
+ counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash|
+ hash[provider] = distinct_count(
+ ::AuthenticationEvent.success.for_provider(provider).where(time_period), :user_id)
+ end
+
+ if any_ldap_auth_providers?
+ counts['ldap'] = distinct_count(
+ ::AuthenticationEvent.success.ldap.where(time_period), :user_id
+ )
+ end
+
+ counts
+ end
+ # rubocop:enable CodeReuse/ActiveRecord
+
+ # rubocop:disable UsageData/LargeTable
+ def auth_providers
+ strong_memoize(:auth_providers) do
+ ::AuthenticationEvent.providers
+ end
+ end
+ # rubocop:enable UsageData/LargeTable
+
+ def auth_providers_except_ldap
+ auth_providers.reject { |provider| provider.starts_with?('ldap') }
+ end
+
+ def any_ldap_auth_providers?
+ auth_providers.any? { |provider| provider.starts_with?('ldap') }
+ end
end
end
end
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
index 53bf6daea4c..eb132ef0967 100644
--- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -72,7 +72,8 @@ module Gitlab
events_names = events_for_category(category)
event_results = events_names.each_with_object({}) do |event, hash|
- hash[event] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current)
+ hash["#{event}_weekly"] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current)
+ hash["#{event}_monthly"] = unique_events(event_names: event, start_date: 4.weeks.ago.to_date, end_date: Date.current)
end
if eligible_for_totals?(events_names)
diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
index fc1b5a59487..e8839875109 100644
--- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
@@ -3,14 +3,42 @@
module Gitlab
module UsageDataCounters
module IssueActivityUniqueCounter
- ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed'
- ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed'
+ ISSUE_CATEGORY = 'issues_edit'
+
ISSUE_ASSIGNEE_CHANGED = 'g_project_management_issue_assignee_changed'
+ ISSUE_CREATED = 'g_project_management_issue_created'
+ ISSUE_CLOSED = 'g_project_management_issue_closed'
+ ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed'
+ ISSUE_ITERATION_CHANGED = 'g_project_management_issue_iteration_changed'
+ ISSUE_LABEL_CHANGED = 'g_project_management_issue_label_changed'
ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential'
ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible'
- ISSUE_CATEGORY = 'issues_edit'
+ ISSUE_MILESTONE_CHANGED = 'g_project_management_issue_milestone_changed'
+ ISSUE_REOPENED = 'g_project_management_issue_reopened'
+ ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed'
+ ISSUE_WEIGHT_CHANGED = 'g_project_management_issue_weight_changed'
+ ISSUE_CROSS_REFERENCED = 'g_project_management_issue_cross_referenced'
+ ISSUE_MOVED = 'g_project_management_issue_moved'
+ ISSUE_RELATED = 'g_project_management_issue_related'
+ ISSUE_UNRELATED = 'g_project_management_issue_unrelated'
+ ISSUE_MARKED_AS_DUPLICATE = 'g_project_management_issue_marked_as_duplicate'
+ ISSUE_LOCKED = 'g_project_management_issue_locked'
+ ISSUE_UNLOCKED = 'g_project_management_issue_unlocked'
+ ISSUE_ADDED_TO_EPIC = 'g_project_management_issue_added_to_epic'
+ ISSUE_REMOVED_FROM_EPIC = 'g_project_management_issue_removed_from_epic'
+ ISSUE_CHANGED_EPIC = 'g_project_management_issue_changed_epic'
+ ISSUE_DESIGNS_ADDED = 'g_project_management_issue_designs_added'
+ ISSUE_DESIGNS_MODIFIED = 'g_project_management_issue_designs_modified'
+ ISSUE_DESIGNS_REMOVED = 'g_project_management_issue_designs_removed'
+ ISSUE_DUE_DATE_CHANGED = 'g_project_management_issue_due_date_changed'
+ ISSUE_TIME_ESTIMATE_CHANGED = 'g_project_management_issue_time_estimate_changed'
+ ISSUE_TIME_SPENT_CHANGED = 'g_project_management_issue_time_spent_changed'
class << self
+ def track_issue_created_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_CREATED, author, time)
+ end
+
def track_issue_title_changed_action(author:, time: Time.zone.now)
track_unique_action(ISSUE_TITLE_CHANGED, author, time)
end
@@ -31,10 +59,98 @@ module Gitlab
track_unique_action(ISSUE_MADE_VISIBLE, author, time)
end
+ def track_issue_closed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_CLOSED, author, time)
+ end
+
+ def track_issue_reopened_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_REOPENED, author, time)
+ end
+
+ def track_issue_label_changed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_LABEL_CHANGED, author, time)
+ end
+
+ def track_issue_milestone_changed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_MILESTONE_CHANGED, author, time)
+ end
+
+ def track_issue_iteration_changed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_ITERATION_CHANGED, author, time)
+ end
+
+ def track_issue_weight_changed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_WEIGHT_CHANGED, author, time)
+ end
+
+ def track_issue_cross_referenced_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_CROSS_REFERENCED, author, time)
+ end
+
+ def track_issue_moved_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_MOVED, author, time)
+ end
+
+ def track_issue_related_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_RELATED, author, time)
+ end
+
+ def track_issue_unrelated_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_UNRELATED, author, time)
+ end
+
+ def track_issue_marked_as_duplicate_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author, time)
+ end
+
+ def track_issue_locked_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_LOCKED, author, time)
+ end
+
+ def track_issue_unlocked_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_UNLOCKED, author, time)
+ end
+
+ def track_issue_added_to_epic_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_ADDED_TO_EPIC, author, time)
+ end
+
+ def track_issue_removed_from_epic_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_REMOVED_FROM_EPIC, author, time)
+ end
+
+ def track_issue_changed_epic_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_CHANGED_EPIC, author, time)
+ end
+
+ def track_issue_designs_added_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_DESIGNS_ADDED, author, time)
+ end
+
+ def track_issue_designs_modified_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_DESIGNS_MODIFIED, author, time)
+ end
+
+ def track_issue_designs_removed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_DESIGNS_REMOVED, author, time)
+ end
+
+ def track_issue_due_date_changed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_DUE_DATE_CHANGED, author, time)
+ end
+
+ def track_issue_time_estimate_changed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author, time)
+ end
+
+ def track_issue_time_spent_changed_action(author:, time: Time.zone.now)
+ track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time)
+ end
+
private
def track_unique_action(action, author, time)
- return unless Feature.enabled?(:track_issue_activity_actions)
+ return unless Feature.enabled?(:track_issue_activity_actions, default_enabled: true)
return unless author
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time)
diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml
index 25e7f858bb1..bc56c5d6d9b 100644
--- a/lib/gitlab/usage_data_counters/known_events.yml
+++ b/lib/gitlab/usage_data_counters/known_events.yml
@@ -185,6 +185,11 @@
redis_slot: incident_management
category: incident_management
aggregation: weekly
+# Testing category
+- name: i_testing_test_case_parsed
+ category: testing
+ redis_slot: testing
+ aggregation: weekly
# Project Management group
- name: g_project_management_issue_title_changed
category: issues_edit
@@ -206,3 +211,95 @@
category: issues_edit
redis_slot: project_management
aggregation: daily
+- name: g_project_management_issue_created
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_closed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_reopened
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_label_changed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_milestone_changed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_iteration_changed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_weight_changed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_cross_referenced
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_moved
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_related
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_unrelated
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_marked_as_duplicate
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_locked
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_unlocked
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_added_to_epic
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_removed_from_epic
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_changed_epic
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_designs_added
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_designs_modified
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_designs_removed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_due_date_changed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_time_estimate_changed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
+- name: g_project_management_issue_time_spent_changed
+ category: issues_edit
+ redis_slot: project_management
+ aggregation: daily
diff --git a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb
new file mode 100644
index 00000000000..8886a106da8
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module UsageDataCounters
+ class StaticSiteEditorCounter < BaseCounter
+ KNOWN_EVENTS = %w[views].freeze
+ PREFIX = 'static_site_editor'
+
+ class << self
+ def increment_views_count
+ count(:views)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb
index bacd63ab282..c54e766230e 100644
--- a/lib/gitlab/usage_data_queries.rb
+++ b/lib/gitlab/usage_data_queries.rb
@@ -22,7 +22,7 @@ module Gitlab
end
def sum(relation, column, *rest)
- relation.select(relation.all.table[column].sum).to_sql # rubocop:disable CodeReuse/ActiveRecord
+ relation.select(relation.all.table[column].sum).to_sql
end
private
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb
index ca6a36c9cea..5267733d220 100644
--- a/lib/gitlab/utils/usage_data.rb
+++ b/lib/gitlab/utils/usage_data.rb
@@ -39,9 +39,9 @@ module Gitlab
FALLBACK = -1
- def count(relation, column = nil, batch: true, start: nil, finish: nil)
+ def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
if batch
- Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish)
+ Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish)
else
relation.count
end
@@ -106,7 +106,6 @@ module Gitlab
# @param values [Array|String] the values counted
def track_usage_event(event_name, values)
return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true)
- return unless Gitlab::CurrentSettings.usage_ping_enabled?
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name.to_s)
end
diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb
index 302697ff8eb..5768bab03c6 100644
--- a/lib/gitlab/view/presenter/factory.rb
+++ b/lib/gitlab/view/presenter/factory.rb
@@ -10,7 +10,7 @@ module Gitlab
end
def fabricate!
- presenter_class.new(subject, attributes)
+ presenter_class.new(subject, **attributes)
end
private
diff --git a/lib/gitlab/visibility_level_checker.rb b/lib/gitlab/visibility_level_checker.rb
index f15f1486a4e..3ffd86c4f8c 100644
--- a/lib/gitlab/visibility_level_checker.rb
+++ b/lib/gitlab/visibility_level_checker.rb
@@ -3,7 +3,7 @@
# Gitlab::VisibilityLevelChecker verifies that:
# - Current @project.visibility_level is not restricted
# - Override visibility param is not restricted
-# - @see https://docs.gitlab.com/ce/api/project_import_export.html#import-a-file
+# - @see https://docs.gitlab.com/ee/api/project_import_export.html#import-a-file
#
# @param current_user [User] Current user object to verify visibility level against
# @param project [Project] Current project that is being created/imported
diff --git a/lib/gitlab/webpack/dev_server_middleware.rb b/lib/gitlab/webpack/dev_server_middleware.rb
index fda41da5a94..069e68e8d29 100644
--- a/lib/gitlab/webpack/dev_server_middleware.rb
+++ b/lib/gitlab/webpack/dev_server_middleware.rb
@@ -11,8 +11,15 @@ module Gitlab
@proxy_host = opts.fetch(:proxy_host, 'localhost')
@proxy_port = opts.fetch(:proxy_port, 3808)
@proxy_path = opts[:proxy_path] if opts[:proxy_path]
+ @proxy_scheme = opts[:proxy_https] ? 'https' : 'http'
- super(app, backend: "http://#{@proxy_host}:#{@proxy_port}", **opts)
+ super(app, backend: "#{@proxy_scheme}://#{@proxy_host}:#{@proxy_port}", **opts)
+ end
+
+ # disable SSL check since any cert used here will likely be self-signed
+ def rewrite_env(env)
+ env["rack.ssl_verify_none"] = true
+ env
end
def perform_request(env)
diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb
index d2c01bbd55e..5873d9c2b99 100644
--- a/lib/gitlab/webpack/manifest.rb
+++ b/lib/gitlab/webpack/manifest.rb
@@ -1,16 +1,33 @@
# frozen_string_literal: true
-require 'webpack/rails/manifest'
+require 'net/http'
+require 'uri'
module Gitlab
module Webpack
- class Manifest < ::Webpack::Rails::Manifest
- # Raised if a supplied asset does not exist in the webpack manifest
+ class Manifest
+ # Raised if we can't read our webpack manifest for whatever reason
+ class ManifestLoadError < StandardError
+ def initialize(message, orig)
+ super "#{message}\n\n(original error #{orig.class.name}: #{orig})"
+ end
+ end
+
+ # Raised if webpack couldn't build one of your entry points
+ class WebpackError < StandardError
+ def initialize(errors)
+ super "Error in webpack compile, details follow below:\n#{errors.join("\n\n")}"
+ end
+ end
+
+ # Raised if a supplied entry point does not exist in the webpack manifest
AssetMissingError = Class.new(StandardError)
class << self
+ include Gitlab::Utils::StrongMemoize
+
def entrypoint_paths(source)
- raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled?
+ raise WebpackError, manifest["errors"] unless manifest_bundled?
dll_assets = manifest.fetch("dllAssets", [])
entrypoint = manifest["entrypoints"][source]
@@ -18,12 +35,93 @@ module Gitlab
# Can be either a string or an array of strings.
# Do not include source maps as they are not javascript
[dll_assets, entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p|
- "/#{::Rails.configuration.webpack.public_path}/#{p}"
+ "/#{Gitlab.config.webpack.public_path}/#{p}"
+ end
+ else
+ raise AssetMissingError, "Can't find asset '#{source}' in webpack manifest"
+ end
+ end
+
+ def asset_paths(source)
+ raise WebpackError, manifest["errors"] unless manifest_bundled?
+
+ paths = manifest["assetsByChunkName"][source]
+ if paths
+ # Can be either a string or an array of strings.
+ # Do not include source maps as they are not javascript
+ [paths].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p|
+ "/#{Gitlab.config.webpack.public_path}/#{p}"
end
else
raise AssetMissingError, "Can't find entry point '#{source}' in webpack manifest"
end
end
+
+ def clear_manifest!
+ clear_memoization(:manifest)
+ end
+
+ private
+
+ def manifest_bundled?
+ !manifest["errors"].any? { |error| error.include? "Module build failed" }
+ end
+
+ def manifest
+ if Gitlab.config.webpack.dev_server.enabled
+ # Don't cache if we're in dev server mode, manifest may change ...
+ load_manifest
+ else
+ # ... otherwise cache at class level, as JSON loading/parsing can be expensive
+ strong_memoize(:manifest) { load_manifest }
+ end
+ end
+
+ def load_manifest
+ data = if Gitlab.config.webpack.dev_server.enabled
+ load_dev_server_manifest
+ else
+ load_static_manifest
+ end
+
+ Gitlab::Json.parse(data)
+ end
+
+ def load_dev_server_manifest
+ host = Gitlab.config.webpack.dev_server.host
+ port = Gitlab.config.webpack.dev_server.port
+ scheme = Gitlab.config.webpack.dev_server.https ? 'https' : 'http'
+ uri = Addressable::URI.new(scheme: scheme, host: host, port: port, path: dev_server_path)
+
+ # localhost could be blocked via Gitlab::HTTP
+ response = HTTParty.get(uri.to_s, verify: false) # rubocop:disable Gitlab/HTTParty
+
+ return response.body if response.code == 200
+
+ raise "HTTP error #{response.code}"
+ rescue OpenSSL::SSL::SSLError, EOFError => e
+ ssl_status = Gitlab.config.webpack.dev_server.https ? ' over SSL' : ''
+ raise ManifestLoadError.new("Could not connect to webpack-dev-server at #{uri}#{ssl_status}.\n\nIs SSL enabled? Check that settings in `gitlab.yml` and webpack-dev-server match.", e)
+ rescue => e
+ raise ManifestLoadError.new("Could not load manifest from webpack-dev-server at #{uri}.\n\nIs webpack-dev-server running? Try running `gdk status webpack` or `gdk tail webpack`.", e)
+ end
+
+ def load_static_manifest
+ File.read(static_manifest_path)
+ rescue => e
+ raise ManifestLoadError.new("Could not load compiled manifest from #{static_manifest_path}.\n\nHave you run `rake gitlab:assets:compile`?", e)
+ end
+
+ def static_manifest_path
+ ::Rails.root.join(
+ Gitlab.config.webpack.output_dir,
+ Gitlab.config.webpack.manifest_filename
+ )
+ end
+
+ def dev_server_path
+ "/#{Gitlab.config.webpack.public_path}/#{Gitlab.config.webpack.manifest_filename}"
+ end
end
end
end
diff --git a/lib/gitlab/whats_new.rb b/lib/gitlab/whats_new.rb
new file mode 100644
index 00000000000..e95ace2c475
--- /dev/null
+++ b/lib/gitlab/whats_new.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module WhatsNew
+ CACHE_DURATION = 1.day
+ WHATS_NEW_FILES_PATH = Rails.root.join('data', 'whats_new', '*.yml')
+
+ private
+
+ def whats_new_most_recent_release_items
+ Rails.cache.fetch('whats_new:release_items', expires_in: CACHE_DURATION) do
+ file = File.read(most_recent_release_file_path)
+
+ items = YAML.safe_load(file, permitted_classes: [Date])
+
+ items if items.is_a?(Array)
+ end
+ rescue => e
+ Gitlab::ErrorTracking.track_exception(e, yaml_file_path: most_recent_release_file_path)
+
+ nil
+ end
+
+ def most_recent_release_file_path
+ @most_recent_release_file_path ||= Dir.glob(WHATS_NEW_FILES_PATH).max
+ end
+ end
+end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 8a5acd242d9..eb780a2f7f6 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -269,7 +269,8 @@ module Gitlab
commit_id: metadata['CommitId'],
prefix: metadata['ArchivePrefix'],
format: format,
- path: path.presence || ""
+ path: path.presence || "",
+ include_lfs_blobs: Feature.enabled?(:include_lfs_blobs_in_archive)
).to_proto
)
}
diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb
index a906beda80e..bbb64e0d5da 100644
--- a/lib/gitlab_danger.rb
+++ b/lib/gitlab_danger.rb
@@ -11,7 +11,7 @@ class GitlabDanger
karma
database
commit_messages
- telemetry
+ product_analytics
utility_css
pajamas
].freeze
@@ -24,6 +24,7 @@ class GitlabDanger
ce_ee_vue_templates
sidekiq_queues
specialization_labels
+ ci_templates
].freeze
MESSAGE_PREFIX = '==>'.freeze
diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb
index 9668badc757..f16bd7c735b 100644
--- a/lib/google_api/cloud_platform/client.rb
+++ b/lib/google_api/cloud_platform/client.rb
@@ -66,7 +66,7 @@ module GoogleApi
cluster_options = make_cluster_options(cluster_name, cluster_size, machine_type, legacy_abac, enable_addons)
- request_body = Google::Apis::ContainerV1beta1::CreateClusterRequest.new(cluster_options)
+ request_body = Google::Apis::ContainerV1beta1::CreateClusterRequest.new(**cluster_options)
service.create_cluster(project_id, zone, request_body, options: user_agent_header)
end
diff --git a/lib/grafana/client.rb b/lib/grafana/client.rb
index b419f79bace..7c0e56b61c8 100644
--- a/lib/grafana/client.rb
+++ b/lib/grafana/client.rb
@@ -19,8 +19,8 @@ module Grafana
# @param name [String] Unique identifier for a Grafana datasource
def get_datasource(name:)
# CGI#escape formats strings such that the Grafana endpoint
- # will not recognize the dashboard name. Preferring URI#escape.
- http_get("#{@api_url}/api/datasources/name/#{URI.escape(name)}") # rubocop:disable Lint/UriEscapeUnescape
+ # will not recognize the dashboard name. Prefer Addressable::URI#encode_component.
+ http_get("#{@api_url}/api/datasources/name/#{Addressable::URI.encode_component(name)}")
end
# @param datasource_id [String] Grafana ID for the datasource
diff --git a/lib/pager_duty/validator/schemas/message.json b/lib/pager_duty/validator/schemas/message.json
new file mode 100644
index 00000000000..b1a3185cd1a
--- /dev/null
+++ b/lib/pager_duty/validator/schemas/message.json
@@ -0,0 +1,47 @@
+{
+ "type": "object",
+ "required": ["event", "incident"],
+ "properties": {
+ "event": { "type": "string" },
+ "incident": {
+ "type": "object",
+ "required": [
+ "html_url",
+ "incident_number",
+ "title",
+ "status",
+ "created_at",
+ "urgency",
+ "incident_key"
+ ],
+ "properties": {
+ "html_url": { "type": "string" },
+ "incindent_number": { "type": "integer" },
+ "title": { "type": "string" },
+ "status": { "type": "string" },
+ "created_at": { "type": "string" },
+ "urgency": { "type": "string", "enum": ["high", "low"] },
+ "incident_key": { "type": ["string", "null"] },
+ "assignments": {
+ "type": "array",
+ "items": {
+ "assignee": {
+ "type": "array",
+ "items": {
+ "summary": { "type": "string" },
+ "html_url": { "type": "string" }
+ }
+ }
+ }
+ },
+ "impacted_services": {
+ "type": "array",
+ "items": {
+ "summary": { "type": "string" },
+ "html_url": { "type": "string" }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/pager_duty/webhook_payload_parser.rb b/lib/pager_duty/webhook_payload_parser.rb
index 573fb36f0ca..11071926cf2 100644
--- a/lib/pager_duty/webhook_payload_parser.rb
+++ b/lib/pager_duty/webhook_payload_parser.rb
@@ -2,6 +2,8 @@
module PagerDuty
class WebhookPayloadParser
+ SCHEMA_PATH = File.join('lib', 'pager_duty', 'validator', 'schemas', 'message.json')
+
def initialize(payload)
@payload = payload
end
@@ -11,7 +13,7 @@ module PagerDuty
end
def call
- Array(payload['messages']).map { |msg| parse_message(msg) }
+ Array(payload['messages']).map { |msg| parse_message(msg) }.reject(&:empty?)
end
private
@@ -19,6 +21,8 @@ module PagerDuty
attr_reader :payload
def parse_message(message)
+ return {} unless valid_message?(message)
+
{
'event' => message['event'],
'incident' => parse_incident(message['incident'])
@@ -26,8 +30,6 @@ module PagerDuty
end
def parse_incident(incident)
- return {} if incident.blank?
-
{
'url' => incident['html_url'],
'incident_number' => incident['incident_number'],
@@ -62,5 +64,9 @@ module PagerDuty
def reject_empty(entities)
Array(entities).reject { |e| e['summary'].blank? && e['url'].blank? }
end
+
+ def valid_message?(message)
+ ::JSONSchemer.schema(Pathname.new(SCHEMA_PATH)).valid?(message)
+ end
end
end
diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb
index ed3470f81f4..77108bb81ca 100644
--- a/lib/peek/views/active_record.rb
+++ b/lib/peek/views/active_record.rb
@@ -17,24 +17,35 @@ module Peek
}
}.freeze
+ def results
+ super.merge(calls: detailed_calls)
+ end
+
def self.thresholds
@thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS)
end
private
+ def detailed_calls
+ "#{calls} (#{cached_calls} cached)"
+ end
+
+ def cached_calls
+ detail_store.count { |item| item[:cached] == 'cached' }
+ end
+
def setup_subscribers
super
subscribe('sql.active_record') do |_, start, finish, _, data|
if Gitlab::PerformanceBar.enabled_for_request?
- unless data[:cached]
- detail_store << {
- duration: finish - start,
- sql: data[:sql].strip,
- backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)
- }
- end
+ detail_store << {
+ duration: finish - start,
+ sql: data[:sql].strip,
+ backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller),
+ cached: data[:cached] ? 'cached' : ''
+ }
end
end
end
diff --git a/lib/peek/views/detailed_view.rb b/lib/peek/views/detailed_view.rb
index 389f5301079..4cc2e85c7bb 100644
--- a/lib/peek/views/detailed_view.rb
+++ b/lib/peek/views/detailed_view.rb
@@ -23,7 +23,7 @@ module Peek
private
def duration
- detail_store.map { |entry| entry[:duration] }.sum * 1000 # rubocop:disable CodeReuse/ActiveRecord
+ detail_store.map { |entry| entry[:duration] }.sum * 1000
end
def calls
diff --git a/lib/safe_zip/extract.rb b/lib/safe_zip/extract.rb
index 679c021c730..ac33f0b3c2c 100644
--- a/lib/safe_zip/extract.rb
+++ b/lib/safe_zip/extract.rb
@@ -19,11 +19,7 @@ module SafeZip
def extract(opts = {})
params = SafeZip::ExtractParams.new(**opts)
- if Feature.enabled?(:safezip_use_rubyzip, default_enabled: true)
- extract_with_ruby_zip(params)
- else
- legacy_unsafe_extract_with_system_zip(params)
- end
+ extract_with_ruby_zip(params)
end
private
@@ -53,21 +49,5 @@ module SafeZip
.extract
end
end
-
- def legacy_unsafe_extract_with_system_zip(params)
- # Requires UnZip at least 6.00 Info-ZIP.
- # -n never overwrite existing files
- args = %W(unzip -n -qq #{archive_path})
-
- # We add * to end of directory, because we want to extract directory and all subdirectories
- args += params.directories_wildcard
-
- # Target directory where we extract
- args += %W(-d #{params.extract_path})
-
- unless system(*args)
- raise Error, 'archive failed to extract'
- end
- end
end
end
diff --git a/lib/sentry/client/issue.rb b/lib/sentry/client/issue.rb
index c5e9df9cd21..f714bda49fd 100644
--- a/lib/sentry/client/issue.rb
+++ b/lib/sentry/client/issue.rb
@@ -14,7 +14,7 @@ module Sentry
}.freeze
def list_issues(**keyword_args)
- response = get_issues(keyword_args)
+ response = get_issues(**keyword_args)
issues = response[:issues]
pagination = response[:pagination]
@@ -44,7 +44,7 @@ module Sentry
def get_issues(**keyword_args)
response = http_get(
api_urls.issues_url,
- query: list_issue_sentry_query(keyword_args)
+ query: list_issue_sentry_query(**keyword_args)
)
{
diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb
index 697ced3590b..a205861b9a9 100644
--- a/lib/system_check/app/redis_version_check.rb
+++ b/lib/system_check/app/redis_version_check.rb
@@ -37,7 +37,7 @@ module SystemCheck
@custom_error_message
)
for_more_information(
- 'doc/administration/high_availability/redis.md#provide-your-own-redis-instance'
+ 'doc/administration/redis/index.html#redis-replication-and-failover-using-the-non-bundled-redis'
)
fix_and_rerun
end
diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index 1e28d15f75e..e2c92054d62 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -38,7 +38,7 @@ namespace :gettext do
Rake::Task['gettext:find'].invoke
# leave only the required changes.
- unless system(*%w(git checkout -- locale/*/gitlab.po))
+ unless system(*%w(git -c core.hooksPath=/dev/null checkout -- locale/*/gitlab.po))
raise 'failed to cleanup generated locale/*/gitlab.po files'
end
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index caa583fb3a9..ab2d77eeaf0 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -81,7 +81,7 @@ namespace :gitlab do
if head_assets_md5 != master_assets_md5 || !public_assets_webpack_dir_exists
FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) if public_assets_webpack_dir_exists
- Rake::Task['webpack:compile'].invoke
+ system('yarn webpack')
end
end
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index 2a3713ed85c..de2dfca8c1b 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -107,7 +107,7 @@ namespace :gitlab do
puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red)
exit 1
else
- Backup::Repository.new(progress).dump(
+ Backup::Repositories.new(progress).dump(
max_concurrency: max_concurrency,
max_storage_concurrency: max_storage_concurrency
)
@@ -117,7 +117,7 @@ namespace :gitlab do
task restore: :gitlab_environment do
puts_time "Restoring repositories ...".color(:blue)
- Backup::Repository.new(progress).restore
+ Backup::Repositories.new(progress).restore
puts_time "done".color(:green)
end
end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 425f66918b0..a3f20f31f64 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -35,6 +35,11 @@ namespace :gitlab do
# Truncate schema_migrations to ensure migrations re-run
connection.execute('TRUNCATE schema_migrations') if connection.table_exists? 'schema_migrations'
+ # Drop any views
+ connection.views.each do |view|
+ connection.execute("DROP VIEW IF EXISTS #{connection.quote_table_name(view)} CASCADE")
+ end
+
# Drop tables with cascade to avoid dependent table errors
# PG: http://www.postgresql.org/docs/current/static/ddl-depend.html
# Add `IF EXISTS` because cascade could have already deleted a table.
@@ -60,6 +65,19 @@ namespace :gitlab do
end
end
+ desc 'GitLab | DB | Run database migrations and print `unattended_migrations_completed` if action taken'
+ task unattended: :environment do
+ no_database = !ActiveRecord::Base.connection.schema_migration.table_exists?
+ needs_migrations = ActiveRecord::Base.connection.migration_context.needs_migration?
+
+ if no_database || needs_migrations
+ Rake::Task['gitlab:db:configure'].invoke
+ puts "unattended_migrations_completed"
+ else
+ puts "unattended_migrations_static"
+ end
+ end
+
desc 'GitLab | DB | Checks if migrations require downtime or not'
task :downtime_check, [:ref] => :environment do |_, args|
abort 'You must specify a Git reference to compare with' unless args[:ref]
@@ -169,9 +187,21 @@ namespace :gitlab do
desc 'reindex a regular (non-unique) index without downtime to eliminate bloat'
task :reindex, [:index_name] => :environment do |_, args|
- raise ArgumentError, 'must give the index name to reindex' unless args[:index_name]
+ unless Feature.enabled?(:database_reindexing, type: :ops)
+ puts "This feature (database_reindexing) is currently disabled.".color(:yellow)
+ exit
+ end
+
+ indexes = if args[:index_name]
+ [Gitlab::Database::PostgresIndex.by_identifier(args[:index_name])]
+ else
+ Gitlab::Database::Reindexing.candidate_indexes.random_few(2)
+ end
- Gitlab::Database::ConcurrentReindex.new(args[:index_name], logger: Logger.new(STDOUT)).execute
+ Gitlab::Database::Reindexing.perform(indexes)
+ rescue => e
+ Gitlab::AppLogger.error(e)
+ raise
end
end
end
diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake
index 0b98755a77c..b242329d720 100644
--- a/lib/tasks/gitlab/web_hook.rake
+++ b/lib/tasks/gitlab/web_hook.rake
@@ -37,7 +37,10 @@ namespace :gitlab do
web_hooks.find_each do |hook|
next unless hook.url == web_hook_url
- hook.destroy!
+ result = WebHooks::DestroyService.new(nil).sync_destroy(hook)
+
+ raise "Unable to destroy Web hook" unless result[:status] == :success
+
count += 1
end
diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb
index cd5943b552e..9b034d1c6c2 100644
--- a/lib/uploaded_file.rb
+++ b/lib/uploaded_file.rb
@@ -42,6 +42,9 @@ class UploadedFile
@remote_id = remote_id
end
+ # TODO this function is meant to replace .from_params when the feature flag
+ # upload_middleware_jwt_params_handler is removed
+ # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps
def self.from_params_without_field(params, upload_paths)
path = params['path']
remote_id = params['remote_id']
@@ -68,6 +71,10 @@ class UploadedFile
)
end
+ # Deprecated. Don't use it.
+ # .from_params_without_field will replace this one
+ # See .from_params_without_field and
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps
def self.from_params(params, field, upload_paths, path_override = nil)
path = path_override || params["#{field}.path"]
remote_id = params["#{field}.remote_id"]