summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-08-20 18:42:06 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-08-20 18:42:06 +0000
commit6e4e1050d9dba2b7b2523fdd1768823ab85feef4 (patch)
tree78be5963ec075d80116a932011d695dd33910b4e /lib
parent1ce776de4ae122aba3f349c02c17cebeaa8ecf07 (diff)
downloadgitlab-ce-6e4e1050d9dba2b7b2523fdd1768823ab85feef4.tar.gz
Add latest changes from gitlab-org/gitlab@13-3-stable-ee
Diffstat (limited to 'lib')
-rw-r--r--lib/api/admin/ci/variables.rb18
-rw-r--r--lib/api/api.rb6
-rw-r--r--lib/api/api_guard.rb11
-rw-r--r--lib/api/branches.rb8
-rw-r--r--lib/api/ci/pipeline_schedules.rb34
-rw-r--r--lib/api/ci/pipelines.rb34
-rw-r--r--lib/api/ci/runners.rb4
-rw-r--r--lib/api/commits.rb38
-rw-r--r--lib/api/composer_packages.rb10
-rw-r--r--lib/api/conan_packages.rb90
-rw-r--r--lib/api/entities/branch.rb2
-rw-r--r--lib/api/entities/bridge.rb9
-rw-r--r--lib/api/entities/ci/bridge.rb11
-rw-r--r--lib/api/entities/ci/job.rb15
-rw-r--r--lib/api/entities/ci/job_artifact.rb11
-rw-r--r--lib/api/entities/ci/job_artifact_file.rb12
-rw-r--r--lib/api/entities/ci/job_basic.rb20
-rw-r--r--lib/api/entities/ci/job_basic_with_project.rb11
-rw-r--r--lib/api/entities/ci/pipeline.rb19
-rw-r--r--lib/api/entities/ci/pipeline_basic.rb16
-rw-r--r--lib/api/entities/ci/pipeline_schedule.rb14
-rw-r--r--lib/api/entities/ci/pipeline_schedule_details.rb12
-rw-r--r--lib/api/entities/ci/variable.rb14
-rw-r--r--lib/api/entities/commit_detail.rb2
-rw-r--r--lib/api/entities/deployment.rb2
-rw-r--r--lib/api/entities/event.rb1
-rw-r--r--lib/api/entities/feature.rb2
-rw-r--r--lib/api/entities/job.rb13
-rw-r--r--lib/api/entities/job_artifact.rb9
-rw-r--r--lib/api/entities/job_artifact_file.rb10
-rw-r--r--lib/api/entities/job_basic.rb18
-rw-r--r--lib/api/entities/job_basic_with_project.rb9
-rw-r--r--lib/api/entities/job_request/dependency.rb2
-rw-r--r--lib/api/entities/job_request/response.rb2
-rw-r--r--lib/api/entities/merge_request.rb4
-rw-r--r--lib/api/entities/package.rb4
-rw-r--r--lib/api/entities/package/pipeline.rb2
-rw-r--r--lib/api/entities/personal_access_token.rb2
-rw-r--r--lib/api/entities/pipeline.rb17
-rw-r--r--lib/api/entities/pipeline_basic.rb14
-rw-r--r--lib/api/entities/pipeline_schedule.rb12
-rw-r--r--lib/api/entities/pipeline_schedule_details.rb10
-rw-r--r--lib/api/entities/project.rb1
-rw-r--r--lib/api/entities/project_hook.rb2
-rw-r--r--lib/api/entities/user_details_with_admin.rb1
-rw-r--r--lib/api/entities/variable.rb12
-rw-r--r--lib/api/group_variables.rb55
-rw-r--r--lib/api/groups.rb22
-rw-r--r--lib/api/helpers.rb12
-rw-r--r--lib/api/helpers/internal_helpers.rb2
-rw-r--r--lib/api/helpers/merge_requests_helpers.rb17
-rw-r--r--lib/api/helpers/notes_helpers.rb5
-rw-r--r--lib/api/helpers/packages/basic_auth_helpers.rb13
-rw-r--r--lib/api/helpers/packages/conan/api_helpers.rb24
-rw-r--r--lib/api/helpers/packages_manager_clients_helpers.rb10
-rw-r--r--lib/api/helpers/pagination_strategies.rb2
-rw-r--r--lib/api/helpers/performance_bar_helpers.rb21
-rw-r--r--lib/api/helpers/projects_helpers.rb2
-rw-r--r--lib/api/helpers/services_helpers.rb6
-rw-r--r--lib/api/helpers/snippets_helpers.rb24
-rw-r--r--lib/api/import_github.rb6
-rw-r--r--lib/api/internal/base.rb62
-rw-r--r--lib/api/internal/kubernetes.rb95
-rw-r--r--lib/api/issues.rb2
-rw-r--r--lib/api/job_artifacts.rb4
-rw-r--r--lib/api/jobs.rb34
-rw-r--r--lib/api/merge_requests.rb42
-rw-r--r--lib/api/milestone_responses.rb48
-rw-r--r--lib/api/notes.rb3
-rw-r--r--lib/api/nuget_packages.rb37
-rw-r--r--lib/api/project_export.rb2
-rw-r--r--lib/api/project_hooks.rb1
-rw-r--r--lib/api/project_snippets.rb10
-rw-r--r--lib/api/project_templates.rb14
-rw-r--r--lib/api/projects.rb2
-rw-r--r--lib/api/pypi_packages.rb12
-rw-r--r--lib/api/release/links.rb2
-rw-r--r--lib/api/releases.rb6
-rw-r--r--lib/api/snippets.rb11
-rw-r--r--lib/api/support/git_access_actor.rb9
-rw-r--r--lib/api/triggers.rb4
-rw-r--r--lib/api/users.rb2
-rw-r--r--lib/api/validations/validators/file_path.rb8
-rw-r--r--lib/api/variables.rb47
-rw-r--r--lib/api/wikis.rb5
-rw-r--r--lib/backup/files.rb2
-rw-r--r--lib/backup/repository.rb104
-rw-r--r--lib/banzai/filter/abstract_reference_filter.rb4
-rw-r--r--lib/banzai/filter/ascii_doc_post_processing_filter.rb3
-rw-r--r--lib/banzai/filter/base_sanitization_filter.rb2
-rw-r--r--lib/banzai/filter/design_reference_filter.rb2
-rw-r--r--lib/banzai/filter/gollum_tags_filter.rb2
-rw-r--r--lib/banzai/filter/inline_alert_metrics_filter.rb47
-rw-r--r--lib/banzai/filter/inline_cluster_metrics_filter.rb2
-rw-r--r--lib/banzai/filter/inline_embeds_filter.rb4
-rw-r--r--lib/banzai/filter/inline_metrics_filter.rb2
-rw-r--r--lib/banzai/filter/inline_metrics_redactor_filter.rb6
-rw-r--r--lib/banzai/filter/issue_reference_filter.rb4
-rw-r--r--lib/banzai/filter/label_reference_filter.rb8
-rw-r--r--lib/banzai/filter/reference_filter.rb22
-rw-r--r--lib/banzai/filter/syntax_highlight_filter.rb2
-rw-r--r--lib/banzai/reference_parser/base_parser.rb3
-rw-r--r--lib/banzai/reference_parser/snippet_parser.rb17
-rw-r--r--lib/bitbucket/representation/repo.rb2
-rw-r--r--lib/bitbucket_server/paginator.rb2
-rw-r--r--lib/declarative_policy/base.rb2
-rw-r--r--lib/declarative_policy/runner.rb2
-rw-r--r--lib/extracts_ref.rb84
-rw-r--r--lib/gitlab/alerting/alert.rb12
-rw-r--r--lib/gitlab/analytics/cycle_analytics/records_fetcher.rb18
-rw-r--r--lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb4
-rw-r--r--lib/gitlab/analytics/unique_visits.rb67
-rw-r--r--lib/gitlab/app_logger.rb6
-rw-r--r--lib/gitlab/asciidoc.rb2
-rw-r--r--lib/gitlab/asciidoc/mermaid_block_processor.rb35
-rw-r--r--lib/gitlab/audit/deleted_author.rb8
-rw-r--r--lib/gitlab/audit/null_author.rb36
-rw-r--r--lib/gitlab/audit/unauthenticated_author.rb17
-rw-r--r--lib/gitlab/auth.rb6
-rw-r--r--lib/gitlab/auth/auth_finders.rb12
-rw-r--r--lib/gitlab/auth/ldap/adapter.rb2
-rw-r--r--lib/gitlab/auth/ldap/person.rb2
-rw-r--r--lib/gitlab/auth/o_auth/user.rb8
-rw-r--r--lib/gitlab/background_migration/archive_legacy_traces.rb21
-rw-r--r--lib/gitlab/background_migration/backfill_designs_relative_position.rb15
-rw-r--r--lib/gitlab/background_migration/backfill_hashed_project_repositories.rb15
-rw-r--r--lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb213
-rw-r--r--lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb25
-rw-r--r--lib/gitlab/background_migration/fill_file_store_job_artifact.rb19
-rw-r--r--lib/gitlab/background_migration/fill_file_store_lfs_object.rb19
-rw-r--r--lib/gitlab/background_migration/fill_store_upload.rb20
-rw-r--r--lib/gitlab/background_migration/fix_cross_project_label_links.rb140
-rw-r--r--lib/gitlab/background_migration/migrate_build_stage.rb48
-rw-r--r--lib/gitlab/background_migration/migrate_build_stage_id_reference.rb22
-rw-r--r--lib/gitlab/background_migration/migrate_stage_index.rb35
-rw-r--r--lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb82
-rw-r--r--lib/gitlab/background_migration/populate_personal_snippet_statistics.rb49
-rw-r--r--lib/gitlab/background_migration/populate_untracked_uploads.rb111
-rw-r--r--lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb190
-rw-r--r--lib/gitlab/background_migration/prepare_untracked_uploads.rb173
-rw-r--r--lib/gitlab/background_migration/remove_restricted_todos.rb158
-rw-r--r--lib/gitlab/background_migration/set_confidential_note_events_on_services.rb26
-rw-r--r--lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb26
-rw-r--r--lib/gitlab/background_migration/set_merge_request_diff_files_count.rb28
-rw-r--r--lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb24
-rw-r--r--lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb24
-rw-r--r--lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb12
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/commit.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/design_management/design.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/epic.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/merge_request.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/note.rb1
-rw-r--r--lib/gitlab/backtrace_cleaner.rb2
-rw-r--r--lib/gitlab/bare_repository_import/importer.rb2
-rw-r--r--lib/gitlab/build_access.rb2
-rw-r--r--lib/gitlab/checks/change_access.rb2
-rw-r--r--lib/gitlab/checks/lfs_check.rb3
-rw-r--r--lib/gitlab/ci/build/artifacts/expire_in_parser.rb45
-rw-r--r--lib/gitlab/ci/build/artifacts/metadata/entry.rb2
-rw-r--r--lib/gitlab/ci/build/auto_retry.rb57
-rw-r--r--lib/gitlab/ci/build/step.rb2
-rw-r--r--lib/gitlab/ci/config/entry/artifacts.rb2
-rw-r--r--lib/gitlab/ci/config/entry/bridge.rb2
-rw-r--r--lib/gitlab/ci/config/entry/job.rb54
-rw-r--r--lib/gitlab/ci/config/entry/processable.rb20
-rw-r--r--lib/gitlab/ci/config/entry/product/matrix.rb61
-rw-r--r--lib/gitlab/ci/config/entry/product/parallel.rb57
-rw-r--r--lib/gitlab/ci/config/entry/product/variables.rb36
-rw-r--r--lib/gitlab/ci/config/external/context.rb2
-rw-r--r--lib/gitlab/ci/config/normalizer.rb20
-rw-r--r--lib/gitlab/ci/config/normalizer/factory.rb38
-rw-r--r--lib/gitlab/ci/config/normalizer/matrix_strategy.rb68
-rw-r--r--lib/gitlab/ci/config/normalizer/number_strategy.rb47
-rw-r--r--lib/gitlab/ci/features.rb50
-rw-r--r--lib/gitlab/ci/parsers/coverage/cobertura.rb2
-rw-r--r--lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb40
-rw-r--r--lib/gitlab/ci/pipeline/chain/command.rb4
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/content/parameter.rb1
-rw-r--r--lib/gitlab/ci/pipeline/chain/helpers.rb8
-rw-r--r--lib/gitlab/ci/pipeline/chain/metrics.rb23
-rw-r--r--lib/gitlab/ci/pipeline/chain/pipeline/process.rb24
-rw-r--r--lib/gitlab/ci/pipeline/chain/sequence.rb19
-rw-r--r--lib/gitlab/ci/pipeline/chain/stop_dry_run.rb22
-rw-r--r--lib/gitlab/ci/pipeline/chain/validate/abilities.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/and.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/base.rb8
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/equals.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb35
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/matches.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/null.rb6
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/operator.rb16
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/or.rb2
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb23
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb24
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb6
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/string.rb6
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/value.rb4
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/variable.rb8
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexer.rb26
-rw-r--r--lib/gitlab/ci/pipeline/expression/parser.rb43
-rw-r--r--lib/gitlab/ci/pipeline/metrics.rb9
-rw-r--r--lib/gitlab/ci/pipeline/seed/build.rb10
-rw-r--r--lib/gitlab/ci/reports/accessibility_reports_comparer.rb2
-rw-r--r--lib/gitlab/ci/reports/test_report_summary.rb35
-rw-r--r--lib/gitlab/ci/reports/test_reports.rb4
-rw-r--r--lib/gitlab/ci/reports/test_suite.rb2
-rw-r--r--lib/gitlab/ci/reports/test_suite_summary.rb31
-rw-r--r--lib/gitlab/ci/runner_instructions.rb137
-rw-r--r--lib/gitlab/ci/runner_instructions/templates/linux/install.sh12
-rw-r--r--lib/gitlab/ci/runner_instructions/templates/osx/install.sh11
-rw-r--r--lib/gitlab/ci/runner_instructions/templates/windows/install.ps113
-rw-r--r--lib/gitlab/ci/status/build/failed.rb3
-rw-r--r--lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml16
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml6
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml146
-rw-r--r--lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml18
-rw-r--r--lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml5
-rw-r--r--lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml14
-rw-r--r--lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml8
-rw-r--r--lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/yaml_processor.rb2
-rw-r--r--lib/gitlab/config/entry/legacy_validation_helpers.rb34
-rw-r--r--lib/gitlab/config/entry/validators.rb18
-rw-r--r--lib/gitlab/config_checker/external_database_checker.rb32
-rw-r--r--lib/gitlab/cycle_analytics/base_event_fetcher.rb2
-rw-r--r--lib/gitlab/cycle_analytics/summary/value.rb2
-rw-r--r--lib/gitlab/cycle_analytics/summary_helper.rb2
-rw-r--r--lib/gitlab/danger/helper.rb15
-rw-r--r--lib/gitlab/danger/roulette.rb85
-rw-r--r--lib/gitlab/danger/teammate.rb17
-rw-r--r--lib/gitlab/database.rb16
-rw-r--r--lib/gitlab/database/batch_count.rb23
-rw-r--r--lib/gitlab/database/connection_timer.rb2
-rw-r--r--lib/gitlab/database/migration_helpers.rb4
-rw-r--r--lib/gitlab/database/partitioning/partition_creator.rb2
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb42
-rw-r--r--lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb18
-rw-r--r--lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin.rb28
-rw-r--r--lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb16
-rw-r--r--lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb2
-rw-r--r--lib/gitlab/database/schema_cleaner.rb5
-rw-r--r--lib/gitlab/database/schema_helpers.rb2
-rw-r--r--lib/gitlab/database/schema_version_files.rb64
-rw-r--r--lib/gitlab/database/similarity_score.rb110
-rw-r--r--lib/gitlab/database/with_lock_retries.rb22
-rw-r--r--lib/gitlab/devise_failure.rb2
-rw-r--r--lib/gitlab/diff/formatters/base_formatter.rb1
-rw-r--r--lib/gitlab/diff/highlight_cache.rb57
-rw-r--r--lib/gitlab/diff/stats_cache.rb7
-rw-r--r--lib/gitlab/exception_log_formatter.rb2
-rw-r--r--lib/gitlab/exclusive_lease.rb2
-rw-r--r--lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb2
-rw-r--r--lib/gitlab/experimentation.rb24
-rw-r--r--lib/gitlab/external_authorization/client.rb11
-rw-r--r--lib/gitlab/external_authorization/response.rb10
-rw-r--r--lib/gitlab/file_hook.rb2
-rw-r--r--lib/gitlab/git/blob.rb22
-rw-r--r--lib/gitlab/git/commit.rb4
-rw-r--r--lib/gitlab/git/pre_receive_error.rb12
-rw-r--r--lib/gitlab/git/repository.rb14
-rw-r--r--lib/gitlab/git/rugged_impl/blob.rb2
-rw-r--r--lib/gitlab/git/tree.rb4
-rw-r--r--lib/gitlab/git/wiki_page.rb2
-rw-r--r--lib/gitlab/git_access.rb183
-rw-r--r--lib/gitlab/git_access_design.rb9
-rw-r--r--lib/gitlab/git_access_project.rb38
-rw-r--r--lib/gitlab/git_access_snippet.rb86
-rw-r--r--lib/gitlab/git_access_wiki.rb41
-rw-r--r--lib/gitlab/gitaly_client.rb9
-rw-r--r--lib/gitlab/gitaly_client/operation_service.rb17
-rw-r--r--lib/gitlab/github_import/user_finder.rb2
-rw-r--r--lib/gitlab/gl_repository/repo_type.rb8
-rw-r--r--lib/gitlab/gon_helper.rb3
-rw-r--r--lib/gitlab/hashed_path.rb24
-rw-r--r--lib/gitlab/hashed_storage/migrator.rb2
-rw-r--r--lib/gitlab/http.rb41
-rw-r--r--lib/gitlab/i18n.rb18
-rw-r--r--lib/gitlab/i18n/html_todo.yml315
-rw-r--r--lib/gitlab/i18n/po_linter.rb31
-rw-r--r--lib/gitlab/i18n/translation_entry.rb36
-rw-r--r--lib/gitlab/import_export/command_line_util.rb4
-rw-r--r--lib/gitlab/import_export/file_importer.rb4
-rw-r--r--lib/gitlab/import_export/project/import_export.yml2
-rw-r--r--lib/gitlab/incident_management/pager_duty/incident_issue_description.rb2
-rw-r--r--lib/gitlab/incoming_email.rb4
-rw-r--r--lib/gitlab/instrumentation/redis_base.rb2
-rw-r--r--lib/gitlab/issuables_count_for_state.rb14
-rw-r--r--lib/gitlab/json.rb28
-rw-r--r--lib/gitlab/kubernetes/cilium_network_policy.rb89
-rw-r--r--lib/gitlab/kubernetes/helm/base_command.rb7
-rw-r--r--lib/gitlab/kubernetes/helm/client_command.rb29
-rw-r--r--lib/gitlab/kubernetes/helm/delete_command.rb5
-rw-r--r--lib/gitlab/kubernetes/helm/install_command.rb2
-rw-r--r--lib/gitlab/kubernetes/helm/patch_command.rb2
-rw-r--r--lib/gitlab/kubernetes/kube_client.rb19
-rw-r--r--lib/gitlab/kubernetes/network_policy.rb79
-rw-r--r--lib/gitlab/kubernetes/network_policy_common.rb65
-rw-r--r--lib/gitlab/kubernetes/node.rb21
-rw-r--r--lib/gitlab/manifest_import/project_creator.rb1
-rw-r--r--lib/gitlab/markdown_cache.rb2
-rw-r--r--lib/gitlab/metrics.rb14
-rw-r--r--lib/gitlab/metrics/dashboard/cache.rb61
-rw-r--r--lib/gitlab/metrics/dashboard/defaults.rb1
-rw-r--r--lib/gitlab/metrics/dashboard/finder.rb17
-rw-r--r--lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb37
-rw-r--r--lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb6
-rw-r--r--lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb4
-rw-r--r--lib/gitlab/metrics/dashboard/stages/sorter.rb34
-rw-r--r--lib/gitlab/metrics/dashboard/stages/track_panel_type.rb27
-rw-r--r--lib/gitlab/metrics/dashboard/url.rb57
-rw-r--r--lib/gitlab/metrics/dashboard/validator.rb30
-rw-r--r--lib/gitlab/metrics/dashboard/validator/client.rb56
-rw-r--r--lib/gitlab/metrics/dashboard/validator/custom_formats.rb23
-rw-r--r--lib/gitlab/metrics/dashboard/validator/errors.rb60
-rw-r--r--lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb52
-rw-r--r--lib/gitlab/metrics/dashboard/validator/schemas/axis.json14
-rw-r--r--lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json18
-rw-r--r--lib/gitlab/metrics/dashboard/validator/schemas/link.json12
-rw-r--r--lib/gitlab/metrics/dashboard/validator/schemas/metric.json16
-rw-r--r--lib/gitlab/metrics/dashboard/validator/schemas/panel.json24
-rw-r--r--lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json12
-rw-r--r--lib/gitlab/metrics/dashboard/validator/schemas/templating.json7
-rw-r--r--lib/gitlab/metrics/elasticsearch_rack_middleware.rb23
-rw-r--r--lib/gitlab/metrics/method_call.rb25
-rw-r--r--lib/gitlab/metrics/methods.rb56
-rw-r--r--lib/gitlab/metrics/methods/metric_options.rb20
-rw-r--r--lib/gitlab/metrics/rack_middleware.rb18
-rw-r--r--lib/gitlab/metrics/redis_rack_middleware.rb39
-rw-r--r--lib/gitlab/metrics/samplers/threads_sampler.rb78
-rw-r--r--lib/gitlab/metrics/sidekiq_middleware.rb4
-rw-r--r--lib/gitlab/metrics/subscribers/action_view.rb20
-rw-r--r--lib/gitlab/metrics/subscribers/active_record.rb19
-rw-r--r--lib/gitlab/metrics/subscribers/rails_cache.rb48
-rw-r--r--lib/gitlab/metrics/templates/Area.metrics-dashboard.yml15
-rw-r--r--lib/gitlab/metrics/templates/Default.metrics-dashboard.yml24
-rw-r--r--lib/gitlab/metrics/templates/gauge.metrics-dashboard.yml23
-rw-r--r--lib/gitlab/metrics/templates/index.md3
-rw-r--r--lib/gitlab/metrics/templates/k8s_area.metrics-dashboard.yml15
-rw-r--r--lib/gitlab/metrics/templates/k8s_gauge.metrics-dashboard.yml23
-rw-r--r--lib/gitlab/metrics/templates/k8s_single-stat.metrics-dashboard.yml17
-rw-r--r--lib/gitlab/metrics/templates/single-stat.metrics-dashboard.yml17
-rw-r--r--lib/gitlab/metrics/transaction.rb139
-rw-r--r--lib/gitlab/middleware/rails_queue_duration.rb18
-rw-r--r--lib/gitlab/middleware/read_only/controller.rb2
-rw-r--r--lib/gitlab/multi_collection_paginator.rb2
-rw-r--r--lib/gitlab/pages.rb2
-rw-r--r--lib/gitlab/pages/settings.rb21
-rw-r--r--lib/gitlab/pagination/gitaly_keyset_pager.rb54
-rw-r--r--lib/gitlab/pagination/keyset/header_builder.rb45
-rw-r--r--lib/gitlab/pagination/keyset/request_context.rb27
-rw-r--r--lib/gitlab/polling_interval.rb2
-rw-r--r--lib/gitlab/project_search_results.rb2
-rw-r--r--lib/gitlab/prometheus_client.rb4
-rw-r--r--lib/gitlab/reactive_cache_set_cache.rb2
-rw-r--r--lib/gitlab/redis/hll.rb45
-rw-r--r--lib/gitlab/regex.rb11
-rw-r--r--lib/gitlab/repository_cache_adapter.rb18
-rw-r--r--lib/gitlab/repository_hash_cache.rb16
-rw-r--r--lib/gitlab/search/parsed_query.rb45
-rw-r--r--lib/gitlab/search/query.rb8
-rw-r--r--lib/gitlab/seeder.rb2
-rw-r--r--lib/gitlab/service_desk_email.rb6
-rw-r--r--lib/gitlab/set_cache.rb14
-rw-r--r--lib/gitlab/sidekiq_cluster.rb2
-rw-r--r--lib/gitlab/sidekiq_daemon/memory_killer.rb2
-rw-r--r--lib/gitlab/sidekiq_logger.rb9
-rw-r--r--lib/gitlab/sidekiq_logging/exception_handler.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware.rb1
-rw-r--r--lib/gitlab/sidekiq_middleware/memory_killer.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/server_metrics.rb4
-rw-r--r--lib/gitlab/sidekiq_status.rb2
-rw-r--r--lib/gitlab/sidekiq_versioning/middleware.rb13
-rw-r--r--lib/gitlab/sidekiq_versioning/worker.rb31
-rw-r--r--lib/gitlab/slash_commands/presenters/base.rb1
-rw-r--r--lib/gitlab/slash_commands/presenters/issue_search.rb6
-rw-r--r--lib/gitlab/slash_commands/presenters/issue_show.rb8
-rw-r--r--lib/gitlab/static_site_editor/config.rb4
-rw-r--r--lib/gitlab/suggestions/suggestion_set.rb2
-rw-r--r--lib/gitlab/task_helpers.rb2
-rw-r--r--lib/gitlab/template/metrics_dashboard_template.rb32
-rw-r--r--lib/gitlab/untrusted_regexp.rb4
-rw-r--r--lib/gitlab/usage_data.rb150
-rw-r--r--lib/gitlab/usage_data/topology.rb73
-rw-r--r--lib/gitlab/usage_data_counters/hll_redis_counter.rb149
-rw-r--r--lib/gitlab/usage_data_counters/known_events.yml88
-rw-r--r--lib/gitlab/usage_data_counters/track_unique_actions.rb24
-rw-r--r--lib/gitlab/usage_data_counters/wiki_page_counter.rb2
-rw-r--r--lib/gitlab/user_access.rb77
-rw-r--r--lib/gitlab/user_access_snippet.rb15
-rw-r--r--lib/gitlab/utils.rb38
-rw-r--r--lib/gitlab/utils/usage_data.rb2
-rw-r--r--lib/gitlab/view/presenter/base.rb12
-rw-r--r--lib/gitlab/workhorse.rb12
-rw-r--r--lib/object_storage/config.rb83
-rw-r--r--lib/object_storage/direct_upload.rb59
-rw-r--r--lib/product_analytics/event_params.rb7
-rw-r--r--lib/product_analytics/tracker.rb11
-rw-r--r--lib/system_check/app/git_version_check.rb2
-rw-r--r--lib/system_check/app/redis_version_check.rb4
-rw-r--r--lib/system_check/sidekiq_check.rb2
-rw-r--r--lib/tasks/gettext.rake12
-rw-r--r--lib/tasks/gitlab/backup.rake11
-rw-r--r--lib/tasks/gitlab/gitaly.rake2
-rw-r--r--lib/tasks/gitlab/snippets.rake8
-rw-r--r--lib/tasks/gitlab/workhorse.rake2
413 files changed, 6031 insertions, 3231 deletions
diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb
index 6b0ff5e9395..8721d94d642 100644
--- a/lib/api/admin/ci/variables.rb
+++ b/lib/api/admin/ci/variables.rb
@@ -12,7 +12,7 @@ module API
namespace 'ci' do
namespace 'variables' do
desc 'Get instance-level variables' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
use :pagination
@@ -20,11 +20,11 @@ module API
get '/' do
variables = ::Ci::InstanceVariable.all
- present paginate(variables), with: Entities::Variable
+ present paginate(variables), with: Entities::Ci::Variable
end
desc 'Get a specific variable from a group' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :key, type: String, desc: 'The key of the variable'
@@ -35,11 +35,11 @@ module API
break not_found!('InstanceVariable') unless variable
- present variable, with: Entities::Variable
+ present variable, with: Entities::Ci::Variable
end
desc 'Create a new instance-level variable' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :key,
@@ -69,14 +69,14 @@ module API
variable = ::Ci::InstanceVariable.new(variable_params)
if variable.save
- present variable, with: Entities::Variable
+ present variable, with: Entities::Ci::Variable
else
render_validation_error!(variable)
end
end
desc 'Update an existing instance-variable' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
optional :key,
@@ -108,14 +108,14 @@ module API
variable_params = declared_params(include_missing: false).except(:key)
if variable.update(variable_params)
- present variable, with: Entities::Variable
+ present variable, with: Entities::Ci::Variable
else
render_validation_error!(variable)
end
end
desc 'Delete an existing instance-level variable' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :key, type: String, desc: 'The key of the variable'
diff --git a/lib/api/api.rb b/lib/api/api.rb
index a89dc0fa6fa..2be6792af5f 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -56,6 +56,10 @@ module API
)
end
+ before do
+ set_peek_enabled_for_current_request
+ end
+
# The locale is set to the current user's locale when `current_user` is loaded
after { Gitlab::I18n.use_default_locale }
@@ -116,6 +120,7 @@ module API
# Ensure the namespace is right, otherwise we might load Grape::API::Helpers
helpers ::API::Helpers
helpers ::API::Helpers::CommonHelpers
+ helpers ::API::Helpers::PerformanceBarHelpers
namespace do
after do
@@ -237,6 +242,7 @@ module API
mount ::API::Internal::Base
mount ::API::Internal::Pages
+ mount ::API::Internal::Kubernetes
route :any, '*path' do
error!('404 Not Found', 404)
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index 4b87861a3de..59978962b1d 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -7,6 +7,7 @@ require 'rack/oauth2'
module API
module APIGuard
extend ActiveSupport::Concern
+ include Gitlab::Utils::StrongMemoize
included do |base|
# OAuth2 Resource Server Authentication
@@ -64,10 +65,12 @@ module API
end
def find_user_from_sources
- deploy_token_from_request ||
- find_user_from_bearer_token ||
- find_user_from_job_token ||
- find_user_from_warden
+ strong_memoize(:find_user_from_sources) do
+ deploy_token_from_request ||
+ find_user_from_bearer_token ||
+ find_user_from_job_token ||
+ find_user_from_warden
+ end
end
private
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 5e9c2caf8f5..44f7610384e 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -40,12 +40,8 @@ module API
repository = user_project.repository
- if Feature.enabled?(:branch_list_keyset_pagination, user_project)
- branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute(gitaly_pagination: true)
- else
- branches = BranchesFinder.new(repository, declared_params(include_missing: false)).execute
- branches = paginate(::Kaminari.paginate_array(branches))
- end
+ branches_finder = BranchesFinder.new(repository, declared_params(include_missing: false))
+ branches = Gitlab::Pagination::GitalyKeysetPager.new(self, user_project).paginate(branches_finder)
merged_branch_names = repository.merged_branch_names(branches.map(&:name))
diff --git a/lib/api/ci/pipeline_schedules.rb b/lib/api/ci/pipeline_schedules.rb
index 80ad8aa04dd..1afdb0ad34c 100644
--- a/lib/api/ci/pipeline_schedules.rb
+++ b/lib/api/ci/pipeline_schedules.rb
@@ -12,7 +12,7 @@ module API
end
resource :projects, requirements: ::API::API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get all pipeline schedules' do
- success Entities::PipelineSchedule
+ success Entities::Ci::PipelineSchedule
end
params do
use :pagination
@@ -25,22 +25,22 @@ module API
schedules = ::Ci::PipelineSchedulesFinder.new(user_project).execute(scope: params[:scope])
.preload([:owner, :last_pipeline])
- present paginate(schedules), with: Entities::PipelineSchedule
+ present paginate(schedules), with: Entities::Ci::PipelineSchedule
end
# rubocop: enable CodeReuse/ActiveRecord
desc 'Get a single pipeline schedule' do
- success Entities::PipelineScheduleDetails
+ success Entities::Ci::PipelineScheduleDetails
end
params do
requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
end
get ':id/pipeline_schedules/:pipeline_schedule_id' do
- present pipeline_schedule, with: Entities::PipelineScheduleDetails
+ present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails
end
desc 'Create a new pipeline schedule' do
- success Entities::PipelineScheduleDetails
+ success Entities::Ci::PipelineScheduleDetails
end
params do
requires :description, type: String, desc: 'The description of pipeline schedule'
@@ -57,14 +57,14 @@ module API
.execute
if pipeline_schedule.persisted?
- present pipeline_schedule, with: Entities::PipelineScheduleDetails
+ present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails
else
render_validation_error!(pipeline_schedule)
end
end
desc 'Edit a pipeline schedule' do
- success Entities::PipelineScheduleDetails
+ success Entities::Ci::PipelineScheduleDetails
end
params do
requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
@@ -78,14 +78,14 @@ module API
authorize! :update_pipeline_schedule, pipeline_schedule
if pipeline_schedule.update(declared_params(include_missing: false))
- present pipeline_schedule, with: Entities::PipelineScheduleDetails
+ present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails
else
render_validation_error!(pipeline_schedule)
end
end
desc 'Take ownership of a pipeline schedule' do
- success Entities::PipelineScheduleDetails
+ success Entities::Ci::PipelineScheduleDetails
end
params do
requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
@@ -94,14 +94,14 @@ module API
authorize! :update_pipeline_schedule, pipeline_schedule
if pipeline_schedule.own!(current_user)
- present pipeline_schedule, with: Entities::PipelineScheduleDetails
+ present pipeline_schedule, with: Entities::Ci::PipelineScheduleDetails
else
render_validation_error!(pipeline_schedule)
end
end
desc 'Delete a pipeline schedule' do
- success Entities::PipelineScheduleDetails
+ success Entities::Ci::PipelineScheduleDetails
end
params do
requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
@@ -132,7 +132,7 @@ module API
end
desc 'Create a new pipeline schedule variable' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
@@ -146,14 +146,14 @@ module API
variable_params = declared_params(include_missing: false)
variable = pipeline_schedule.variables.create(variable_params)
if variable.persisted?
- present variable, with: Entities::Variable
+ present variable, with: Entities::Ci::Variable
else
render_validation_error!(variable)
end
end
desc 'Edit a pipeline schedule variable' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
@@ -165,14 +165,14 @@ module API
authorize! :update_pipeline_schedule, pipeline_schedule
if pipeline_schedule_variable.update(declared_params(include_missing: false))
- present pipeline_schedule_variable, with: Entities::Variable
+ present pipeline_schedule_variable, with: Entities::Ci::Variable
else
render_validation_error!(pipeline_schedule_variable)
end
end
desc 'Delete a pipeline schedule variable' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :pipeline_schedule_id, type: Integer, desc: 'The pipeline schedule id'
@@ -182,7 +182,7 @@ module API
authorize! :admin_pipeline_schedule, pipeline_schedule
status :accepted
- present pipeline_schedule_variable.destroy, with: Entities::Variable
+ present pipeline_schedule_variable.destroy, with: Entities::Ci::Variable
end
end
diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb
index 33bb8b38d92..a010e0dd761 100644
--- a/lib/api/ci/pipelines.rb
+++ b/lib/api/ci/pipelines.rb
@@ -13,7 +13,7 @@ module API
resource :projects, requirements: ::API::API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get all Pipelines of the project' do
detail 'This feature was introduced in GitLab 8.11.'
- success Entities::PipelineBasic
+ success Entities::Ci::PipelineBasic
end
params do
use :pagination
@@ -38,12 +38,12 @@ module API
authorize! :read_build, user_project
pipelines = ::Ci::PipelinesFinder.new(user_project, current_user, params).execute
- present paginate(pipelines), with: Entities::PipelineBasic
+ present paginate(pipelines), with: Entities::Ci::PipelineBasic
end
desc 'Create a new pipeline' do
detail 'This feature was introduced in GitLab 8.14'
- success Entities::Pipeline
+ success Entities::Ci::Pipeline
end
params do
requires :ref, type: String, desc: 'Reference'
@@ -64,7 +64,7 @@ module API
.execute(:api, ignore_skip_ci: true, save_on_errors: false)
if new_pipeline.persisted?
- present new_pipeline, with: Entities::Pipeline
+ present new_pipeline, with: Entities::Ci::Pipeline
else
render_validation_error!(new_pipeline)
end
@@ -72,7 +72,7 @@ module API
desc 'Gets a the latest pipeline for the project branch' do
detail 'This feature was introduced in GitLab 12.3'
- success Entities::Pipeline
+ success Entities::Ci::Pipeline
end
params do
optional :ref, type: String, desc: 'branch ref of pipeline'
@@ -80,12 +80,12 @@ module API
get ':id/pipelines/latest' do
authorize! :read_pipeline, latest_pipeline
- present latest_pipeline, with: Entities::Pipeline
+ present latest_pipeline, with: Entities::Ci::Pipeline
end
desc 'Gets a specific pipeline for the project' do
detail 'This feature was introduced in GitLab 8.11'
- success Entities::Pipeline
+ success Entities::Ci::Pipeline
end
params do
requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
@@ -93,12 +93,12 @@ module API
get ':id/pipelines/:pipeline_id' do
authorize! :read_pipeline, pipeline
- present pipeline, with: Entities::Pipeline
+ present pipeline, with: Entities::Ci::Pipeline
end
desc 'Gets the variables for a given pipeline' do
detail 'This feature was introduced in GitLab 11.11'
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
@@ -106,19 +106,17 @@ module API
get ':id/pipelines/:pipeline_id/variables' do
authorize! :read_pipeline_variable, pipeline
- present pipeline.variables, with: Entities::Variable
+ present pipeline.variables, with: Entities::Ci::Variable
end
desc 'Gets the test report for a given pipeline' do
- detail 'This feature was introduced in GitLab 13.0. Disabled by default behind feature flag `junit_pipeline_view`'
+ detail 'This feature was introduced in GitLab 13.0.'
success TestReportEntity
end
params do
requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
end
get ':id/pipelines/:pipeline_id/test_report' do
- not_found! unless Feature.enabled?(:junit_pipeline_view, user_project)
-
authorize! :read_build, pipeline
present pipeline.test_reports, with: TestReportEntity, details: true
@@ -141,7 +139,7 @@ module API
desc 'Retry builds in the pipeline' do
detail 'This feature was introduced in GitLab 8.11.'
- success Entities::Pipeline
+ success Entities::Ci::Pipeline
end
params do
requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
@@ -151,12 +149,12 @@ module API
pipeline.retry_failed(current_user)
- present pipeline, with: Entities::Pipeline
+ present pipeline, with: Entities::Ci::Pipeline
end
desc 'Cancel all builds in the pipeline' do
detail 'This feature was introduced in GitLab 8.11.'
- success Entities::Pipeline
+ success Entities::Ci::Pipeline
end
params do
requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
@@ -167,14 +165,14 @@ module API
pipeline.cancel_running
status 200
- present pipeline.reset, with: Entities::Pipeline
+ present pipeline.reset, with: Entities::Ci::Pipeline
end
end
helpers do
def pipeline
strong_memoize(:pipeline) do
- user_project.ci_pipelines.find(params[:pipeline_id])
+ user_project.all_pipelines.find(params[:pipeline_id])
end
end
diff --git a/lib/api/ci/runners.rb b/lib/api/ci/runners.rb
index 2c156a71160..7bca72f8028 100644
--- a/lib/api/ci/runners.rb
+++ b/lib/api/ci/runners.rb
@@ -111,7 +111,7 @@ module API
end
desc 'List jobs running on a runner' do
- success Entities::JobBasicWithProject
+ success Entities::Ci::JobBasicWithProject
end
params do
requires :id, type: Integer, desc: 'The ID of the runner'
@@ -126,7 +126,7 @@ module API
jobs = ::Ci::RunnerJobsFinder.new(runner, params).execute
- present paginate(jobs), with: Entities::JobBasicWithProject
+ present paginate(jobs), with: Entities::Ci::JobBasicWithProject
end
end
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index 1a0fe393753..3c7ed2a25a0 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -13,7 +13,7 @@ module API
helpers do
def user_access
- @user_access ||= Gitlab::UserAccess.new(current_user, project: user_project)
+ @user_access ||= Gitlab::UserAccess.new(current_user, container: user_project)
end
def authorize_push_to_branch!(branch)
@@ -203,6 +203,7 @@ module API
params do
requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked'
requires :branch, type: String, desc: 'The name of the branch', allow_blank: false
+ optional :dry_run, type: Boolean, default: false, desc: "Does not commit any changes"
end
post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do
authorize_push_to_branch!(params[:branch])
@@ -215,7 +216,8 @@ module API
commit_params = {
commit: commit,
start_branch: params[:branch],
- branch_name: params[:branch]
+ branch_name: params[:branch],
+ dry_run: params[:dry_run]
}
result = ::Commits::CherryPickService
@@ -223,10 +225,18 @@ module API
.execute
if result[:status] == :success
- present user_project.repository.commit(result[:result]),
- with: Entities::Commit
+ if params[:dry_run]
+ present dry_run: :success
+ status :ok
+ else
+ present user_project.repository.commit(result[:result]),
+ with: Entities::Commit
+ end
else
- error!(result.slice(:message, :error_code), 400, header)
+ response = result.slice(:message, :error_code)
+ response[:dry_run] = :error if params[:dry_run]
+
+ error!(response, 400, header)
end
end
@@ -237,6 +247,7 @@ module API
params do
requires :sha, type: String, desc: 'Commit SHA to revert'
requires :branch, type: String, desc: 'Target branch name', allow_blank: false
+ optional :dry_run, type: Boolean, default: false, desc: "Does not commit any changes"
end
post ':id/repository/commits/:sha/revert', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do
authorize_push_to_branch!(params[:branch])
@@ -249,7 +260,8 @@ module API
commit_params = {
commit: commit,
start_branch: params[:branch],
- branch_name: params[:branch]
+ branch_name: params[:branch],
+ dry_run: params[:dry_run]
}
result = ::Commits::RevertService
@@ -257,10 +269,18 @@ module API
.execute
if result[:status] == :success
- present user_project.repository.commit(result[:result]),
- with: Entities::Commit
+ if params[:dry_run]
+ present dry_run: :success
+ status :ok
+ else
+ present user_project.repository.commit(result[:result]),
+ with: Entities::Commit
+ end
else
- error!(result.slice(:message, :error_code), 400, header)
+ response = result.slice(:message, :error_code)
+ response[:dry_run] = :error if params[:dry_run]
+
+ error!(response, 400, header)
end
end
diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb
index 726dc89271a..05887e58425 100644
--- a/lib/api/composer_packages.rb
+++ b/lib/api/composer_packages.rb
@@ -59,7 +59,7 @@ module API
desc 'Composer packages endpoint at group level'
- route_setting :authentication, job_token_allowed: true
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
get ':id/-/packages/composer/packages' do
presenter.root
@@ -71,7 +71,7 @@ module API
requires :sha, type: String, desc: 'Shasum of current json'
end
- route_setting :authentication, job_token_allowed: true
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
get ':id/-/packages/composer/p/:sha' do
presenter.provider
@@ -83,7 +83,7 @@ module API
requires :package_name, type: String, file_path: true, desc: 'The Composer package name'
end
- route_setting :authentication, job_token_allowed: true
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
get ':id/-/packages/composer/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true do
not_found! if packages.empty?
@@ -104,7 +104,7 @@ module API
desc 'Composer packages endpoint for registering packages'
namespace ':id/packages/composer' do
- route_setting :authentication, job_token_allowed: true
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
params do
optional :branch, type: String, desc: 'The name of the branch'
@@ -123,7 +123,7 @@ module API
bad_request!
end
- track_event('register_package')
+ track_event('push_package')
::Packages::Composer::CreatePackageService
.new(authorized_user_project, current_user, declared_params)
diff --git a/lib/api/conan_packages.rb b/lib/api/conan_packages.rb
index 1d941e422a7..6923d252fbd 100644
--- a/lib/api/conan_packages.rb
+++ b/lib/api/conan_packages.rb
@@ -38,7 +38,9 @@ module API
desc 'Ping the Conan API' do
detail 'This feature was introduced in GitLab 12.2'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get 'ping' do
header 'X-Conan-Server-Capabilities', [].join(',')
end
@@ -46,10 +48,13 @@ module API
desc 'Search for packages' do
detail 'This feature was introduced in GitLab 12.4'
end
+
params do
requires :q, type: String, desc: 'Search query'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get 'conans/search' do
service = ::Packages::Conan::SearchService.new(current_user, query: params[:q]).execute
service.payload
@@ -61,7 +66,9 @@ module API
desc 'Authenticate user against conan CLI' do
detail 'This feature was introduced in GitLab 12.2'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get 'authenticate' do
unauthorized! unless token
@@ -71,7 +78,9 @@ module API
desc 'Check for valid user credentials per conan CLI' do
detail 'This feature was introduced in GitLab 12.4'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get 'check_credentials' do
authenticate!
:ok
@@ -93,10 +102,13 @@ module API
desc 'Package Snapshot' do
detail 'This feature was introduced in GitLab 12.5'
end
+
params do
requires :conan_package_reference, type: String, desc: 'Conan package ID'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get 'packages/:conan_package_reference' do
authorize!(:read_package, project)
@@ -113,7 +125,9 @@ module API
desc 'Recipe Snapshot' do
detail 'This feature was introduced in GitLab 12.5'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get do
authorize!(:read_package, project)
@@ -133,7 +147,9 @@ module API
params do
requires :conan_package_reference, type: String, desc: 'Conan package ID'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get 'packages/:conan_package_reference/digest' do
present_package_download_urls
end
@@ -141,7 +157,9 @@ module API
desc 'Recipe Digest' do
detail 'This feature was introduced in GitLab 12.5'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get 'digest' do
present_recipe_download_urls
end
@@ -155,10 +173,13 @@ module API
desc 'Package Download Urls' do
detail 'This feature was introduced in GitLab 12.5'
end
+
params do
requires :conan_package_reference, type: String, desc: 'Conan package ID'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get 'packages/:conan_package_reference/download_urls' do
present_package_download_urls
end
@@ -166,7 +187,9 @@ module API
desc 'Recipe Download Urls' do
detail 'This feature was introduced in GitLab 12.5'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get 'download_urls' do
present_recipe_download_urls
end
@@ -181,36 +204,39 @@ module API
desc 'Package Upload Urls' do
detail 'This feature was introduced in GitLab 12.4'
end
+
params do
requires :conan_package_reference, type: String, desc: 'Conan package ID'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
post 'packages/:conan_package_reference/upload_urls' do
authorize!(:read_package, project)
status 200
- upload_urls = package_upload_urls(::Packages::Conan::FileMetadatum::PACKAGE_FILES)
-
- present upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls
+ present package_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls
end
desc 'Recipe Upload Urls' do
detail 'This feature was introduced in GitLab 12.4'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
post 'upload_urls' do
authorize!(:read_package, project)
status 200
- upload_urls = recipe_upload_urls(::Packages::Conan::FileMetadatum::RECIPE_FILES)
-
- present upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls
+ present recipe_upload_urls, with: ::API::Entities::ConanPackage::ConanUploadUrls
end
desc 'Delete Package' do
detail 'This feature was introduced in GitLab 12.5'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
delete do
authorize!(:destroy_package, project)
@@ -239,7 +265,9 @@ module API
desc 'Download recipe files' do
detail 'This feature was introduced in GitLab 12.6'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get do
download_package_file(:recipe_file)
end
@@ -247,10 +275,13 @@ module API
desc 'Upload recipe package files' do
detail 'This feature was introduced in GitLab 12.6'
end
+
params do
use :workhorse_upload_params
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
put do
upload_package_file(:recipe_file)
end
@@ -258,7 +289,9 @@ module API
desc 'Workhorse authorize the conan recipe file' do
detail 'This feature was introduced in GitLab 12.6'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
put 'authorize' do
authorize_workhorse!(subject: project)
end
@@ -273,7 +306,9 @@ module API
desc 'Download package files' do
detail 'This feature was introduced in GitLab 12.5'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
get do
download_package_file(:package_file)
end
@@ -281,7 +316,9 @@ module API
desc 'Workhorse authorize the conan package file' do
detail 'This feature was introduced in GitLab 12.6'
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
put 'authorize' do
authorize_workhorse!(subject: project)
end
@@ -289,10 +326,13 @@ module API
desc 'Upload package files' do
detail 'This feature was introduced in GitLab 12.6'
end
+
params do
use :workhorse_upload_params
end
- route_setting :authentication, job_token_allowed: true
+
+ route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true
+
put do
upload_package_file(:package_file)
end
diff --git a/lib/api/entities/branch.rb b/lib/api/entities/branch.rb
index f9d06082ad6..6a75dcddeda 100644
--- a/lib/api/entities/branch.rb
+++ b/lib/api/entities/branch.rb
@@ -32,7 +32,7 @@ module API
end
expose :can_push do |repo_branch, options|
- Gitlab::UserAccess.new(options[:current_user], project: options[:project]).can_push_to_branch?(repo_branch.name)
+ Gitlab::UserAccess.new(options[:current_user], container: options[:project]).can_push_to_branch?(repo_branch.name)
end
expose :default do |repo_branch, options|
diff --git a/lib/api/entities/bridge.rb b/lib/api/entities/bridge.rb
deleted file mode 100644
index 8f0ee69399a..00000000000
--- a/lib/api/entities/bridge.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class Bridge < Entities::JobBasic
- expose :downstream_pipeline, with: Entities::PipelineBasic
- end
- end
-end
diff --git a/lib/api/entities/ci/bridge.rb b/lib/api/entities/ci/bridge.rb
new file mode 100644
index 00000000000..502d97fff90
--- /dev/null
+++ b/lib/api/entities/ci/bridge.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class Bridge < JobBasic
+ expose :downstream_pipeline, with: ::API::Entities::Ci::PipelineBasic
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/job.rb b/lib/api/entities/ci/job.rb
new file mode 100644
index 00000000000..7fe1a802e24
--- /dev/null
+++ b/lib/api/entities/ci/job.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class Job < JobBasic
+ # artifacts_file is included in job_artifacts, but kept for backward compatibility (remove in api/v5)
+ expose :artifacts_file, using: ::API::Entities::Ci::JobArtifactFile, if: -> (job, opts) { job.artifacts? }
+ expose :job_artifacts, as: :artifacts, using: ::API::Entities::Ci::JobArtifact
+ expose :runner, with: ::API::Entities::Runner
+ expose :artifacts_expire_at
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/job_artifact.rb b/lib/api/entities/ci/job_artifact.rb
new file mode 100644
index 00000000000..9e504aee383
--- /dev/null
+++ b/lib/api/entities/ci/job_artifact.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class JobArtifact < Grape::Entity
+ expose :file_type, :size, :filename, :file_format
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/job_artifact_file.rb b/lib/api/entities/ci/job_artifact_file.rb
new file mode 100644
index 00000000000..418eb408ab6
--- /dev/null
+++ b/lib/api/entities/ci/job_artifact_file.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class JobArtifactFile < Grape::Entity
+ expose :filename
+ expose :cached_size, as: :size
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/job_basic.rb b/lib/api/entities/ci/job_basic.rb
new file mode 100644
index 00000000000..a29788c7abf
--- /dev/null
+++ b/lib/api/entities/ci/job_basic.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class JobBasic < Grape::Entity
+ expose :id, :status, :stage, :name, :ref, :tag, :coverage, :allow_failure
+ expose :created_at, :started_at, :finished_at
+ expose :duration
+ expose :user, with: ::API::Entities::User
+ expose :commit, with: ::API::Entities::Commit
+ expose :pipeline, with: ::API::Entities::Ci::PipelineBasic
+
+ expose :web_url do |job, _options|
+ Gitlab::Routing.url_helpers.project_job_url(job.project, job)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/job_basic_with_project.rb b/lib/api/entities/ci/job_basic_with_project.rb
new file mode 100644
index 00000000000..736e611e5b1
--- /dev/null
+++ b/lib/api/entities/ci/job_basic_with_project.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class JobBasicWithProject < Entities::Ci::JobBasic
+ expose :project, with: Entities::ProjectIdentity
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/pipeline.rb b/lib/api/entities/ci/pipeline.rb
new file mode 100644
index 00000000000..3dd3b9c9eff
--- /dev/null
+++ b/lib/api/entities/ci/pipeline.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class Pipeline < PipelineBasic
+ expose :before_sha, :tag, :yaml_errors
+
+ expose :user, with: Entities::UserBasic
+ expose :created_at, :updated_at, :started_at, :finished_at, :committed_at
+ expose :duration
+ expose :coverage
+ expose :detailed_status, using: DetailedStatusEntity do |pipeline, options|
+ pipeline.detailed_status(options[:current_user])
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/pipeline_basic.rb b/lib/api/entities/ci/pipeline_basic.rb
new file mode 100644
index 00000000000..dbb9b828757
--- /dev/null
+++ b/lib/api/entities/ci/pipeline_basic.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class PipelineBasic < Grape::Entity
+ expose :id, :sha, :ref, :status
+ expose :created_at, :updated_at
+
+ expose :web_url do |pipeline, _options|
+ Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/pipeline_schedule.rb b/lib/api/entities/ci/pipeline_schedule.rb
new file mode 100644
index 00000000000..f1596b7d285
--- /dev/null
+++ b/lib/api/entities/ci/pipeline_schedule.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class PipelineSchedule < Grape::Entity
+ expose :id
+ expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active
+ expose :created_at, :updated_at
+ expose :owner, using: ::API::Entities::UserBasic
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/pipeline_schedule_details.rb b/lib/api/entities/ci/pipeline_schedule_details.rb
new file mode 100644
index 00000000000..b233728b95b
--- /dev/null
+++ b/lib/api/entities/ci/pipeline_schedule_details.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class PipelineScheduleDetails < PipelineSchedule
+ expose :last_pipeline, using: ::API::Entities::Ci::PipelineBasic
+ expose :variables, using: ::API::Entities::Ci::Variable
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/ci/variable.rb b/lib/api/entities/ci/variable.rb
new file mode 100644
index 00000000000..f4d5248245a
--- /dev/null
+++ b/lib/api/entities/ci/variable.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Ci
+ class Variable < Grape::Entity
+ expose :variable_type, :key, :value
+ expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) }
+ expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) }
+ expose :environment_scope, if: -> (entity, _) { entity.respond_to?(:environment_scope) }
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/commit_detail.rb b/lib/api/entities/commit_detail.rb
index 22424b38bb9..61238102e9d 100644
--- a/lib/api/entities/commit_detail.rb
+++ b/lib/api/entities/commit_detail.rb
@@ -9,7 +9,7 @@ module API
expose :last_pipeline do |commit, options|
pipeline = commit.last_pipeline if can_read_pipeline?
- ::API::Entities::PipelineBasic.represent(pipeline, options)
+ ::API::Entities::Ci::PipelineBasic.represent(pipeline, options)
end
private
diff --git a/lib/api/entities/deployment.rb b/lib/api/entities/deployment.rb
index 3a97d3e3c09..4e3a4c289d9 100644
--- a/lib/api/entities/deployment.rb
+++ b/lib/api/entities/deployment.rb
@@ -6,7 +6,7 @@ module API
expose :id, :iid, :ref, :sha, :created_at, :updated_at
expose :user, using: Entities::UserBasic
expose :environment, using: Entities::EnvironmentBasic
- expose :deployable, using: Entities::Job
+ expose :deployable, using: Entities::Ci::Job
expose :status
end
end
diff --git a/lib/api/entities/event.rb b/lib/api/entities/event.rb
index 8fd0bac13f4..f750d728e03 100644
--- a/lib/api/entities/event.rb
+++ b/lib/api/entities/event.rb
@@ -3,6 +3,7 @@
module API
module Entities
class Event < Grape::Entity
+ expose :id
expose :project_id, :action_name
expose :target_id, :target_iid, :target_type, :author_id
expose :target_title
diff --git a/lib/api/entities/feature.rb b/lib/api/entities/feature.rb
index 3c9182340ea..618a7be9c7b 100644
--- a/lib/api/entities/feature.rb
+++ b/lib/api/entities/feature.rb
@@ -10,7 +10,7 @@ module API
value = model.gate_values[gate.key]
# By default all gate values are populated. Only show relevant ones.
- if (value.is_a?(Integer) && value.zero?) || (value.is_a?(Set) && value.empty?)
+ if (value.is_a?(Integer) && value == 0) || (value.is_a?(Set) && value.empty?)
next
end
diff --git a/lib/api/entities/job.rb b/lib/api/entities/job.rb
deleted file mode 100644
index cbee8794007..00000000000
--- a/lib/api/entities/job.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class Job < Entities::JobBasic
- # artifacts_file is included in job_artifacts, but kept for backward compatibility (remove in api/v5)
- expose :artifacts_file, using: Entities::JobArtifactFile, if: -> (job, opts) { job.artifacts? }
- expose :job_artifacts, as: :artifacts, using: Entities::JobArtifact
- expose :runner, with: Entities::Runner
- expose :artifacts_expire_at
- end
- end
-end
diff --git a/lib/api/entities/job_artifact.rb b/lib/api/entities/job_artifact.rb
deleted file mode 100644
index 94dbdb38fee..00000000000
--- a/lib/api/entities/job_artifact.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class JobArtifact < Grape::Entity
- expose :file_type, :size, :filename, :file_format
- end
- end
-end
diff --git a/lib/api/entities/job_artifact_file.rb b/lib/api/entities/job_artifact_file.rb
deleted file mode 100644
index fa2851a7f0e..00000000000
--- a/lib/api/entities/job_artifact_file.rb
+++ /dev/null
@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class JobArtifactFile < Grape::Entity
- expose :filename
- expose :cached_size, as: :size
- end
- end
-end
diff --git a/lib/api/entities/job_basic.rb b/lib/api/entities/job_basic.rb
deleted file mode 100644
index a8541039934..00000000000
--- a/lib/api/entities/job_basic.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class JobBasic < Grape::Entity
- expose :id, :status, :stage, :name, :ref, :tag, :coverage, :allow_failure
- expose :created_at, :started_at, :finished_at
- expose :duration
- expose :user, with: Entities::User
- expose :commit, with: Entities::Commit
- expose :pipeline, with: Entities::PipelineBasic
-
- expose :web_url do |job, _options|
- Gitlab::Routing.url_helpers.project_job_url(job.project, job)
- end
- end
- end
-end
diff --git a/lib/api/entities/job_basic_with_project.rb b/lib/api/entities/job_basic_with_project.rb
deleted file mode 100644
index 09387e045ec..00000000000
--- a/lib/api/entities/job_basic_with_project.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class JobBasicWithProject < Entities::JobBasic
- expose :project, with: Entities::ProjectIdentity
- end
- end
-end
diff --git a/lib/api/entities/job_request/dependency.rb b/lib/api/entities/job_request/dependency.rb
index 64d779f6575..7d6ec832ba1 100644
--- a/lib/api/entities/job_request/dependency.rb
+++ b/lib/api/entities/job_request/dependency.rb
@@ -5,7 +5,7 @@ module API
module JobRequest
class Dependency < Grape::Entity
expose :id, :name, :token
- expose :artifacts_file, using: Entities::JobArtifactFile, if: ->(job, _) { job.artifacts? }
+ expose :artifacts_file, using: Entities::Ci::JobArtifactFile, if: ->(job, _) { job.artifacts? }
end
end
end
diff --git a/lib/api/entities/job_request/response.rb b/lib/api/entities/job_request/response.rb
index fdacd3af2da..8db9aff3dc9 100644
--- a/lib/api/entities/job_request/response.rb
+++ b/lib/api/entities/job_request/response.rb
@@ -33,3 +33,5 @@ module API
end
end
end
+
+API::Entities::JobRequest::Response.prepend_if_ee('EE::API::Entities::JobRequest::Response')
diff --git a/lib/api/entities/merge_request.rb b/lib/api/entities/merge_request.rb
index 7fc76a4071e..05ae041c7a9 100644
--- a/lib/api/entities/merge_request.rb
+++ b/lib/api/entities/merge_request.rb
@@ -23,11 +23,11 @@ module API
merge_request.metrics&.first_deployed_to_production_at
end
- expose :pipeline, using: Entities::PipelineBasic, if: -> (_, options) { build_available?(options) } do |merge_request, _options|
+ expose :pipeline, using: Entities::Ci::PipelineBasic, if: -> (_, options) { build_available?(options) } do |merge_request, _options|
merge_request.metrics&.pipeline
end
- expose :head_pipeline, using: 'API::Entities::Pipeline', if: -> (_, options) do
+ expose :head_pipeline, using: '::API::Entities::Ci::Pipeline', if: -> (_, options) do
Ability.allowed?(options[:current_user], :read_pipeline, options[:project])
end
diff --git a/lib/api/entities/package.rb b/lib/api/entities/package.rb
index 73473f16da9..670965b225c 100644
--- a/lib/api/entities/package.rb
+++ b/lib/api/entities/package.rb
@@ -13,9 +13,7 @@ module API
expose :_links do
expose :web_path do |package|
- if ::Gitlab.ee?
- ::Gitlab::Routing.url_helpers.project_package_path(package.project, package)
- end
+ ::Gitlab::Routing.url_helpers.project_package_path(package.project, package)
end
expose :delete_api_path, if: can_destroy(:package, &:project) do |package|
diff --git a/lib/api/entities/package/pipeline.rb b/lib/api/entities/package/pipeline.rb
index e91a12e47fa..0aa888e30ee 100644
--- a/lib/api/entities/package/pipeline.rb
+++ b/lib/api/entities/package/pipeline.rb
@@ -3,7 +3,7 @@
module API
module Entities
class Package < Grape::Entity
- class Pipeline < ::API::Entities::PipelineBasic
+ class Pipeline < ::API::Entities::Ci::PipelineBasic
expose :user, using: ::API::Entities::UserBasic
end
end
diff --git a/lib/api/entities/personal_access_token.rb b/lib/api/entities/personal_access_token.rb
index d6fb9af6ab3..3846929c903 100644
--- a/lib/api/entities/personal_access_token.rb
+++ b/lib/api/entities/personal_access_token.rb
@@ -3,7 +3,7 @@
module API
module Entities
class PersonalAccessToken < Grape::Entity
- expose :id, :name, :revoked, :created_at, :scopes
+ expose :id, :name, :revoked, :created_at, :scopes, :user_id
expose :active?, as: :active
expose :expires_at do |personal_access_token|
personal_access_token.expires_at ? personal_access_token.expires_at.strftime("%Y-%m-%d") : nil
diff --git a/lib/api/entities/pipeline.rb b/lib/api/entities/pipeline.rb
deleted file mode 100644
index 778efbe4bcc..00000000000
--- a/lib/api/entities/pipeline.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class Pipeline < Entities::PipelineBasic
- expose :before_sha, :tag, :yaml_errors
-
- expose :user, with: Entities::UserBasic
- expose :created_at, :updated_at, :started_at, :finished_at, :committed_at
- expose :duration
- expose :coverage
- expose :detailed_status, using: DetailedStatusEntity do |pipeline, options|
- pipeline.detailed_status(options[:current_user])
- end
- end
- end
-end
diff --git a/lib/api/entities/pipeline_basic.rb b/lib/api/entities/pipeline_basic.rb
deleted file mode 100644
index 359f6a447ab..00000000000
--- a/lib/api/entities/pipeline_basic.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class PipelineBasic < Grape::Entity
- expose :id, :sha, :ref, :status
- expose :created_at, :updated_at
-
- expose :web_url do |pipeline, _options|
- Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline)
- end
- end
- end
-end
diff --git a/lib/api/entities/pipeline_schedule.rb b/lib/api/entities/pipeline_schedule.rb
deleted file mode 100644
index a72fe3f3141..00000000000
--- a/lib/api/entities/pipeline_schedule.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class PipelineSchedule < Grape::Entity
- expose :id
- expose :description, :ref, :cron, :cron_timezone, :next_run_at, :active
- expose :created_at, :updated_at
- expose :owner, using: Entities::UserBasic
- end
- end
-end
diff --git a/lib/api/entities/pipeline_schedule_details.rb b/lib/api/entities/pipeline_schedule_details.rb
deleted file mode 100644
index 5e54489a0f9..00000000000
--- a/lib/api/entities/pipeline_schedule_details.rb
+++ /dev/null
@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class PipelineScheduleDetails < Entities::PipelineSchedule
- expose :last_pipeline, using: Entities::PipelineBasic
- expose :variables, using: Entities::Variable
- end
- end
-end
diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb
index e3c5177cd0b..fb599d68d72 100644
--- a/lib/api/entities/project.rb
+++ b/lib/api/entities/project.rb
@@ -35,6 +35,7 @@ module API
end
end
+ expose :packages_enabled
expose :empty_repo?, as: :empty_repo
expose :archived?, as: :archived
expose :visibility
diff --git a/lib/api/entities/project_hook.rb b/lib/api/entities/project_hook.rb
index cdd3714ed64..751f9500252 100644
--- a/lib/api/entities/project_hook.rb
+++ b/lib/api/entities/project_hook.rb
@@ -4,7 +4,7 @@ module API
module Entities
class ProjectHook < Hook
expose :project_id, :issues_events, :confidential_issues_events
- expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events
+ expose :note_events, :confidential_note_events, :pipeline_events, :wiki_page_events, :deployment_events
expose :job_events
expose :push_events_branch_filter
end
diff --git a/lib/api/entities/user_details_with_admin.rb b/lib/api/entities/user_details_with_admin.rb
index 22a842983e2..e48b1da7859 100644
--- a/lib/api/entities/user_details_with_admin.rb
+++ b/lib/api/entities/user_details_with_admin.rb
@@ -6,6 +6,7 @@ module API
expose :highest_role
expose :current_sign_in_ip
expose :last_sign_in_ip
+ expose :sign_in_count
end
end
end
diff --git a/lib/api/entities/variable.rb b/lib/api/entities/variable.rb
deleted file mode 100644
index 6705df30b2e..00000000000
--- a/lib/api/entities/variable.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-module API
- module Entities
- class Variable < Grape::Entity
- expose :variable_type, :key, :value
- expose :protected?, as: :protected, if: -> (entity, _) { entity.respond_to?(:protected?) }
- expose :masked?, as: :masked, if: -> (entity, _) { entity.respond_to?(:masked?) }
- expose :environment_scope, if: -> (entity, _) { entity.respond_to?(:environment_scope) }
- end
- end
-end
diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb
index d3ca1c79e73..e7b8cd10197 100644
--- a/lib/api/group_variables.rb
+++ b/lib/api/group_variables.rb
@@ -13,18 +13,18 @@ module API
resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get group-level variables' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
use :pagination
end
get ':id/variables' do
variables = user_group.variables
- present paginate(variables), with: Entities::Variable
+ present paginate(variables), with: Entities::Ci::Variable
end
desc 'Get a specific variable from a group' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :key, type: String, desc: 'The key of the variable'
@@ -36,12 +36,12 @@ module API
break not_found!('GroupVariable') unless variable
- present variable, with: Entities::Variable
+ present variable, with: Entities::Ci::Variable
end
# rubocop: enable CodeReuse/ActiveRecord
desc 'Create a new variable in a group' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :key, type: String, desc: 'The key of the variable'
@@ -51,19 +51,21 @@ module API
optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var'
end
post ':id/variables' do
- variable_params = declared_params(include_missing: false)
-
- variable = user_group.variables.create(variable_params)
+ variable = ::Ci::ChangeVariableService.new(
+ container: user_group,
+ current_user: current_user,
+ params: { action: :create, variable_params: declared_params(include_missing: false) }
+ ).execute
if variable.valid?
- present variable, with: Entities::Variable
+ present variable, with: Entities::Ci::Variable
else
render_validation_error!(variable)
end
end
desc 'Update an existing variable from a group' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
optional :key, type: String, desc: 'The key of the variable'
@@ -74,32 +76,41 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
put ':id/variables/:key' do
- variable = user_group.variables.find_by(key: params[:key])
-
- break not_found!('GroupVariable') unless variable
-
- variable_params = declared_params(include_missing: false).except(:key)
+ variable = ::Ci::ChangeVariableService.new(
+ container: user_group,
+ current_user: current_user,
+ params: { action: :update, variable_params: declared_params(include_missing: false) }
+ ).execute
- if variable.update(variable_params)
- present variable, with: Entities::Variable
+ if variable.valid?
+ present variable, with: Entities::Ci::Variable
else
render_validation_error!(variable)
end
+ rescue ::ActiveRecord::RecordNotFound
+ not_found!('GroupVariable')
end
# rubocop: enable CodeReuse/ActiveRecord
desc 'Delete an existing variable from a group' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :key, type: String, desc: 'The key of the variable'
end
# rubocop: disable CodeReuse/ActiveRecord
delete ':id/variables/:key' do
- variable = user_group.variables.find_by(key: params[:key])
- not_found!('GroupVariable') unless variable
-
- destroy_conditionally!(variable)
+ variable = user_group.variables.find_by!(key: params[:key])
+
+ destroy_conditionally!(variable) do |target_variable|
+ ::Ci::ChangeVariableService.new(
+ container: user_group,
+ current_user: current_user,
+ params: { action: :destroy, variable_params: declared_params(include_missing: false) }
+ ).execute
+ end
+ rescue ::ActiveRecord::RecordNotFound
+ not_found!('GroupVariable')
end
# rubocop: enable CodeReuse/ActiveRecord
end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 9ac3ac818fc..813e41b4d39 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -76,7 +76,7 @@ module API
params: project_finder_params,
options: finder_options
).execute
- projects = reorder_projects(projects)
+ projects = reorder_projects_with_similarity_order_support(group, projects)
paginate(projects)
end
@@ -112,6 +112,24 @@ module API
accepted!
end
+
+ def reorder_projects_with_similarity_order_support(group, projects)
+ return handle_similarity_order(group, projects) if params[:order_by] == 'similarity'
+
+ reorder_projects(projects)
+ end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def handle_similarity_order(group, projects)
+ if params[:search].present? && Feature.enabled?(:similarity_search, group, default_enabled: true)
+ projects.sorted_by_similarity_desc(params[:search])
+ else
+ order_options = { name: :asc }
+ order_options['id'] ||= params[:sort] || 'asc'
+ projects.reorder(order_options)
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
end
resource :groups do
@@ -222,7 +240,7 @@ module API
optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
desc: 'Limit by visibility'
optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
- optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
+ optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at similarity],
default: 'created_at', desc: 'Return projects ordered by field'
optional :sort, type: String, values: %w[asc desc], default: 'desc',
desc: 'Return projects sorted in ascending and descending order'
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 01b89959c14..599d5bd0baf 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -404,6 +404,10 @@ module API
render_api_error!(message || '409 Conflict', 409)
end
+ def unprocessable_entity!(message = nil)
+ render_api_error!(message || '422 Unprocessable Entity', :unprocessable_entity)
+ end
+
def file_too_large!
render_api_error!('413 Request Entity Too Large', 413)
end
@@ -555,8 +559,8 @@ module API
finder_params[:search_namespaces] = true if params[:search_namespaces].present?
finder_params[:user] = params.delete(:user) if params[:user]
finder_params[:custom_attributes] = params[:custom_attributes] if params[:custom_attributes]
- finder_params[:id_after] = params[:id_after] if params[:id_after]
- finder_params[:id_before] = params[:id_before] if params[:id_before]
+ finder_params[:id_after] = sanitize_id_param(params[:id_after]) if params[:id_after]
+ finder_params[:id_before] = sanitize_id_param(params[:id_before]) if params[:id_before]
finder_params[:last_activity_after] = params[:last_activity_after] if params[:last_activity_after]
finder_params[:last_activity_before] = params[:last_activity_before] if params[:last_activity_before]
finder_params[:repository_storage] = params[:repository_storage] if params[:repository_storage]
@@ -655,6 +659,10 @@ module API
def ip_address
env["action_dispatch.remote_ip"].to_s || request.ip
end
+
+ def sanitize_id_param(id)
+ id.present? ? id.to_i : nil
+ end
end
end
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index b69930b447c..b7ce1eba3f9 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -117,7 +117,7 @@ module API
return unless %w[git-receive-pack git-upload-pack git-upload-archive].include?(action)
{
- repository: repository.gitaly_repository,
+ repository: repository.gitaly_repository.to_h,
address: Gitlab::GitalyClient.address(repository.shard),
token: Gitlab::GitalyClient.token(repository.shard),
features: Feature::Gitaly.server_feature_flags
diff --git a/lib/api/helpers/merge_requests_helpers.rb b/lib/api/helpers/merge_requests_helpers.rb
index 4d5350498a7..e4163c63575 100644
--- a/lib/api/helpers/merge_requests_helpers.rb
+++ b/lib/api/helpers/merge_requests_helpers.rb
@@ -4,6 +4,9 @@ module API
module Helpers
module MergeRequestsHelpers
extend Grape::API::Helpers
+ extend ActiveSupport::Concern
+
+ UNPROCESSABLE_ERROR_KEYS = [:project_access, :branch_conflict, :validate_fork, :base].freeze
params :merge_requests_negatable_params do
optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID'
@@ -79,6 +82,20 @@ module API
default: 'created_by_me',
desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`'
end
+
+ def handle_merge_request_errors!(merge_request)
+ return if merge_request.valid?
+
+ errors = merge_request.errors
+
+ UNPROCESSABLE_ERROR_KEYS.each do |error|
+ unprocessable_entity!(errors[error]) if errors.has_key?(error)
+ end
+
+ conflict!(errors[:validate_branches]) if errors.has_key?(:validate_branches)
+
+ render_validation_error!(merge_request)
+ end
end
end
end
diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb
index f88624ed63e..f61bcfe963e 100644
--- a/lib/api/helpers/notes_helpers.rb
+++ b/lib/api/helpers/notes_helpers.rb
@@ -17,8 +17,9 @@ module API
authorize! :admin_note, note
opts = {
- note: params[:body]
- }
+ note: params[:body],
+ confidential: params[:confidential]
+ }.compact
parent = noteable_parent(noteable)
project = parent if parent.is_a?(Project)
diff --git a/lib/api/helpers/packages/basic_auth_helpers.rb b/lib/api/helpers/packages/basic_auth_helpers.rb
index 835b5f4614c..e35a8712131 100644
--- a/lib/api/helpers/packages/basic_auth_helpers.rb
+++ b/lib/api/helpers/packages/basic_auth_helpers.rb
@@ -4,6 +4,8 @@ module API
module Helpers
module Packages
module BasicAuthHelpers
+ extend ::Gitlab::Utils::Override
+
module Constants
AUTHENTICATE_REALM_HEADER = 'Www-Authenticate: Basic realm'
AUTHENTICATE_REALM_NAME = 'GitLab Packages Registry'
@@ -11,10 +13,6 @@ module API
include Constants
- def find_personal_access_token
- find_personal_access_token_from_http_basic_auth
- end
-
def unauthorized_user_project
@unauthorized_user_project ||= find_project(params[:id])
end
@@ -44,12 +42,13 @@ module API
end
def unauthorized_or!
- current_user ? yield : unauthorized_with_header!
+ current_user ? yield : unauthorized!
end
- def unauthorized_with_header!
+ override :unauthorized!
+ def unauthorized!
header(AUTHENTICATE_REALM_HEADER, AUTHENTICATE_REALM_NAME)
- unauthorized!
+ super
end
end
end
diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb
index 30e690a5a1d..a5fde1af41e 100644
--- a/lib/api/helpers/packages/conan/api_helpers.rb
+++ b/lib/api/helpers/packages/conan/api_helpers.rb
@@ -28,22 +28,30 @@ module API
present_download_urls(::API::Entities::ConanPackage::ConanRecipeManifest, &:recipe_urls)
end
- def recipe_upload_urls(file_names)
+ def recipe_upload_urls
{ upload_urls: Hash[
- file_names.collect do |file_name|
+ file_names.select(&method(:recipe_file?)).map do |file_name|
[file_name, recipe_file_upload_url(file_name)]
end
] }
end
- def package_upload_urls(file_names)
+ def package_upload_urls
{ upload_urls: Hash[
- file_names.collect do |file_name|
+ file_names.select(&method(:package_file?)).map do |file_name|
[file_name, package_file_upload_url(file_name)]
end
] }
end
+ def recipe_file?(file_name)
+ file_name.in?(::Packages::Conan::FileMetadatum::RECIPE_FILES)
+ end
+
+ def package_file?(file_name)
+ file_name.in?(::Packages::Conan::FileMetadatum::PACKAGE_FILES)
+ end
+
def package_file_upload_url(file_name)
expose_url(
api_v4_packages_conan_v1_files_package_path(
@@ -130,6 +138,14 @@ module API
end
end
+ def file_names
+ json_payload = Gitlab::Json.parse(request.body.string)
+
+ bad_request!(nil) unless json_payload.is_a?(Hash)
+
+ json_payload.keys
+ end
+
def create_package_file_with_type(file_type, current_package)
unless params['file.size'] == 0
# conan sends two upload requests, the first has no file, so we skip record creation if file.size == 0
diff --git a/lib/api/helpers/packages_manager_clients_helpers.rb b/lib/api/helpers/packages_manager_clients_helpers.rb
index 7b5d0dd708d..ae16b65aaa8 100644
--- a/lib/api/helpers/packages_manager_clients_helpers.rb
+++ b/lib/api/helpers/packages_manager_clients_helpers.rb
@@ -16,16 +16,6 @@ module API
optional 'file.sha256', type: String, desc: 'SHA256 checksum of the file (generated by Workhorse)'
end
- def find_personal_access_token_from_http_basic_auth
- return unless headers
-
- token = decode_token
-
- return unless token
-
- PersonalAccessToken.find_by_token(token)
- end
-
def find_job_from_http_basic_auth
return unless headers
diff --git a/lib/api/helpers/pagination_strategies.rb b/lib/api/helpers/pagination_strategies.rb
index 823891d6fe7..61cff37e4ab 100644
--- a/lib/api/helpers/pagination_strategies.rb
+++ b/lib/api/helpers/pagination_strategies.rb
@@ -48,7 +48,7 @@ module API
end
def offset_limit_exceeded?(offset_limit)
- offset_limit.positive? && params[:page] * params[:per_page] > offset_limit
+ offset_limit > 0 && params[:page] * params[:per_page] > offset_limit
end
end
end
diff --git a/lib/api/helpers/performance_bar_helpers.rb b/lib/api/helpers/performance_bar_helpers.rb
new file mode 100644
index 00000000000..8430e889dff
--- /dev/null
+++ b/lib/api/helpers/performance_bar_helpers.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module API
+ module Helpers
+ module PerformanceBarHelpers
+ def set_peek_enabled_for_current_request
+ Gitlab::SafeRequestStore.fetch(:peek_enabled) { perf_bar_cookie_enabled? && perf_bar_enabled_for_user? }
+ end
+
+ def perf_bar_cookie_enabled?
+ cookies[:perf_bar_enabled] == 'true'
+ end
+
+ def perf_bar_enabled_for_user?
+ # We cannot use `current_user` here because that method raises an exception when the user
+ # is unauthorized and some API endpoints require that `current_user` is not called.
+ Gitlab::PerformanceBar.enabled_for_user?(find_user_from_sources)
+ end
+ end
+ end
+end
diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb
index 76e5bb95c4d..8c20f5b8fc2 100644
--- a/lib/api/helpers/projects_helpers.rb
+++ b/lib/api/helpers/projects_helpers.rb
@@ -61,6 +61,7 @@ module API
optional :auto_devops_deploy_strategy, type: String, values: %w(continuous manual timed_incremental), desc: 'Auto Deploy strategy'
optional :autoclose_referenced_issues, type: Boolean, desc: 'Flag indication if referenced issues auto-closing is enabled'
optional :repository_storage, type: String, desc: 'Which storage shard the repository is on. Available only to admins'
+ optional :packages_enabled, type: Boolean, desc: 'Enable project packages feature'
end
params :optional_project_params_ee do
@@ -137,6 +138,7 @@ module API
:suggestion_commit_message,
:repository_storage,
:compliance_framework_setting,
+ :packages_enabled,
:service_desk_enabled,
# TODO: remove in API v5, replaced by *_access_level
diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb
index d4870b96575..ff938358439 100644
--- a/lib/api/helpers/services_helpers.rb
+++ b/lib/api/helpers/services_helpers.rb
@@ -247,15 +247,15 @@ module API
required: true,
name: :project_url,
type: String,
- desc: 'The buildkite project URL'
+ desc: 'The Buildkite pipeline URL'
},
{
required: false,
name: :enable_ssl_verification,
type: Boolean,
- desc: 'Enable SSL verification for communication'
+ desc: 'DEPRECATED: This parameter has no effect since SSL verification will always be enabled'
}
- ],
+ ],
'campfire' => [
{
required: true,
diff --git a/lib/api/helpers/snippets_helpers.rb b/lib/api/helpers/snippets_helpers.rb
index f95d066bd7c..79367da8d1f 100644
--- a/lib/api/helpers/snippets_helpers.rb
+++ b/lib/api/helpers/snippets_helpers.rb
@@ -10,6 +10,23 @@ module API
requires :ref, type: String, desc: 'The name of branch, tag or commit'
end
+ params :create_file_params do
+ optional :files, type: Array, desc: 'An array of files' do
+ requires :file_path, type: String, file_path: true, allow_blank: false, desc: 'The path of a snippet file'
+ requires :content, type: String, allow_blank: false, desc: 'The content of a snippet file'
+ end
+
+ optional :content, type: String, allow_blank: false, desc: 'The content of a snippet'
+
+ given :content do
+ requires :file_name, type: String, desc: 'The name of a snippet file'
+ end
+
+ mutually_exclusive :files, :content
+
+ exactly_one_of :files, :content
+ end
+
def content_for(snippet)
if snippet.empty_repo?
env['api.format'] = :txt
@@ -35,5 +52,12 @@ module API
send_git_blob(repo, blob)
end
end
+
+ def process_file_args(args)
+ args[:snippet_actions] = args.delete(:files)&.map do |file|
+ file[:action] = :create
+ file.symbolize_keys
+ end
+ end
end
end
diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb
index 1e839816006..0bab891eada 100644
--- a/lib/api/import_github.rb
+++ b/lib/api/import_github.rb
@@ -10,7 +10,11 @@ module API
helpers do
def client
- @client ||= Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options)
+ @client ||= if Feature.enabled?(:remove_legacy_github_client)
+ Gitlab::GithubImport::Client.new(params[:personal_access_token])
+ else
+ Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], client_options)
+ end
end
def access_params
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index 6d4a4fc9c8b..17599c72243 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -18,6 +18,10 @@ module API
UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result'.freeze
+ VALID_PAT_SCOPES = Set.new(
+ Gitlab::Auth::API_SCOPES + Gitlab::Auth::REPOSITORY_SCOPES + Gitlab::Auth::REGISTRY_SCOPES
+ ).freeze
+
helpers do
def response_with_status(code: 200, success: true, message: nil, **extra_options)
status code
@@ -67,7 +71,7 @@ module API
"uploadpack.allowAnySHA1InWant=true"],
gitaly: gitaly_payload(params[:action]),
gl_console_messages: check_result.console_messages
- }
+ }.merge!(actor.key_details)
# Custom option for git-receive-pack command
@@ -92,7 +96,7 @@ module API
# If we have created a project directly from a git push
# we have to assign its value to both @project and @container
- @project = @container = access_checker.project
+ @project = @container = access_checker.container
end
end
end
@@ -194,6 +198,60 @@ module API
{ success: true, recovery_codes: codes }
end
+ post '/personal_access_token' do
+ status 200
+
+ actor.update_last_used_at!
+ user = actor.user
+
+ if params[:key_id]
+ unless actor.key
+ break { success: false, message: 'Could not find the given key' }
+ end
+
+ if actor.key.is_a?(DeployKey)
+ break { success: false, message: 'Deploy keys cannot be used to create personal access tokens' }
+ end
+
+ unless user
+ break { success: false, message: 'Could not find a user for the given key' }
+ end
+ elsif params[:user_id] && user.nil?
+ break { success: false, message: 'Could not find the given user' }
+ end
+
+ if params[:name].blank?
+ break { success: false, message: "No token name specified" }
+ end
+
+ if params[:scopes].blank?
+ break { success: false, message: "No token scopes specified" }
+ end
+
+ invalid_scope = params[:scopes].find { |scope| VALID_PAT_SCOPES.exclude?(scope.to_sym) }
+
+ if invalid_scope
+ valid_scopes = VALID_PAT_SCOPES.map(&:to_s).sort
+ break { success: false, message: "Invalid scope: '#{invalid_scope}'. Valid scopes are: #{valid_scopes}" }
+ end
+
+ begin
+ expires_at = params[:expires_at].presence && Date.parse(params[:expires_at])
+ rescue ArgumentError
+ break { success: false, message: "Invalid token expiry date: '#{params[:expires_at]}'" }
+ end
+
+ access_token = nil
+
+ ::Users::UpdateService.new(current_user, user: user).execute! do |user|
+ access_token = user.personal_access_tokens.create!(
+ name: params[:name], scopes: params[:scopes], expires_at: expires_at
+ )
+ end
+
+ { success: true, token: access_token.token, scopes: access_token.scopes, expires_at: access_token.expires_at }
+ end
+
post '/pre_receive' do
status 200
diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb
new file mode 100644
index 00000000000..7f64fd7efe3
--- /dev/null
+++ b/lib/api/internal/kubernetes.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module API
+ # Kubernetes Internal API
+ module Internal
+ class Kubernetes < Grape::API::Instance
+ helpers do
+ def agent_token
+ @agent_token ||= cluster_agent_token_from_authorization_token
+ end
+
+ def agent
+ @agent ||= agent_token.agent
+ end
+
+ def repo_type
+ Gitlab::GlRepository::PROJECT
+ end
+
+ def gitaly_info(project)
+ shard = repo_type.repository_for(project).shard
+ {
+ address: Gitlab::GitalyClient.address(shard),
+ token: Gitlab::GitalyClient.token(shard),
+ features: Feature::Gitaly.server_feature_flags
+ }
+ end
+
+ def gitaly_repository(project)
+ {
+ storage_name: project.repository_storage,
+ relative_path: project.disk_path + '.git',
+ gl_repository: repo_type.identifier_for_container(project),
+ gl_project_path: repo_type.repository_for(project).full_path
+ }
+ end
+
+ def check_feature_enabled
+ not_found! unless Feature.enabled?(:kubernetes_agent_internal_api)
+ end
+
+ def check_agent_token
+ forbidden! unless agent_token
+ end
+ end
+
+ namespace 'internal' do
+ namespace 'kubernetes' do
+ before do
+ check_feature_enabled
+ check_agent_token
+ end
+
+ desc 'Gets agent info' do
+ detail 'Retrieves agent info for the given token'
+ end
+ route_setting :authentication, cluster_agent_token_allowed: true
+ get '/agent_info' do
+ project = agent.project
+
+ status 200
+ {
+ project_id: project.id,
+ agent_id: agent.id,
+ agent_name: agent.name,
+ gitaly_info: gitaly_info(project),
+ gitaly_repository: gitaly_repository(project)
+ }
+ end
+
+ desc 'Gets project info' do
+ detail 'Retrieves project info (if authorized)'
+ end
+ route_setting :authentication, cluster_agent_token_allowed: true
+ get '/project_info' do
+ project = find_project(params[:id])
+
+ # TODO sort out authorization for real
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/220912
+ if !project || !project.public?
+ not_found!
+ end
+
+ status 200
+ {
+ project_id: project.id,
+ gitaly_info: gitaly_info(project),
+ gitaly_repository: gitaly_repository(project)
+ }
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 455511caabb..1694a967f26 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -55,6 +55,8 @@ module API
desc: 'Return issues ordered by `created_at` or `updated_at` fields.'
optional :sort, type: String, values: %w[asc desc], default: 'desc',
desc: 'Return issues sorted in `asc` or `desc` order.'
+ optional :due_date, type: String, values: %w[0 overdue week month next_month_and_previous_two_weeks] << '',
+ desc: 'Return issues that have no due date (`0`), or whose due date is this week, this month, between two weeks ago and next month, or which are overdue. Accepts: `overdue`, `week`, `month`, `next_month_and_previous_two_weeks`, `0`'
use :issues_stats_params
use :pagination
diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb
index 61c279a76e9..bc7bc956580 100644
--- a/lib/api/job_artifacts.rb
+++ b/lib/api/job_artifacts.rb
@@ -94,7 +94,7 @@ module API
end
desc 'Keep the artifacts to prevent them from being deleted' do
- success Entities::Job
+ success ::API::Entities::Ci::Job
end
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
@@ -109,7 +109,7 @@ module API
build.keep_artifacts!
status 200
- present build, with: Entities::Job
+ present build, with: ::API::Entities::Ci::Job
end
desc 'Delete the artifacts files from a job' do
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index bcc00429dd6..084c146abe7 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -30,7 +30,7 @@ module API
end
desc 'Get a projects jobs' do
- success Entities::Job
+ success Entities::Ci::Job
end
params do
use :optional_scope
@@ -44,12 +44,12 @@ module API
builds = filter_builds(builds, params[:scope])
builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, pipeline: :project)
- present paginate(builds), with: Entities::Job
+ present paginate(builds), with: Entities::Ci::Job
end
# rubocop: enable CodeReuse/ActiveRecord
desc 'Get pipeline jobs' do
- success Entities::Job
+ success Entities::Ci::Job
end
params do
requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
@@ -59,19 +59,19 @@ module API
# rubocop: disable CodeReuse/ActiveRecord
get ':id/pipelines/:pipeline_id/jobs' do
authorize!(:read_pipeline, user_project)
- pipeline = user_project.ci_pipelines.find(params[:pipeline_id])
+ pipeline = user_project.all_pipelines.find(params[:pipeline_id])
authorize!(:read_build, pipeline)
builds = pipeline.builds
builds = filter_builds(builds, params[:scope])
builds = builds.preload(:job_artifacts_archive, :job_artifacts, project: [:namespace])
- present paginate(builds), with: Entities::Job
+ present paginate(builds), with: Entities::Ci::Job
end
# rubocop: enable CodeReuse/ActiveRecord
desc 'Get pipeline bridge jobs' do
- success Entities::Bridge
+ success ::API::Entities::Ci::Bridge
end
params do
requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
@@ -92,12 +92,12 @@ module API
project: [:namespace]
)
- present paginate(bridges), with: Entities::Bridge
+ present paginate(bridges), with: ::API::Entities::Ci::Bridge
end
# rubocop: enable CodeReuse/ActiveRecord
desc 'Get a specific job of a project' do
- success Entities::Job
+ success Entities::Ci::Job
end
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
@@ -107,7 +107,7 @@ module API
build = find_build!(params[:job_id])
- present build, with: Entities::Job
+ present build, with: Entities::Ci::Job
end
# TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace
@@ -131,7 +131,7 @@ module API
end
desc 'Cancel a specific job of a project' do
- success Entities::Job
+ success Entities::Ci::Job
end
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
@@ -144,11 +144,11 @@ module API
build.cancel
- present build, with: Entities::Job
+ present build, with: Entities::Ci::Job
end
desc 'Retry a specific build of a project' do
- success Entities::Job
+ success Entities::Ci::Job
end
params do
requires :job_id, type: Integer, desc: 'The ID of a build'
@@ -162,11 +162,11 @@ module API
build = ::Ci::Build.retry(build, current_user)
- present build, with: Entities::Job
+ present build, with: Entities::Ci::Job
end
desc 'Erase job (remove artifacts and the trace)' do
- success Entities::Job
+ success Entities::Ci::Job
end
params do
requires :job_id, type: Integer, desc: 'The ID of a build'
@@ -179,11 +179,11 @@ module API
break forbidden!('Job is not erasable!') unless build.erasable?
build.erase(erased_by: current_user)
- present build, with: Entities::Job
+ present build, with: Entities::Ci::Job
end
desc 'Trigger a actionable job (manual, delayed, etc)' do
- success Entities::Job
+ success Entities::Ci::Job
detail 'This feature was added in GitLab 8.11'
end
params do
@@ -200,7 +200,7 @@ module API
build.play(current_user)
status 200
- present build, with: Entities::Job
+ present build, with: Entities::Ci::Job
end
end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 2e6ac40a593..6f25df720c4 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -97,7 +97,7 @@ module API
user_access = Gitlab::UserAccess.new(
current_user,
- project: merge_request.source_project
+ container: merge_request.source_project
)
forbidden!('Cannot push to source branch') unless
@@ -153,22 +153,6 @@ module API
include TimeTrackingEndpoints
helpers do
- def handle_merge_request_errors!(errors)
- if errors[:project_access].any?
- error!(errors[:project_access], 422)
- elsif errors[:branch_conflict].any?
- error!(errors[:branch_conflict], 422)
- elsif errors[:validate_fork].any?
- error!(errors[:validate_fork], 422)
- elsif errors[:validate_branches].any?
- conflict!(errors[:validate_branches])
- elsif errors[:base].any?
- error!(errors[:base], 422)
- end
-
- render_api_error!(errors, 400)
- end
-
params :optional_params do
optional :description, type: String, desc: 'The description of the merge request'
optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request'
@@ -226,11 +210,9 @@ module API
merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute
- if merge_request.valid?
- present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project
- else
- handle_merge_request_errors! merge_request.errors
- end
+ handle_merge_request_errors!(merge_request)
+
+ present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project
end
desc 'Delete a merge request'
@@ -370,16 +352,16 @@ module API
end
desc 'Get the merge request pipelines' do
- success Entities::PipelineBasic
+ success Entities::Ci::PipelineBasic
end
get ':id/merge_requests/:merge_request_iid/pipelines' do
pipelines = merge_request_pipelines_with_access
- present paginate(pipelines), with: Entities::PipelineBasic
+ present paginate(pipelines), with: Entities::Ci::PipelineBasic
end
desc 'Create a pipeline for merge request' do
- success Entities::Pipeline
+ success ::API::Entities::Ci::Pipeline
end
post ':id/merge_requests/:merge_request_iid/pipelines' do
pipeline = ::MergeRequests::CreatePipelineService
@@ -390,7 +372,7 @@ module API
not_allowed!
elsif pipeline.persisted?
status :ok
- present pipeline, with: Entities::Pipeline
+ present pipeline, with: ::API::Entities::Ci::Pipeline
else
render_validation_error!(pipeline)
end
@@ -420,11 +402,9 @@ module API
merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request)
- if merge_request.valid?
- present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project
- else
- handle_merge_request_errors! merge_request.errors
- end
+ handle_merge_request_errors!(merge_request)
+
+ present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project
end
desc 'Merge a merge request' do
diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb
index 8ff885983bc..b337b992841 100644
--- a/lib/api/milestone_responses.rb
+++ b/lib/api/milestone_responses.rb
@@ -14,10 +14,12 @@ module API
params :list_params do
optional :state, type: String, values: %w[active closed all], default: 'all',
- desc: 'Return "active", "closed", or "all" milestones'
+ desc: 'Return "active", "closed", or "all" milestones'
optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IIDs of the milestones'
optional :title, type: String, desc: 'The title of the milestones'
optional :search, type: String, desc: 'The search criteria for the title or description of the milestone'
+ optional :include_parent_milestones, type: Grape::API::Boolean, default: false,
+ desc: 'Include group milestones from parent and its ancestors'
use :pagination
end
@@ -25,15 +27,18 @@ module API
requires :milestone_id, type: Integer, desc: 'The milestone ID number'
optional :title, type: String, desc: 'The title of the milestone'
optional :state_event, type: String, values: %w[close activate],
- desc: 'The state event of the milestone '
+ desc: 'The state event of the milestone '
use :optional_params
at_least_one_of :title, :description, :start_date, :due_date, :state_event
end
def list_milestones_for(parent)
- milestones = parent.milestones.order_id_desc
+ milestones = init_milestones_collection(parent)
milestones = Milestone.filter_by_state(milestones, params[:state])
- milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present?
+ if params[:iids].present? && !params[:include_parent_milestones]
+ milestones = filter_by_iid(milestones, params[:iids])
+ end
+
milestones = filter_by_title(milestones, params[:title]) if params[:title]
milestones = filter_by_search(milestones, params[:search]) if params[:search]
@@ -96,6 +101,41 @@ module API
[MergeRequestsFinder, Entities::MergeRequestBasic]
end
end
+
+ def init_milestones_collection(parent)
+ milestones = if params[:include_parent_milestones].present?
+ parent_and_ancestors_milestones(parent)
+ else
+ parent.milestones
+ end
+
+ milestones.order_id_desc
+ end
+
+ def parent_and_ancestors_milestones(parent)
+ project_id, group_ids = if parent.is_a?(Project)
+ [parent.id, project_group_ids(parent)]
+ else
+ [nil, parent_group_ids(parent)]
+ end
+
+ Milestone.for_projects_and_groups(project_id, group_ids)
+ end
+
+ def project_group_ids(parent)
+ group = parent.group
+ return unless group.present?
+
+ group.self_and_ancestors.select(:id)
+ end
+
+ def parent_group_ids(group)
+ return unless group.present?
+
+ group.self_and_ancestors
+ .public_or_visible_to_user(current_user)
+ .select(:id)
+ end
end
end
end
diff --git a/lib/api/notes.rb b/lib/api/notes.rb
index bfd09dcd496..e4989243f3d 100644
--- a/lib/api/notes.rb
+++ b/lib/api/notes.rb
@@ -101,7 +101,8 @@ module API
params do
requires :noteable_id, type: Integer, desc: 'The ID of the noteable'
requires :note_id, type: Integer, desc: 'The ID of a note'
- requires :body, type: String, desc: 'The content of a note'
+ optional :body, type: String, allow_blank: false, desc: 'The content of a note'
+ optional :confidential, type: Boolean, desc: 'Confidentiality note flag'
end
put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do
noteable = find_noteable(noteable_type, params[:noteable_id])
diff --git a/lib/api/nuget_packages.rb b/lib/api/nuget_packages.rb
index eb7d320a0f5..56c4de2071d 100644
--- a/lib/api/nuget_packages.rb
+++ b/lib/api/nuget_packages.rb
@@ -54,7 +54,9 @@ module API
params do
requires :id, type: String, desc: 'The ID of a project', regexp: POSITIVE_INTEGER_REGEX
end
- route_setting :authentication, deploy_token_allowed: true
+
+ route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
+
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
before do
authorized_user_project
@@ -65,7 +67,9 @@ module API
desc 'The NuGet Service Index' do
detail 'This feature was introduced in GitLab 12.6'
end
- route_setting :authentication, deploy_token_allowed: true
+
+ route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
+
get 'index', format: :json do
authorize_read_package!(authorized_user_project)
@@ -79,10 +83,13 @@ module API
desc 'The NuGet Package Publish endpoint' do
detail 'This feature was introduced in GitLab 12.6'
end
+
params do
requires :package, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)'
end
- route_setting :authentication, deploy_token_allowed: true
+
+ route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
+
put do
authorize_upload!(authorized_user_project)
@@ -107,7 +114,9 @@ module API
forbidden!
end
- route_setting :authentication, deploy_token_allowed: true
+
+ route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
+
put 'authorize' do
authorize_workhorse!(subject: authorized_user_project, has_length: false)
end
@@ -124,7 +133,9 @@ module API
desc 'The NuGet Metadata Service - Package name level' do
detail 'This feature was introduced in GitLab 12.8'
end
- route_setting :authentication, deploy_token_allowed: true
+
+ route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
+
get 'index', format: :json do
present ::Packages::Nuget::PackagesMetadataPresenter.new(find_packages),
with: ::API::Entities::Nuget::PackagesMetadata
@@ -136,7 +147,9 @@ module API
params do
requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX
end
- route_setting :authentication, deploy_token_allowed: true
+
+ route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
+
get '*package_version', format: :json do
present ::Packages::Nuget::PackageMetadataPresenter.new(find_package),
with: ::API::Entities::Nuget::PackageMetadata
@@ -155,7 +168,9 @@ module API
desc 'The NuGet Content Service - index request' do
detail 'This feature was introduced in GitLab 12.8'
end
- route_setting :authentication, deploy_token_allowed: true
+
+ route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
+
get 'index', format: :json do
present ::Packages::Nuget::PackagesVersionsPresenter.new(find_packages),
with: ::API::Entities::Nuget::PackagesVersions
@@ -168,7 +183,9 @@ module API
requires :package_version, type: String, desc: 'The NuGet package version', regexp: API::NO_SLASH_URL_PART_REGEX
requires :package_filename, type: String, desc: 'The NuGet package filename', regexp: API::NO_SLASH_URL_PART_REGEX
end
- route_setting :authentication, deploy_token_allowed: true
+
+ route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
+
get '*package_version/*package_filename', format: :nupkg do
filename = "#{params[:package_filename]}.#{params[:format]}"
package_file = ::Packages::PackageFileFinder.new(find_package, filename, with_file_name_like: true)
@@ -198,7 +215,9 @@ module API
desc 'The NuGet Search Service' do
detail 'This feature was introduced in GitLab 12.8'
end
- route_setting :authentication, deploy_token_allowed: true
+
+ route_setting :authentication, deploy_token_allowed: true, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true
+
get format: :json do
search_options = {
include_prerelease_versions: params[:prerelease],
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index d11c47f8d78..377d61689b3 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -47,6 +47,8 @@ module API
post ':id/export' do
check_rate_limit! :project_export, [current_user]
+ user_project.remove_exports
+
project_export_params = declared_params(include_missing: false)
after_export_params = project_export_params.delete(:upload) || {}
diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb
index 7cea44e6304..e68a3b106b1 100644
--- a/lib/api/project_hooks.rb
+++ b/lib/api/project_hooks.rb
@@ -20,6 +20,7 @@ module API
optional :job_events, type: Boolean, desc: "Trigger hook on job events"
optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events"
optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events"
+ optional :deployment_events, type: Boolean, desc: "Trigger hook on deployment events"
optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook"
optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response"
optional :push_events_branch_filter, type: String, desc: "Trigger hook on specified branch only"
diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb
index 09934502e85..fba4c60504f 100644
--- a/lib/api/project_snippets.rb
+++ b/lib/api/project_snippets.rb
@@ -56,16 +56,20 @@ module API
end
params do
requires :title, type: String, allow_blank: false, desc: 'The title of the snippet'
- requires :file_name, type: String, desc: 'The file name of the snippet'
- requires :content, type: String, allow_blank: false, desc: 'The content of the snippet'
optional :description, type: String, desc: 'The description of a snippet'
requires :visibility, type: String,
values: Gitlab::VisibilityLevel.string_values,
desc: 'The visibility of the snippet'
+ use :create_file_params
end
post ":id/snippets" do
authorize! :create_snippet, user_project
- snippet_params = declared_params(include_missing: false).merge(request: request, api: true)
+ snippet_params = declared_params(include_missing: false).tap do |create_args|
+ create_args[:request] = request
+ create_args[:api] = true
+
+ process_file_args(create_args)
+ end
service_response = ::Snippets::CreateService.new(user_project, current_user, snippet_params).execute
snippet = service_response.payload[:snippet]
diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb
index f0fe4d85c8f..48c3dbed3b0 100644
--- a/lib/api/project_templates.rb
+++ b/lib/api/project_templates.rb
@@ -4,7 +4,7 @@ module API
class ProjectTemplates < Grape::API::Instance
include PaginationParams
- TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses].freeze
+ TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses metrics_dashboard_ymls issues merge_requests].freeze
# The regex is needed to ensure a period (e.g. agpl-3.0)
# isn't confused with a format type. We also need to allow encoded
# values (e.g. C%2B%2B for C++), so allow % and + as well.
@@ -14,7 +14,7 @@ module API
params do
requires :id, type: String, desc: 'The ID of a project'
- requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses) of the template'
+ requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses|metrics_dashboard_ymls|issues|merge_requests) of the template'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get a list of templates available to this project' do
@@ -42,9 +42,13 @@ module API
end
get ':id/templates/:type/:name', requirements: TEMPLATE_NAMES_ENDPOINT_REQUIREMENTS do
- template = TemplateFinder
- .build(params[:type], user_project, name: params[:name])
- .execute
+ begin
+ template = TemplateFinder
+ .build(params[:type], user_project, name: params[:name])
+ .execute
+ rescue ::Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
+ not_found!('Template')
+ end
not_found!('Template') unless template.present?
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index d24dab63bd9..abbdb11a3f7 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -448,7 +448,7 @@ module API
.execute.map { |lang| [lang.name, lang.share] }.to_h
end
- desc 'Remove a project'
+ desc 'Delete a project'
delete ":id" do
authorize! :remove_project, user_project
diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb
index a6caacd7df8..739928a61ed 100644
--- a/lib/api/pypi_packages.rb
+++ b/lib/api/pypi_packages.rb
@@ -22,10 +22,6 @@ module API
render_api_error!(e.message, 400)
end
- rescue_from ActiveRecord::RecordInvalid do |e|
- render_api_error!(e.message, 400)
- end
-
helpers do
def packages_finder(project = authorized_user_project)
project
@@ -68,7 +64,7 @@ module API
requires :sha256, type: String, desc: 'The PyPi package sha256 check sum'
end
- route_setting :authentication, deploy_token_allowed: true
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
get 'files/:sha256/*file_identifier' do
project = unauthorized_user_project!
@@ -91,7 +87,7 @@ module API
# An Api entry point but returns an HTML file instead of JSON.
# PyPi simple API returns the package descriptor as a simple HTML file.
- route_setting :authentication, deploy_token_allowed: true
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
get 'simple/*package_name', format: :txt do
authorize_read_package!(authorized_user_project)
@@ -121,7 +117,7 @@ module API
optional :sha256_digest, type: String
end
- route_setting :authentication, deploy_token_allowed: true
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
post do
authorize_upload!(authorized_user_project)
@@ -138,7 +134,7 @@ module API
forbidden!
end
- route_setting :authentication, deploy_token_allowed: true
+ route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true
post 'authorize' do
authorize_workhorse!(subject: authorized_user_project, has_length: false)
end
diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb
index 7e1815480a5..9624b8924e5 100644
--- a/lib/api/release/links.rb
+++ b/lib/api/release/links.rb
@@ -40,7 +40,7 @@ module API
requires :name, type: String, desc: 'The name of the link'
requires :url, type: String, desc: 'The URL of the link'
optional :filepath, type: String, desc: 'The filepath of the link'
- optional :link_type, type: String, desc: 'The link type'
+ optional :link_type, type: String, desc: 'The link type, one of: "runbook", "image", "package" or "other"'
end
post 'links' do
authorize! :create_release, release
diff --git a/lib/api/releases.rb b/lib/api/releases.rb
index 30c5e06053e..3c38721129f 100644
--- a/lib/api/releases.rb
+++ b/lib/api/releases.rb
@@ -50,8 +50,10 @@ module API
optional :ref, type: String, desc: 'The commit sha or branch name'
optional :assets, type: Hash do
optional :links, type: Array do
- requires :name, type: String
- requires :url, type: String
+ requires :name, type: String, desc: 'The name of the link'
+ requires :url, type: String, desc: 'The URL of the link'
+ optional :filepath, type: String, desc: 'The filepath of the link'
+ optional :link_type, type: String, desc: 'The link type, one of: "runbook", "image", "package" or "other"'
end
end
optional :milestones, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The titles of the related milestones', default: []
diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb
index 118045e3af2..1a3283aed98 100644
--- a/lib/api/snippets.rb
+++ b/lib/api/snippets.rb
@@ -66,18 +66,23 @@ module API
end
params do
requires :title, type: String, allow_blank: false, desc: 'The title of a snippet'
- requires :file_name, type: String, desc: 'The name of a snippet file'
- requires :content, type: String, allow_blank: false, desc: 'The content of a snippet'
optional :description, type: String, desc: 'The description of a snippet'
optional :visibility, type: String,
values: Gitlab::VisibilityLevel.string_values,
default: 'internal',
desc: 'The visibility of the snippet'
+ use :create_file_params
end
post do
authorize! :create_snippet
- attrs = declared_params(include_missing: false).merge(request: request, api: true)
+ attrs = declared_params(include_missing: false).tap do |create_args|
+ create_args[:request] = request
+ create_args[:api] = true
+
+ process_file_args(create_args)
+ end
+
service_response = ::Snippets::CreateService.new(nil, current_user, attrs).execute
snippet = service_response.payload[:snippet]
diff --git a/lib/api/support/git_access_actor.rb b/lib/api/support/git_access_actor.rb
index cb9bf4472eb..39730da1251 100644
--- a/lib/api/support/git_access_actor.rb
+++ b/lib/api/support/git_access_actor.rb
@@ -39,6 +39,15 @@ module API
def update_last_used_at!
key&.update_last_used_at
end
+
+ def key_details
+ return {} unless key
+
+ {
+ gl_key_type: key.model_name.singular,
+ gl_key_id: key.id
+ }
+ end
end
end
end
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
index de67a149274..f398bbf3e32 100644
--- a/lib/api/triggers.rb
+++ b/lib/api/triggers.rb
@@ -11,7 +11,7 @@ module API
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Trigger a GitLab project pipeline' do
- success Entities::Pipeline
+ success Entities::Ci::Pipeline
end
params do
requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false
@@ -38,7 +38,7 @@ module API
if result[:http_status]
render_api_error!(result[:message], result[:http_status])
else
- present result[:pipeline], with: Entities::Pipeline
+ present result[:pipeline], with: Entities::Ci::Pipeline
end
end
diff --git a/lib/api/users.rb b/lib/api/users.rb
index 7942777287b..335624963aa 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -218,7 +218,7 @@ module API
user_params = declared_params(include_missing: false)
- user_params[:password_expires_at] = Time.now if user_params[:password].present?
+ user_params[:password_expires_at] = Time.current if user_params[:password].present?
result = ::Users::UpdateService.new(current_user, user_params.merge(user: user)).execute
if result[:status] == :success
diff --git a/lib/api/validations/validators/file_path.rb b/lib/api/validations/validators/file_path.rb
index fee71373170..8a815c3b2b8 100644
--- a/lib/api/validations/validators/file_path.rb
+++ b/lib/api/validations/validators/file_path.rb
@@ -5,10 +5,12 @@ module API
module Validators
class FilePath < Grape::Validations::Base
def validate_param!(attr_name, params)
+ options = @option.is_a?(Hash) ? @option : {}
+ path_allowlist = options.fetch(:allowlist, [])
path = params[attr_name]
-
- Gitlab::Utils.check_path_traversal!(path)
- rescue ::Gitlab::Utils::PathTraversalAttackError
+ path = Gitlab::Utils.check_path_traversal!(path)
+ Gitlab::Utils.check_allowed_absolute_path!(path, path_allowlist)
+ rescue
raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)],
message: "should be a valid file path"
end
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index 50d137ec7c1..cea0cb3a19c 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -30,18 +30,18 @@ module API
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get project variables' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
use :pagination
end
get ':id/variables' do
variables = user_project.variables
- present paginate(variables), with: Entities::Variable
+ present paginate(variables), with: Entities::Ci::Variable
end
desc 'Get a specific variable from a project' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :key, type: String, desc: 'The key of the variable'
@@ -51,12 +51,12 @@ module API
variable = find_variable(params)
not_found!('Variable') unless variable
- present variable, with: Entities::Variable
+ present variable, with: Entities::Ci::Variable
end
# rubocop: enable CodeReuse/ActiveRecord
desc 'Create a new variable in a project' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :key, type: String, desc: 'The key of the variable'
@@ -67,20 +67,21 @@ module API
optional :environment_scope, type: String, desc: 'The environment_scope of the variable'
end
post ':id/variables' do
- variable_params = declared_params(include_missing: false)
- variable_params = filter_variable_parameters(variable_params)
-
- variable = user_project.variables.create(variable_params)
+ variable = ::Ci::ChangeVariableService.new(
+ container: user_project,
+ current_user: current_user,
+ params: { action: :create, variable_params: filter_variable_parameters(declared_params(include_missing: false)) }
+ ).execute
if variable.valid?
- present variable, with: Entities::Variable
+ present variable, with: Entities::Ci::Variable
else
render_validation_error!(variable)
end
end
desc 'Update an existing variable from a project' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
optional :key, type: String, desc: 'The key of the variable'
@@ -96,11 +97,18 @@ module API
variable = find_variable(params)
not_found!('Variable') unless variable
- variable_params = declared_params(include_missing: false).except(:key, :filter)
- variable_params = filter_variable_parameters(variable_params)
+ variable_params = filter_variable_parameters(
+ declared_params(include_missing: false)
+ .except(:key, :filter)
+ )
+ variable = ::Ci::ChangeVariableService.new(
+ container: user_project,
+ current_user: current_user,
+ params: { action: :update, variable: variable, variable_params: variable_params }
+ ).execute
- if variable.update(variable_params)
- present variable, with: Entities::Variable
+ if variable.valid?
+ present variable, with: Entities::Ci::Variable
else
render_validation_error!(variable)
end
@@ -108,7 +116,7 @@ module API
# rubocop: enable CodeReuse/ActiveRecord
desc 'Delete an existing variable from a project' do
- success Entities::Variable
+ success Entities::Ci::Variable
end
params do
requires :key, type: String, desc: 'The key of the variable'
@@ -119,8 +127,11 @@ module API
variable = find_variable(params)
not_found!('Variable') unless variable
- # Variables don't have a timestamp. Therefore, destroy unconditionally.
- variable.destroy
+ ::Ci::ChangeVariableService.new(
+ container: user_project,
+ current_user: current_user,
+ params: { action: :destroy, variable: variable }
+ ).execute
no_content!
end
diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb
index 713136e0887..95afa36113c 100644
--- a/lib/api/wikis.rb
+++ b/lib/api/wikis.rb
@@ -61,9 +61,10 @@ module API
post ':id/wikis' do
authorize! :create_wiki, container
- page = WikiPages::CreateService.new(container: container, current_user: current_user, params: params).execute
+ response = WikiPages::CreateService.new(container: container, current_user: current_user, params: params).execute
+ page = response.payload[:page]
- if page.valid?
+ if response.success?
present page, with: Entities::WikiPage
else
render_validation_error!(page)
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index 5e784dadb14..dae9056a47b 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -26,7 +26,7 @@ module Backup
cmd = %W(rsync -a --exclude=lost+found #{app_files_dir} #{Gitlab.config.backup.path})
output, status = Gitlab::Popen.popen(cmd)
- unless status.zero?
+ unless status == 0
puts output
raise Backup::Error, 'Backup failed'
end
diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index 1c5108b12ab..1daa29f00ef 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -10,34 +10,31 @@ module Backup
@progress = progress
end
- def dump
+ def dump(max_concurrency:, max_storage_concurrency:)
prepare
- Project.find_each(batch_size: 1000) do |project|
- progress.print " * #{display_repo_path(project)} ... "
-
- if project.hashed_storage?(:repository)
- FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
- else
- FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
- end
+ if max_concurrency <= 1 && max_storage_concurrency <= 1
+ return dump_consecutive
+ end
- if !empty_repo?(project)
- backup_project(project)
- progress.puts "[DONE]".color(:green)
- else
- progress.puts "[SKIPPED]".color(:cyan)
- end
+ if Project.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists?
+ raise Error, 'repositories.storages in gitlab.yml is misconfigured'
+ end
- wiki = ProjectWiki.new(project)
+ semaphore = Concurrent::Semaphore.new(max_concurrency)
+ errors = Queue.new
- if !empty_repo?(wiki)
- backup_project(wiki)
- progress.puts "[DONE] Wiki".color(:green)
- else
- progress.puts "[SKIPPED] Wiki".color(:cyan)
+ threads = Gitlab.config.repositories.storages.keys.map do |storage|
+ Thread.new do
+ dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency)
+ rescue => e
+ errors << e
end
end
+
+ threads.each(&:join)
+
+ raise errors.pop unless errors.empty?
end
def backup_project(project)
@@ -146,6 +143,71 @@ module Backup
private
+ def dump_consecutive
+ Project.find_each(batch_size: 1000) do |project|
+ dump_project(project)
+ end
+ end
+
+ def dump_storage(storage, semaphore, max_storage_concurrency:)
+ errors = Queue.new
+ queue = SizedQueue.new(1)
+
+ threads = Array.new(max_storage_concurrency) do
+ Thread.new do
+ while project = queue.pop
+ semaphore.acquire
+
+ begin
+ dump_project(project)
+ rescue => e
+ errors << e
+ break
+ ensure
+ semaphore.release
+ end
+ end
+ end
+ end
+
+ Project.for_repository_storage(storage).find_each(batch_size: 100) do |project|
+ break unless errors.empty?
+
+ queue.push(project)
+ end
+
+ queue.close
+ threads.each(&:join)
+
+ raise errors.pop unless errors.empty?
+ end
+
+ def dump_project(project)
+ progress.puts " * #{display_repo_path(project)} ... "
+
+ if project.hashed_storage?(:repository)
+ FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path)))
+ else
+ FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace
+ end
+
+ if !empty_repo?(project)
+ backup_project(project)
+ progress.puts " * #{display_repo_path(project)} ... " + "[DONE]".color(:green)
+ else
+ progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED]".color(:cyan)
+ end
+
+ wiki = ProjectWiki.new(project)
+
+ if !empty_repo?(wiki)
+ backup_project(wiki)
+ progress.puts " * #{display_repo_path(project)} ... " + "[DONE] Wiki".color(:green)
+ else
+ progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED] Wiki".color(:cyan)
+ end
+ end
+
def progress_warn(project, cmd, output)
progress.puts "[WARNING] Executing #{cmd}".color(:orange)
progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange)
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
index 38105e2237c..b0a2f6f69d5 100644
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ b/lib/banzai/filter/abstract_reference_filter.rb
@@ -136,7 +136,7 @@ module Banzai
end
def call
- return doc unless project || group
+ return doc unless project || group || user
ref_pattern = object_class.reference_pattern
link_pattern = object_class.link_reference_pattern
@@ -280,7 +280,7 @@ module Banzai
end
def object_link_text(object, matches)
- parent = context[:project] || context[:group]
+ parent = project || group || user
text = object.reference_link_text(parent)
extras = object_link_text_extras(object, matches)
diff --git a/lib/banzai/filter/ascii_doc_post_processing_filter.rb b/lib/banzai/filter/ascii_doc_post_processing_filter.rb
index 88439f06b5f..09f0fd7df45 100644
--- a/lib/banzai/filter/ascii_doc_post_processing_filter.rb
+++ b/lib/banzai/filter/ascii_doc_post_processing_filter.rb
@@ -8,6 +8,9 @@ module Banzai
node.set_attribute('class', 'code math js-render-math')
end
+ doc.search('[data-mermaid-style]').each do |node|
+ node.set_attribute('class', 'js-render-mermaid')
+ end
doc
end
end
diff --git a/lib/banzai/filter/base_sanitization_filter.rb b/lib/banzai/filter/base_sanitization_filter.rb
index 1b7af8aee45..fc3791e0cbf 100644
--- a/lib/banzai/filter/base_sanitization_filter.rb
+++ b/lib/banzai/filter/base_sanitization_filter.rb
@@ -25,7 +25,7 @@ module Banzai
# Allow data-math-style attribute in order to support LaTeX formatting
whitelist[:attributes]['code'] = %w(data-math-style)
- whitelist[:attributes]['pre'] = %w(data-math-style)
+ whitelist[:attributes]['pre'] = %w(data-math-style data-mermaid-style)
# Allow html5 details/summary elements
whitelist[:elements].push('details')
diff --git a/lib/banzai/filter/design_reference_filter.rb b/lib/banzai/filter/design_reference_filter.rb
index 7455dfe00ef..2ab47c5c6db 100644
--- a/lib/banzai/filter/design_reference_filter.rb
+++ b/lib/banzai/filter/design_reference_filter.rb
@@ -114,7 +114,7 @@ module Banzai
end
def enabled?
- Feature.enabled?(FEATURE_FLAG, parent)
+ Feature.enabled?(FEATURE_FLAG, parent, default_enabled: true)
end
end
end
diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb
index 033e3d2c33e..7928272a2cf 100644
--- a/lib/banzai/filter/gollum_tags_filter.rb
+++ b/lib/banzai/filter/gollum_tags_filter.rb
@@ -82,7 +82,7 @@ module Banzai
def process_tag(tag)
parts = tag.split('|')
- return if parts.size.zero?
+ return if parts.empty?
process_image_tag(parts) || process_page_link_tag(parts)
end
diff --git a/lib/banzai/filter/inline_alert_metrics_filter.rb b/lib/banzai/filter/inline_alert_metrics_filter.rb
new file mode 100644
index 00000000000..a6140d1ac81
--- /dev/null
+++ b/lib/banzai/filter/inline_alert_metrics_filter.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ # HTML filter that inserts a placeholder element for each
+ # reference to an alert dashboard.
+ class InlineAlertMetricsFilter < ::Banzai::Filter::InlineEmbedsFilter
+ include ::Gitlab::Routing
+ # Search params for selecting alert metrics links. A few
+ # simple checks is enough to boost performance without
+ # the cost of doing a full regex match.
+ def xpath_search
+ "descendant-or-self::a[contains(@href,'metrics_dashboard') and \
+ contains(@href,'prometheus/alerts') and \
+ starts-with(@href, '#{gitlab_domain}')]"
+ end
+
+ # Regular expression matching alert dashboard urls
+ def link_pattern
+ ::Gitlab::Metrics::Dashboard::Url.alert_regex
+ end
+
+ private
+
+ # Endpoint FE should hit to collect the appropriate
+ # chart information
+ def metrics_dashboard_url(params)
+ metrics_dashboard_namespace_project_prometheus_alert_url(
+ params['namespace'],
+ params['project'],
+ params['alert'],
+ embedded: true,
+ format: :json,
+ **query_params(params['url'])
+ )
+ end
+
+ # Parses query params out from full url string into hash.
+ #
+ # Ex) 'https://<root>/<project>/metrics_dashboard?title=Title&group=Group'
+ # --> { title: 'Title', group: 'Group' }
+ def query_params(url)
+ ::Gitlab::Metrics::Dashboard::Url.parse_query(url)
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/inline_cluster_metrics_filter.rb b/lib/banzai/filter/inline_cluster_metrics_filter.rb
index 5ef68388ea9..a696d3a6f9c 100644
--- a/lib/banzai/filter/inline_cluster_metrics_filter.rb
+++ b/lib/banzai/filter/inline_cluster_metrics_filter.rb
@@ -15,7 +15,7 @@ module Banzai
def xpath_search
"descendant-or-self::a[contains(@href,'clusters') and \
- starts-with(@href, '#{::Gitlab.config.gitlab.url}')]"
+ starts-with(@href, '#{gitlab_domain}')]"
end
def link_pattern
diff --git a/lib/banzai/filter/inline_embeds_filter.rb b/lib/banzai/filter/inline_embeds_filter.rb
index d7d78cf1866..cb9a493e8c6 100644
--- a/lib/banzai/filter/inline_embeds_filter.rb
+++ b/lib/banzai/filter/inline_embeds_filter.rb
@@ -82,6 +82,10 @@ module Banzai
def metrics_dashboard_url
raise NotImplementedError
end
+
+ def gitlab_domain
+ ::Gitlab.config.gitlab.url
+ end
end
end
end
diff --git a/lib/banzai/filter/inline_metrics_filter.rb b/lib/banzai/filter/inline_metrics_filter.rb
index 409e8db87f4..543d98e62be 100644
--- a/lib/banzai/filter/inline_metrics_filter.rb
+++ b/lib/banzai/filter/inline_metrics_filter.rb
@@ -11,7 +11,7 @@ module Banzai
def xpath_search
"descendant-or-self::a[contains(@href,'metrics') and \
contains(@href,'environments') and \
- starts-with(@href, '#{Gitlab.config.gitlab.url}')]"
+ starts-with(@href, '#{gitlab_domain}')]"
end
# Regular expression matching metrics urls
diff --git a/lib/banzai/filter/inline_metrics_redactor_filter.rb b/lib/banzai/filter/inline_metrics_redactor_filter.rb
index 7f98a52d421..2259115acfc 100644
--- a/lib/banzai/filter/inline_metrics_redactor_filter.rb
+++ b/lib/banzai/filter/inline_metrics_redactor_filter.rb
@@ -81,6 +81,10 @@ module Banzai
Route.new(
::Gitlab::Metrics::Dashboard::Url.clusters_regex,
:read_cluster
+ ),
+ Route.new(
+ ::Gitlab::Metrics::Dashboard::Url.alert_regex,
+ :read_prometheus_alerts
)
]
end
@@ -147,5 +151,3 @@ module Banzai
end
end
end
-
-Banzai::Filter::InlineMetricsRedactorFilter.prepend_if_ee('EE::Banzai::Filter::InlineMetricsRedactorFilter')
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index dc4b865bfb6..216418ee5fa 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -23,10 +23,6 @@ module Banzai
issue_url(issue, project)
end
- def projects_relation_for_paths(paths)
- super(paths).includes(:gitlab_issue_tracker_service)
- end
-
def parent_records(parent, ids)
parent.issues.where(iid: ids.to_a)
end
diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb
index 2ab88620df1..a4d3e352051 100644
--- a/lib/banzai/filter/label_reference_filter.rb
+++ b/lib/banzai/filter/label_reference_filter.rb
@@ -19,7 +19,7 @@ module Banzai
unescaped_html = unescape_html_entities(text).gsub(pattern) do |match|
namespace, project = $~[:namespace], $~[:project]
project_path = full_project_path(namespace, project)
- label = find_label(project_path, $~[:label_id], $~[:label_name])
+ label = find_label_cached(project_path, $~[:label_id], $~[:label_name])
if label
labels[label.id] = yield match, label.id, project, namespace, $~
@@ -34,6 +34,12 @@ module Banzai
escape_with_placeholders(unescaped_html, labels)
end
+ def find_label_cached(parent_ref, label_id, label_name)
+ cached_call(:banzai_find_label_cached, label_name&.tr('"', '') || label_id, path: [object_class, parent_ref]) do
+ find_label(parent_ref, label_id, label_name)
+ end
+ end
+
def find_label(parent_ref, label_id, label_name)
parent = parent_from_ref(parent_ref)
return unless parent
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 33df1d655fd..cfd4b932568 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -25,14 +25,12 @@ module Banzai
def initialize(doc, context = nil, result = nil)
super
- if update_nodes_enabled?
- @new_nodes = {}
- @nodes = self.result[:reference_filter_nodes]
- end
+ @new_nodes = {}
+ @nodes = self.result[:reference_filter_nodes]
end
def call_and_update_nodes
- update_nodes_enabled? ? with_update_nodes { call } : call
+ with_update_nodes { call }
end
# Returns a data attribute String to attach to a reference link
@@ -78,6 +76,10 @@ module Banzai
context[:group]
end
+ def user
+ context[:user]
+ end
+
def skip_project_check?
context[:skip_project_check]
end
@@ -164,11 +166,7 @@ module Banzai
end
def replace_text_with_html(node, index, html)
- if update_nodes_enabled?
- replace_and_update_new_nodes(node, index, html)
- else
- node.replace(html)
- end
+ replace_and_update_new_nodes(node, index, html)
end
def replace_and_update_new_nodes(node, index, html)
@@ -208,10 +206,6 @@ module Banzai
end
result[:reference_filter_nodes] = nodes
end
-
- def update_nodes_enabled?
- Feature.enabled?(:update_nodes_for_banzai_reference_filter, project)
- end
end
end
end
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index c1a5f33c225..6dc0cce6050 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -14,7 +14,7 @@ module Banzai
LANG_PARAMS_ATTR = 'data-lang-params'
def call
- doc.search('pre:not([data-math-style]) > code').each do |node|
+ doc.search('pre:not([data-math-style]):not([data-mermaid-style]) > code').each do |node|
highlight_node(node)
end
diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb
index 9ecbc3ecec2..c4d7e40b46c 100644
--- a/lib/banzai/reference_parser/base_parser.rb
+++ b/lib/banzai/reference_parser/base_parser.rb
@@ -169,7 +169,8 @@ module Banzai
# been queried the object is returned from the cache.
def collection_objects_for_ids(collection, ids)
if Gitlab::SafeRequestStore.active?
- ids = ids.map(&:to_i)
+ ids = ids.map(&:to_i).uniq
+
cache = collection_cache[collection_cache_key(collection)]
to_query = ids - cache.keys
diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb
index b86c259efbd..bd42f6e6ed4 100644
--- a/lib/banzai/reference_parser/snippet_parser.rb
+++ b/lib/banzai/reference_parser/snippet_parser.rb
@@ -9,10 +9,23 @@ module Banzai
Snippet
end
+ # Returns all the nodes that are visible to the given user.
+ def nodes_visible_to_user(user, nodes)
+ snippets = lazy { grouped_objects_for_nodes(nodes, references_relation, self.class.data_attribute) }
+
+ nodes.select do |node|
+ if node.has_attribute?(self.class.data_attribute)
+ can_read_reference?(user, snippets[node])
+ else
+ true
+ end
+ end
+ end
+
private
- def can_read_reference?(user, ref_project, node)
- can?(user, :read_snippet, referenced_by([node]).first)
+ def can_read_reference?(user, snippet)
+ can?(user, :read_snippet, snippet)
end
end
end
diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb
index c5bfc91e43d..fa4780dd8de 100644
--- a/lib/bitbucket/representation/repo.rb
+++ b/lib/bitbucket/representation/repo.rb
@@ -3,8 +3,6 @@
module Bitbucket
module Representation
class Repo < Representation::Base
- attr_reader :owner, :slug
-
def initialize(raw)
super(raw)
end
diff --git a/lib/bitbucket_server/paginator.rb b/lib/bitbucket_server/paginator.rb
index 9eda1c921b2..8a494379864 100644
--- a/lib/bitbucket_server/paginator.rb
+++ b/lib/bitbucket_server/paginator.rb
@@ -36,7 +36,7 @@ module BitbucketServer
def over_limit?
return false unless @limit
- @limit.positive? && @total >= @limit
+ @limit > 0 && @total >= @limit
end
def next_offset
diff --git a/lib/declarative_policy/base.rb b/lib/declarative_policy/base.rb
index 4af0251b990..49cbdd2aeb4 100644
--- a/lib/declarative_policy/base.rb
+++ b/lib/declarative_policy/base.rb
@@ -213,7 +213,7 @@ module DeclarativePolicy
#
# It also stores a reference to the cache, so it can be used
# to cache computations by e.g. ManifestCondition.
- attr_reader :user, :subject, :cache
+ attr_reader :user, :subject
def initialize(user, subject, opts = {})
@user = user
@subject = subject
diff --git a/lib/declarative_policy/runner.rb b/lib/declarative_policy/runner.rb
index 70666dc7924..59588b4d84e 100644
--- a/lib/declarative_policy/runner.rb
+++ b/lib/declarative_policy/runner.rb
@@ -170,7 +170,7 @@ module DeclarativePolicy
lowest_score = score
end
- break if lowest_score.zero?
+ break if lowest_score == 0
end
[remaining_steps, remaining_enablers, remaining_preventers].each do |set|
diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb
index 346ed6b6f60..5ef2d888550 100644
--- a/lib/extracts_ref.rb
+++ b/lib/extracts_ref.rb
@@ -40,43 +40,12 @@ module ExtractsRef
# Returns an Array where the first value is the tree-ish and the second is the
# path
def extract_ref(id)
- pair = ['', '']
-
- return pair unless repository_container
-
- if id =~ /^(\h{40})(.+)/
- # If the ref appears to be a SHA, we're done, just split the string
- pair = $~.captures
- else
- # Otherwise, attempt to detect the ref using a list of the repository_container's
- # branches and tags
-
- # Append a trailing slash if we only get a ref and no file path
- unless id.ends_with?('/')
- id = [id, '/'].join
- end
-
- valid_refs = ref_names.select { |v| id.start_with?("#{v}/") }
-
- if valid_refs.empty?
- # No exact ref match, so just try our best
- pair = id.match(%r{([^/]+)(.*)}).captures
- else
- # There is a distinct possibility that multiple refs prefix the ID.
- # Use the longest match to maximize the chance that we have the
- # right ref.
- best_match = valid_refs.max_by(&:length)
- # Partition the string into the ref and the path, ignoring the empty first value
- pair = id.partition(best_match)[1..-1]
- end
- end
-
- pair[0] = pair[0].strip
-
- # Remove ending slashes from path
- pair[1].gsub!(%r{^/|/$}, '')
-
- pair
+ pair = extract_raw_ref(id)
+
+ [
+ pair[0].strip,
+ pair[1].delete_prefix('/').delete_suffix('/')
+ ]
end
# Assigns common instance variables for views working with Git tree-ish objects
@@ -109,6 +78,47 @@ module ExtractsRef
private
+ def extract_raw_ref(id)
+ return ['', ''] unless repository_container
+
+ # If the ref appears to be a SHA, we're done, just split the string
+ return $~.captures if id =~ /^(\h{40})(.+)/
+
+ # No slash means we must have a ref and no path
+ return [id, ''] unless id.include?('/')
+
+ # Otherwise, attempt to detect the ref using a list of the
+ # repository_container's branches and tags
+
+ # Append a trailing slash if we only get a ref and no file path
+ id = [id, '/'].join unless id.ends_with?('/')
+ first_path_segment, rest = id.split('/', 2)
+
+ return [first_path_segment, rest] if use_first_path_segment?(first_path_segment)
+
+ valid_refs = ref_names.select { |v| id.start_with?("#{v}/") }
+
+ # No exact ref match, so just try our best
+ return id.match(%r{([^/]+)(.*)}).captures if valid_refs.empty?
+
+ # There is a distinct possibility that multiple refs prefix the ID.
+ # Use the longest match to maximize the chance that we have the
+ # right ref.
+ best_match = valid_refs.max_by(&:length)
+
+ # Partition the string into the ref and the path, ignoring the empty first value
+ id.partition(best_match)[1..-1]
+ end
+
+ def use_first_path_segment?(ref)
+ return false unless ::Feature.enabled?(:extracts_path_optimization)
+ return false unless repository_container
+ return false if repository_container.repository.has_ambiguous_refs?
+
+ repository_container.repository.branch_names_include?(ref) ||
+ repository_container.repository.tag_names_include?(ref)
+ end
+
# overridden in subclasses, do not remove
def get_id
id = [params[:id] || params[:ref]]
diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb
index dad3dabb4fc..94b81b7d290 100644
--- a/lib/gitlab/alerting/alert.rb
+++ b/lib/gitlab/alerting/alert.rb
@@ -7,7 +7,17 @@ module Gitlab
include Gitlab::Utils::StrongMemoize
include Presentable
- attr_accessor :project, :payload
+ attr_accessor :project, :payload, :am_alert
+
+ def self.for_alert_management_alert(project:, alert:)
+ params = if alert.prometheus?
+ alert.payload
+ else
+ Gitlab::Alerting::NotificationPayloadParser.call(alert.payload.to_h, alert.project)
+ end
+
+ self.new(project: project, payload: params, am_alert: alert)
+ end
def gitlab_alert
strong_memoize(:gitlab_alert) do
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
index 4d47a17545a..be5d9be3d64 100644
--- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -13,7 +13,7 @@ module Gitlab
MAPPINGS = {
Issue => {
serializer_class: AnalyticsIssueSerializer,
- includes_for_query: { project: [:namespace], author: [] },
+ includes_for_query: { project: { namespace: [:route] }, author: [] },
columns_for_select: %I[title iid id created_at author_id project_id]
},
MergeRequest => {
@@ -41,7 +41,7 @@ module Gitlab
project = record.project
attributes = record.attributes.merge({
project_path: project.path,
- namespace_path: project.namespace.path,
+ namespace_path: project.namespace.route.path,
author: record.author
})
serializer.represent(attributes)
@@ -82,7 +82,7 @@ module Gitlab
q = ordered_and_limited_query
.joins(ci_build_join)
- .select(build_table[:id], round_duration_to_seconds.as('total_time'))
+ .select(build_table[:id], *time_columns)
results = execute_query(q).to_a
@@ -90,12 +90,12 @@ module Gitlab
end
def ordered_and_limited_query
- order_by_end_event(query).limit(MAX_RECORDS)
+ order_by_end_event(query, columns).limit(MAX_RECORDS)
end
def records
results = ordered_and_limited_query
- .select(*columns, round_duration_to_seconds.as('total_time'))
+ .select(*columns, *time_columns)
# using preloader instead of includes to avoid AR generating a large column list
ActiveRecord::Associations::Preloader.new.preload(
@@ -106,6 +106,14 @@ module Gitlab
results
end
# rubocop: enable CodeReuse/ActiveRecord
+
+ def time_columns
+ [
+ stage.start_event.timestamp_projection.as('start_event_timestamp'),
+ stage.end_event.timestamp_projection.as('end_event_timestamp'),
+ round_duration_to_seconds.as('total_time')
+ ]
+ end
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
index c9a75b39959..80e426e6e17 100644
--- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
+++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb
@@ -24,13 +24,13 @@ module Gitlab
end
# rubocop: disable CodeReuse/ActiveRecord
- def order_by_end_event(query)
+ def order_by_end_event(query, extra_columns_to_select = [:id])
ordered_query = query.reorder(stage.end_event.timestamp_projection.desc)
# When filtering for more than one label, postgres requires the columns in ORDER BY to be present in the GROUP BY clause
if requires_grouping?
column_list = [
- ordered_query.arel_table[:id],
+ *extra_columns_to_select,
*stage.end_event.column_list,
*stage.start_event.column_list
]
diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb
index 9dd7d048eec..ad746ebbd42 100644
--- a/lib/gitlab/analytics/unique_visits.rb
+++ b/lib/gitlab/analytics/unique_visits.rb
@@ -3,57 +3,36 @@
module Gitlab
module Analytics
class UniqueVisits
- TARGET_IDS = Set[
- 'g_analytics_contribution',
- 'g_analytics_insights',
- 'g_analytics_issues',
- 'g_analytics_productivity',
- 'g_analytics_valuestream',
- 'p_analytics_pipelines',
- 'p_analytics_code_reviews',
- 'p_analytics_valuestream',
- 'p_analytics_insights',
- 'p_analytics_issues',
- 'p_analytics_repo',
- 'u_analytics_todos',
- 'i_analytics_cohorts',
- 'i_analytics_dev_ops_score'
- ].freeze
-
- KEY_EXPIRY_LENGTH = 28.days
-
def track_visit(visitor_id, target_id, time = Time.zone.now)
- target_key = key(target_id, time)
-
- Gitlab::Redis::SharedState.with do |redis|
- redis.multi do |multi|
- multi.pfadd(target_key, visitor_id)
- multi.expire(target_key, KEY_EXPIRY_LENGTH)
- end
- end
+ Gitlab::UsageDataCounters::HLLRedisCounter.track_event(visitor_id, target_id, time)
end
- def weekly_unique_visits_for_target(target_id, week_of: 7.days.ago)
- Gitlab::Redis::SharedState.with do |redis|
- redis.pfcount(key(target_id, week_of))
- end
+ # Returns number of unique visitors for given targets in given time frame
+ #
+ # @param [String, Array[<String>]] targets ids of targets to count visits on. Special case for :any
+ # @param [ActiveSupport::TimeWithZone] start_date start of time frame
+ # @param [ActiveSupport::TimeWithZone] end_date end of time frame
+ # @return [Integer] number of unique visitors
+ def unique_visits_for(targets:, start_date: 7.days.ago, end_date: start_date + 1.week)
+ target_ids = if targets == :analytics
+ self.class.analytics_ids
+ elsif targets == :compliance
+ self.class.compliance_ids
+ else
+ Array(targets)
+ end
+
+ Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: target_ids, start_date: start_date, end_date: end_date)
end
- def weekly_unique_visits_for_any_target(week_of: 7.days.ago)
- keys = TARGET_IDS.map { |target_id| key(target_id, week_of) }
-
- Gitlab::Redis::SharedState.with do |redis|
- redis.pfcount(*keys)
+ class << self
+ def analytics_ids
+ Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('analytics')
end
- end
-
- private
- def key(target_id, time)
- raise "Invalid target id #{target_id}" unless TARGET_IDS.include?(target_id.to_s)
-
- year_week = time.strftime('%G-%V')
- "#{target_id}-{#{year_week}}"
+ def compliance_ids
+ Gitlab::UsageDataCounters::HLLRedisCounter.events_for_category('compliance')
+ end
end
end
end
diff --git a/lib/gitlab/app_logger.rb b/lib/gitlab/app_logger.rb
index 3f5e9adf925..a39e7f31886 100644
--- a/lib/gitlab/app_logger.rb
+++ b/lib/gitlab/app_logger.rb
@@ -5,7 +5,11 @@ module Gitlab
LOGGERS = [Gitlab::AppTextLogger, Gitlab::AppJsonLogger].freeze
def self.loggers
- LOGGERS
+ if Gitlab::Utils.to_boolean(ENV.fetch('UNSTRUCTURED_RAILS_LOG', 'true'))
+ LOGGERS
+ else
+ [Gitlab::AppJsonLogger]
+ end
end
def self.primary_logger
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb
index 2fac76d03e8..5cacd7e5983 100644
--- a/lib/gitlab/asciidoc.rb
+++ b/lib/gitlab/asciidoc.rb
@@ -4,6 +4,7 @@ require 'asciidoctor'
require 'asciidoctor-plantuml'
require 'asciidoctor/extensions'
require 'gitlab/asciidoc/html5_converter'
+require 'gitlab/asciidoc/mermaid_block_processor'
require 'gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter'
module Gitlab
@@ -46,6 +47,7 @@ module Gitlab
def self.render(input, context)
extensions = proc do
include_processor ::Gitlab::Asciidoc::IncludeProcessor.new(context)
+ block ::Gitlab::Asciidoc::MermaidBlockProcessor
end
extra_attrs = path_attrs(context[:requested_path])
diff --git a/lib/gitlab/asciidoc/mermaid_block_processor.rb b/lib/gitlab/asciidoc/mermaid_block_processor.rb
new file mode 100644
index 00000000000..03d1c7f1961
--- /dev/null
+++ b/lib/gitlab/asciidoc/mermaid_block_processor.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'asciidoctor'
+
+module Gitlab
+ module Asciidoc
+ # Mermaid BlockProcessor
+ class MermaidBlockProcessor < ::Asciidoctor::Extensions::BlockProcessor
+ use_dsl
+
+ named :mermaid
+ on_context :literal, :listing
+ parse_content_as :simple
+
+ def process(parent, reader, attrs)
+ create_mermaid_source_block(parent, reader.read, attrs)
+ end
+
+ private
+
+ def create_mermaid_source_block(parent, content, attrs)
+ # If "subs" attribute is specified, substitute accordingly.
+ # Be careful not to specify "specialcharacters" or your diagram code won't be valid anymore!
+ subs = attrs['subs']
+ content = parent.apply_subs(content, parent.resolve_subs(subs)) if subs
+ html = %(<div><pre data-mermaid-style="display">#{CGI.escape_html(content)}</pre></div>)
+ ::Asciidoctor::Block.new(parent, :pass, {
+ content_model: :raw,
+ source: html,
+ subs: :default
+ }.merge(attrs))
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/audit/deleted_author.rb b/lib/gitlab/audit/deleted_author.rb
new file mode 100644
index 00000000000..e3b8ad5ad21
--- /dev/null
+++ b/lib/gitlab/audit/deleted_author.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Audit
+ class DeletedAuthor < Gitlab::Audit::NullAuthor
+ end
+ end
+end
diff --git a/lib/gitlab/audit/null_author.rb b/lib/gitlab/audit/null_author.rb
new file mode 100644
index 00000000000..0b0e6a46fe4
--- /dev/null
+++ b/lib/gitlab/audit/null_author.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Audit
+ class NullAuthor
+ attr_reader :id, :name
+
+ # Creates an Author
+ #
+ # While tracking events that could take place even when
+ # a user is not logged in, (eg: downloading repo of a public project),
+ # we set the author_id of such events as -1
+ #
+ # @param [Integer] id
+ # @param [String] name
+ #
+ # @return [Gitlab::Audit::UnauthenticatedAuthor, Gitlab::Audit::DeletedAuthor]
+ def self.for(id, name)
+ if id == -1
+ Gitlab::Audit::UnauthenticatedAuthor.new(name: name)
+ else
+ Gitlab::Audit::DeletedAuthor.new(id: id, name: name)
+ end
+ end
+
+ def initialize(id:, name:)
+ @id = id
+ @name = name
+ end
+
+ def current_sign_in_ip
+ nil
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/audit/unauthenticated_author.rb b/lib/gitlab/audit/unauthenticated_author.rb
new file mode 100644
index 00000000000..84c323c1950
--- /dev/null
+++ b/lib/gitlab/audit/unauthenticated_author.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Audit
+ class UnauthenticatedAuthor < Gitlab::Audit::NullAuthor
+ def initialize(name: nil)
+ super(id: -1, name: name)
+ end
+
+ # Events that are authored by unauthenticated users, should be
+ # shown as authored by `An unauthenticated user` in the UI.
+ def name
+ @name || _('An unauthenticated user')
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 1a23814959d..332d0bc1478 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -26,6 +26,8 @@ module Gitlab
# Default scopes for OAuth applications that don't define their own
DEFAULT_SCOPES = [:api].freeze
+ CI_JOB_USER = 'gitlab-ci-token'
+
class << self
prepend_if_ee('EE::Gitlab::Auth') # rubocop: disable Cop/InjectEnterpriseEditionModule
@@ -126,7 +128,7 @@ module Gitlab
# rubocop:enable Gitlab/RailsLogger
def skip_rate_limit?(login:)
- ::Ci::Build::CI_REGISTRY_USER == login
+ CI_JOB_USER == login
end
def look_to_limit_user(actor)
@@ -257,7 +259,7 @@ module Gitlab
end
def build_access_token_check(login, password)
- return unless login == 'gitlab-ci-token'
+ return unless login == CI_JOB_USER
return unless password
build = find_build_by_token(password)
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index bd5aed0d964..f3d0c053880 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -20,6 +20,7 @@ module Gitlab
module AuthFinders
include Gitlab::Utils::StrongMemoize
include ActionController::HttpAuthentication::Basic
+ include ActionController::HttpAuthentication::Token
PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'
PRIVATE_TOKEN_PARAM = :private_token
@@ -81,7 +82,7 @@ module Gitlab
login, password = user_name_and_password(current_request)
return unless login.present? && password.present?
- return unless ::Ci::Build::CI_REGISTRY_USER == login
+ return unless ::Gitlab::Auth::CI_JOB_USER == login
job = ::Ci::Build.find_by_token(password)
raise UnauthorizedError unless job
@@ -131,6 +132,15 @@ module Gitlab
deploy_token
end
+ def cluster_agent_token_from_authorization_token
+ return unless route_authentication_setting[:cluster_agent_token_allowed]
+ return unless current_request.authorization.present?
+
+ authorization_token, _options = token_and_options(current_request)
+
+ ::Clusters::AgentToken.find_by_token(authorization_token)
+ end
+
def find_runner_from_token
return unless api_request?
diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb
index f64fcd822c6..4f448211abf 100644
--- a/lib/gitlab/auth/ldap/adapter.rb
+++ b/lib/gitlab/auth/ldap/adapter.rb
@@ -54,7 +54,7 @@ module Gitlab
if results.nil?
response = ldap.get_operation_result
- unless response.code.zero?
+ unless response.code == 0
Rails.logger.warn("LDAP search error: #{response.message}") # rubocop:disable Gitlab/RailsLogger
end
diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb
index b3321c0b1fb..8c5000147c4 100644
--- a/lib/gitlab/auth/ldap/person.rb
+++ b/lib/gitlab/auth/ldap/person.rb
@@ -11,7 +11,7 @@ module Gitlab
InvalidEntryError = Class.new(StandardError)
- attr_accessor :entry, :provider
+ attr_accessor :provider
def self.find_by_uid(uid, adapter)
uid = Net::LDAP::Filter.escape(uid)
diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb
index 8a60d6ef482..086f4a2e91c 100644
--- a/lib/gitlab/auth/o_auth/user.rb
+++ b/lib/gitlab/auth/o_auth/user.rb
@@ -12,7 +12,7 @@ module Gitlab
SignupDisabledError = Class.new(StandardError)
SigninDisabledForProviderError = Class.new(StandardError)
- attr_accessor :auth_hash, :gl_user
+ attr_reader :auth_hash
def initialize(auth_hash)
self.auth_hash = auth_hash
@@ -62,6 +62,7 @@ module Gitlab
def find_user
user = find_by_uid_and_provider
+ user ||= find_by_email if auto_link_user?
user ||= find_or_build_ldap_user if auto_link_ldap_user?
user ||= build_new_user if signup_enabled?
@@ -150,6 +151,7 @@ module Gitlab
def find_ldap_person(auth_hash, adapter)
Gitlab::Auth::Ldap::Person.find_by_uid(auth_hash.uid, adapter) ||
Gitlab::Auth::Ldap::Person.find_by_email(auth_hash.uid, adapter) ||
+ Gitlab::Auth::Ldap::Person.find_by_email(auth_hash.email, adapter) ||
Gitlab::Auth::Ldap::Person.find_by_dn(auth_hash.uid, adapter)
rescue Gitlab::Auth::Ldap::LdapConnectionError
nil
@@ -269,6 +271,10 @@ module Gitlab
.disabled_oauth_sign_in_sources
.include?(auth_hash.provider)
end
+
+ def auto_link_user?
+ Gitlab.config.omniauth.auto_link_user
+ end
end
end
end
diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb
deleted file mode 100644
index 79f38aed9f1..00000000000
--- a/lib/gitlab/background_migration/archive_legacy_traces.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class ArchiveLegacyTraces
- def perform(start_id, stop_id)
- # This background migration directly refers to ::Ci::Build model which is defined in application code.
- # In general, migration code should be isolated as much as possible in order to be idempotent.
- # However, `archive!` method is too complicated to be replicated by coping its subsequent code.
- # So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1
- ::Ci::Build.finished.without_archived_trace
- .where(id: start_id..stop_id).find_each do |build|
- build.trace.archive!
- rescue => e
- Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/backfill_designs_relative_position.rb b/lib/gitlab/background_migration/backfill_designs_relative_position.rb
new file mode 100644
index 00000000000..efbb1b950ad
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_designs_relative_position.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This migration is not needed anymore and was disabled, because we're now
+ # also backfilling design positions immediately before moving a design.
+ #
+ # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39555
+ class BackfillDesignsRelativePosition
+ def perform(issue_ids)
+ # no-op
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb
deleted file mode 100644
index a6194616663..00000000000
--- a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # Class that will fill the project_repositories table for projects that
- # are on hashed storage and an entry is is missing in this table.
- class BackfillHashedProjectRepositories < BackfillProjectRepositories
- private
-
- def projects
- Project.on_hashed_storage
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb
deleted file mode 100644
index 2a079060380..00000000000
--- a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb
+++ /dev/null
@@ -1,213 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # This module is used to write the full path of all projects to
- # the git repository config file.
- # Storing the full project path in the git config allows admins to
- # easily identify a project when it is using hashed storage.
- module BackfillProjectFullpathInRepoConfig
- OrphanedNamespaceError = Class.new(StandardError)
-
- module Storage
- # Class that returns the disk path for a project using hashed storage
- class Hashed
- attr_accessor :project
-
- ROOT_PATH_PREFIX = '@hashed'
-
- def initialize(project)
- @project = project
- end
-
- def disk_path
- "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}"
- end
-
- def disk_hash
- @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
- end
- end
-
- # Class that returns the disk path for a project using legacy storage
- class LegacyProject
- attr_accessor :project
-
- def initialize(project)
- @project = project
- end
-
- def disk_path
- project.full_path
- end
- end
- end
-
- # Concern used by Project and Namespace to determine the full
- # route to the project
- module Routable
- extend ActiveSupport::Concern
-
- def full_path
- @full_path ||= build_full_path
- end
-
- def build_full_path
- return path unless has_parent?
-
- raise OrphanedNamespaceError if parent.nil?
-
- parent.full_path + '/' + path
- end
-
- def has_parent?
- read_attribute(association(:parent).reflection.foreign_key)
- end
- end
-
- # Class used to interact with repository using Gitaly
- class Repository
- attr_reader :storage
-
- def initialize(storage, relative_path)
- @storage = storage
- @relative_path = relative_path
- end
-
- def gitaly_repository
- Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path)
- end
- end
-
- # Namespace can be a user or group. It can be the root or a
- # child of another namespace.
- class Namespace < ActiveRecord::Base
- self.table_name = 'namespaces'
- self.inheritance_column = nil
-
- include Routable
-
- belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces'
- has_many :projects, inverse_of: :parent
- has_many :namespaces, inverse_of: :parent
- end
-
- # Project is where the repository (etc.) is stored
- class Project < ActiveRecord::Base
- self.table_name = 'projects'
-
- include Routable
- include EachBatch
-
- FULLPATH_CONFIG_KEY = 'gitlab.fullpath'
-
- belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects'
- delegate :disk_path, to: :storage
-
- def add_fullpath_config
- entries = { FULLPATH_CONFIG_KEY => full_path }
-
- repository_service.set_config(entries)
- end
-
- def remove_fullpath_config
- repository_service.delete_config([FULLPATH_CONFIG_KEY])
- end
-
- def cleanup_repository
- repository_service.cleanup
- end
-
- def storage
- @storage ||=
- if hashed_storage?
- Storage::Hashed.new(self)
- else
- Storage::LegacyProject.new(self)
- end
- end
-
- def hashed_storage?
- self.storage_version && self.storage_version >= 1
- end
-
- def repository
- @repository ||= Repository.new(repository_storage, disk_path + '.git')
- end
-
- def repository_service
- @repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository)
- end
- end
-
- # Base class for Up and Down migration classes
- class BackfillFullpathMigration
- RETRY_DELAY = 15.minutes
- MAX_RETRIES = 2
-
- # Base class for retrying one project
- class BaseRetryOne
- def perform(project_id, retry_count)
- project = Project.find(project_id)
-
- return unless project
-
- migration_class.new.safe_perform_one(project, retry_count)
- end
- end
-
- def perform(start_id, end_id)
- Project.includes(:parent).where(id: start_id..end_id).each do |project|
- safe_perform_one(project)
- end
- end
-
- def safe_perform_one(project, retry_count = 0)
- perform_one(project)
- rescue GRPC::NotFound, GRPC::InvalidArgument, OrphanedNamespaceError
- nil
- rescue GRPC::BadStatus
- schedule_retry(project, retry_count + 1) if retry_count < MAX_RETRIES
- end
-
- def schedule_retry(project, retry_count)
- # Constants provided to BackgroundMigrationWorker must be within the
- # scope of Gitlab::BackgroundMigration
- retry_class_name = self.class::RetryOne.name.sub('Gitlab::BackgroundMigration::', '')
-
- BackgroundMigrationWorker.perform_in(RETRY_DELAY, retry_class_name, [project.id, retry_count])
- end
- end
-
- # Class to add the fullpath to the git repo config
- class Up < BackfillFullpathMigration
- # Class used to retry
- class RetryOne < BaseRetryOne
- def migration_class
- Up
- end
- end
-
- def perform_one(project)
- project.cleanup_repository
- project.add_fullpath_config
- end
- end
-
- # Class to rollback adding the fullpath to the git repo config
- class Down < BackfillFullpathMigration
- # Class used to retry
- class RetryOne < BaseRetryOne
- def migration_class
- Down
- end
- end
-
- def perform_one(project)
- project.cleanup_repository
- project.remove_fullpath_config
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb
new file mode 100644
index 00000000000..6014ccc12eb
--- /dev/null
+++ b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+ module BackgroundMigration
+ class CopyMergeRequestTargetProjectToMergeRequestMetrics
+ extend ::Gitlab::Utils::Override
+
+ def perform(start_id, stop_id)
+ ActiveRecord::Base.connection.execute <<~SQL
+ WITH merge_requests_batch AS (
+ SELECT id, target_project_id
+ FROM merge_requests WHERE id BETWEEN #{Integer(start_id)} AND #{Integer(stop_id)}
+ )
+ UPDATE
+ merge_request_metrics
+ SET
+ target_project_id = merge_requests_batch.target_project_id
+ FROM merge_requests_batch
+ WHERE merge_request_metrics.merge_request_id=merge_requests_batch.id
+ SQL
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb
deleted file mode 100644
index 103bd98af14..00000000000
--- a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class FillFileStoreJobArtifact
- class JobArtifact < ActiveRecord::Base
- self.table_name = 'ci_job_artifacts'
- end
-
- def perform(start_id, stop_id)
- FillFileStoreJobArtifact::JobArtifact
- .where(file_store: nil)
- .where(id: (start_id..stop_id))
- .update_all(file_store: 1)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb
deleted file mode 100644
index 77c1f1ffaf0..00000000000
--- a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class FillFileStoreLfsObject
- class LfsObject < ActiveRecord::Base
- self.table_name = 'lfs_objects'
- end
-
- def perform(start_id, stop_id)
- FillFileStoreLfsObject::LfsObject
- .where(file_store: nil)
- .where(id: (start_id..stop_id))
- .update_all(file_store: 1)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb
deleted file mode 100644
index cba3e21cea6..00000000000
--- a/lib/gitlab/background_migration/fill_store_upload.rb
+++ /dev/null
@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class FillStoreUpload
- class Upload < ActiveRecord::Base
- self.table_name = 'uploads'
- self.inheritance_column = :_type_disabled
- end
-
- def perform(start_id, stop_id)
- FillStoreUpload::Upload
- .where(store: nil)
- .where(id: (start_id..stop_id))
- .update_all(store: 1)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb
deleted file mode 100644
index 20a98c8e141..00000000000
--- a/lib/gitlab/background_migration/fix_cross_project_label_links.rb
+++ /dev/null
@@ -1,140 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class FixCrossProjectLabelLinks
- GROUP_NESTED_LEVEL = 10.freeze
-
- class Project < ActiveRecord::Base
- self.table_name = 'projects'
- end
-
- class Label < ActiveRecord::Base
- self.inheritance_column = :_type_disabled
- self.table_name = 'labels'
- end
-
- class LabelLink < ActiveRecord::Base
- self.table_name = 'label_links'
- end
-
- class Issue < ActiveRecord::Base
- self.table_name = 'issues'
- end
-
- class MergeRequest < ActiveRecord::Base
- self.table_name = 'merge_requests'
- end
-
- class Namespace < ActiveRecord::Base
- self.inheritance_column = :_type_disabled
- self.table_name = 'namespaces'
-
- def self.groups_with_descendants_ids(start_id, stop_id)
- # To isolate migration code, we avoid usage of
- # Gitlab::GroupHierarchy#base_and_descendants which already
- # does this job better
- ids = Namespace.where(type: 'Group', id: Label.where(type: 'GroupLabel').select('distinct group_id')).where(id: start_id..stop_id).pluck(:id)
- group_ids = ids
-
- GROUP_NESTED_LEVEL.times do
- ids = Namespace.where(type: 'Group', parent_id: ids).pluck(:id)
- break if ids.empty?
-
- group_ids += ids
- end
-
- group_ids.uniq
- end
- end
-
- def perform(start_id, stop_id)
- group_ids = Namespace.groups_with_descendants_ids(start_id, stop_id)
- project_ids = Project.where(namespace_id: group_ids).select(:id)
-
- fix_issues(project_ids)
- fix_merge_requests(project_ids)
- end
-
- private
-
- # select IDs of issues which reference a label which is:
- # a) a project label of a different project, or
- # b) a group label of a different group than issue's project group
- def fix_issues(project_ids)
- issue_ids = Label
- .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'Issue\'
- INNER JOIN issues ON issues.id = label_links.target_id
- INNER JOIN projects ON projects.id = issues.project_id')
- .where('issues.project_id in (?)', project_ids)
- .where('(labels.project_id is not null and labels.project_id != issues.project_id) '\
- 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)')
- .select('distinct issues.id')
-
- Issue.where(id: issue_ids).find_each { |issue| check_resource_labels(issue, issue.project_id) }
- end
-
- # select IDs of MRs which reference a label which is:
- # a) a project label of a different project, or
- # b) a group label of a different group than MR's project group
- def fix_merge_requests(project_ids)
- mr_ids = Label
- .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'MergeRequest\'
- INNER JOIN merge_requests ON merge_requests.id = label_links.target_id
- INNER JOIN projects ON projects.id = merge_requests.target_project_id')
- .where('merge_requests.target_project_id in (?)', project_ids)
- .where('(labels.project_id is not null and labels.project_id != merge_requests.target_project_id) '\
- 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)')
- .select('distinct merge_requests.id')
-
- MergeRequest.where(id: mr_ids).find_each { |merge_request| check_resource_labels(merge_request, merge_request.target_project_id) }
- end
-
- def check_resource_labels(resource, project_id)
- local_labels = available_labels(project_id)
-
- # get all label links for the given resource (issue/MR)
- # which reference a label not included in available_labels
- # (other than its project labels and labels of ancestor groups)
- cross_labels = LabelLink
- .select('label_id, labels.title as title, labels.color as color, label_links.id as label_link_id')
- .joins('INNER JOIN labels ON labels.id = label_links.label_id')
- .where(target_type: resource.class.name.demodulize, target_id: resource.id)
- .where('labels.id not in (?)', local_labels.select(:id))
-
- cross_labels.each do |label|
- matching_label = local_labels.find {|l| l.title == label.title && l.color == label.color}
-
- next unless matching_label
-
- Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" # rubocop:disable Gitlab/RailsLogger
- LabelLink.update(label.label_link_id, label_id: matching_label.id)
- end
- end
-
- # get all labels available for the project (including
- # group labels of ancestor groups)
- def available_labels(project_id)
- @labels ||= {}
- @labels[project_id] ||= Label
- .where("(type = 'GroupLabel' and group_id in (?)) or (type = 'ProjectLabel' and id = ?)",
- project_group_ids(project_id),
- project_id)
- end
-
- def project_group_ids(project_id)
- ids = [Project.find(project_id).namespace_id]
-
- GROUP_NESTED_LEVEL.times do
- group = Namespace.find(ids.last)
- break unless group.parent_id
-
- ids << group.parent_id
- end
-
- ids
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb
deleted file mode 100644
index 268c6083d3c..00000000000
--- a/lib/gitlab/background_migration/migrate_build_stage.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class MigrateBuildStage
- module Migratable
- class Stage < ActiveRecord::Base
- self.table_name = 'ci_stages'
- end
-
- class Build < ActiveRecord::Base
- self.table_name = 'ci_builds'
- self.inheritance_column = :_type_disabled
-
- def ensure_stage!(attempts: 2)
- find_stage || create_stage!
- rescue ActiveRecord::RecordNotUnique
- retry if (attempts -= 1) > 0
- raise
- end
-
- def find_stage
- Stage.find_by(name: self.stage || 'test',
- pipeline_id: self.commit_id,
- project_id: self.project_id)
- end
-
- def create_stage!
- Stage.create!(name: self.stage || 'test',
- pipeline_id: self.commit_id,
- project_id: self.project_id)
- end
- end
- end
-
- def perform(start_id, stop_id)
- stages = Migratable::Build.where('stage_id IS NULL')
- .where('id BETWEEN ? AND ?', start_id, stop_id)
- .map { |build| build.ensure_stage! }
- .compact.map(&:id)
-
- MigrateBuildStageIdReference.new.perform(start_id, stop_id)
- MigrateStageStatus.new.perform(stages.min, stages.max)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb
deleted file mode 100644
index 0a8a4313cd5..00000000000
--- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class MigrateBuildStageIdReference
- def perform(start_id, stop_id)
- sql = <<-SQL.strip_heredoc
- UPDATE ci_builds
- SET stage_id =
- (SELECT id FROM ci_stages
- WHERE ci_stages.pipeline_id = ci_builds.commit_id
- AND ci_stages.name = ci_builds.stage)
- WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i}
- AND ci_builds.stage_id IS NULL
- SQL
-
- ActiveRecord::Base.connection.execute(sql)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb
deleted file mode 100644
index 55608529cee..00000000000
--- a/lib/gitlab/background_migration/migrate_stage_index.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class MigrateStageIndex
- def perform(start_id, stop_id)
- migrate_stage_index_sql(start_id.to_i, stop_id.to_i).tap do |sql|
- ActiveRecord::Base.connection.execute(sql)
- end
- end
-
- private
-
- def migrate_stage_index_sql(start_id, stop_id)
- <<~SQL
- WITH freqs AS (
- SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds
- WHERE stage_id BETWEEN #{start_id} AND #{stop_id}
- AND stage_idx IS NOT NULL
- GROUP BY stage_id, stage_idx
- ), indexes AS (
- SELECT DISTINCT stage_id, first_value(stage_idx)
- OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index
- FROM freqs
- )
-
- UPDATE ci_stages SET position = indexes.index
- FROM indexes WHERE indexes.stage_id = ci_stages.id
- AND ci_stages.position IS NULL;
- SQL
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb
deleted file mode 100644
index fcbcaacb2d6..00000000000
--- a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb
+++ /dev/null
@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-#
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- class PopulateClusterKubernetesNamespaceTable
- include Gitlab::Database::MigrationHelpers
-
- BATCH_SIZE = 1_000
-
- module Migratable
- class KubernetesNamespace < ActiveRecord::Base
- self.table_name = 'clusters_kubernetes_namespaces'
- end
-
- class ClusterProject < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'cluster_projects'
-
- belongs_to :project
-
- def self.with_no_kubernetes_namespace
- where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id))
- end
-
- def namespace
- slug = "#{project.path}-#{project.id}".downcase
- slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '')
- end
-
- def service_account
- "#{namespace}-service-account"
- end
- end
-
- class Project < ActiveRecord::Base
- self.table_name = 'projects'
- end
- end
-
- def perform
- cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index|
- sql_values = sql_values_for(cluster_projects_batch)
-
- insert_into_cluster_kubernetes_namespace(sql_values)
- end
- end
-
- private
-
- def cluster_projects_with_no_kubernetes_namespace
- Migratable::ClusterProject.with_no_kubernetes_namespace
- end
-
- def sql_values_for(cluster_projects)
- cluster_projects.map do |cluster_project|
- values_for_cluster_project(cluster_project)
- end
- end
-
- def values_for_cluster_project(cluster_project)
- {
- cluster_project_id: cluster_project.id,
- cluster_id: cluster_project.cluster_id,
- project_id: cluster_project.project_id,
- namespace: cluster_project.namespace,
- service_account_name: cluster_project.service_account,
- created_at: 'NOW()',
- updated_at: 'NOW()'
- }
- end
-
- def insert_into_cluster_kubernetes_namespace(rows)
- Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, # rubocop:disable Gitlab/BulkInsert
- rows,
- disable_quote: [:created_at, :updated_at])
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb
new file mode 100644
index 00000000000..e8f436b183e
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_personal_snippet_statistics.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This class creates/updates those personal snippets statistics
+ # that haven't been created nor initialized.
+ # It also updates the related root storage namespace stats
+ class PopulatePersonalSnippetStatistics
+ def perform(snippet_ids)
+ personal_snippets(snippet_ids).group_by(&:author).each do |author, author_snippets|
+ upsert_snippet_statistics(author_snippets)
+ update_namespace_statistics(author.namespace)
+ end
+ end
+
+ private
+
+ def personal_snippets(snippet_ids)
+ PersonalSnippet
+ .where(id: snippet_ids)
+ .includes(author: :namespace)
+ .includes(:statistics)
+ .includes(snippet_repository: :shard)
+ end
+
+ def upsert_snippet_statistics(snippets)
+ snippets.each do |snippet|
+ response = Snippets::UpdateStatisticsService.new(snippet).execute
+
+ error_message("#{response.message} snippet: #{snippet.id}") if response.error?
+ end
+ end
+
+ def update_namespace_statistics(namespace)
+ Namespaces::StatisticsRefresherService.new.execute(namespace)
+ rescue => e
+ error_message("Error updating statistics for namespace #{namespace.id}: #{e.message}")
+ end
+
+ def logger
+ @logger ||= Gitlab::BackgroundMigration::Logger.build
+ end
+
+ def error_message(message)
+ logger.error(message: "Snippet Statistics Migration: #{message}")
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb
deleted file mode 100644
index 43698b7955f..00000000000
--- a/lib/gitlab/background_migration/populate_untracked_uploads.rb
+++ /dev/null
@@ -1,111 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # This class processes a batch of rows in `untracked_files_for_uploads` by
- # adding each file to the `uploads` table if it does not exist.
- class PopulateUntrackedUploads
- def perform(start_id, end_id)
- return unless migrate?
-
- files = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.where(id: start_id..end_id)
- processed_files = insert_uploads_if_needed(files)
- processed_files.delete_all
-
- drop_temp_table_if_finished
- end
-
- private
-
- def migrate?
- Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.table_exists? &&
- Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.table_exists?
- end
-
- def insert_uploads_if_needed(files)
- filtered_files, error_files = filter_error_files(files)
- filtered_files = filter_existing_uploads(filtered_files)
- filtered_files = filter_deleted_models(filtered_files)
- insert(filtered_files)
-
- processed_files = files.where.not(id: error_files.map(&:id))
- processed_files
- end
-
- def filter_error_files(files)
- files.partition do |file|
- file.to_h
- true
- rescue => e
- msg = <<~MSG
- Error parsing path "#{file.path}":
- #{e.message}
- #{e.backtrace.join("\n ")}
- MSG
- Rails.logger.error(msg) # rubocop:disable Gitlab/RailsLogger
- false
- end
- end
-
- def filter_existing_uploads(files)
- paths = files.map(&:upload_path)
- existing_paths = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Upload.where(path: paths).pluck(:path).to_set
-
- files.reject do |file|
- existing_paths.include?(file.upload_path)
- end
- end
-
- # There are files on disk that are not in the uploads table because their
- # model was deleted, and we don't delete the files on disk.
- def filter_deleted_models(files)
- ids = deleted_model_ids(files)
-
- files.reject do |file|
- ids[file.model_type].include?(file.model_id)
- end
- end
-
- def deleted_model_ids(files)
- ids = {
- 'Appearance' => [],
- 'Namespace' => [],
- 'Note' => [],
- 'Project' => [],
- 'User' => []
- }
-
- # group model IDs by model type
- files.each do |file|
- ids[file.model_type] << file.model_id
- end
-
- ids.each do |model_type, model_ids|
- model_class = "Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::#{model_type}".constantize
- found_ids = model_class.where(id: model_ids.uniq).pluck(:id)
- deleted_ids = ids[model_type] - found_ids
- ids[model_type] = deleted_ids
- end
-
- ids
- end
-
- def insert(files)
- rows = files.map do |file|
- file.to_h.merge(created_at: 'NOW()')
- end
-
- Gitlab::Database.bulk_insert('uploads', # rubocop:disable Gitlab/BulkInsert
- rows,
- disable_quote: :created_at)
- end
-
- def drop_temp_table_if_finished
- if Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.all.empty? && !Rails.env.test? # Dropping a table intermittently breaks test cleanup
- Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::UntrackedFile.connection.drop_table(:untracked_files_for_uploads,
- if_exists: true)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb b/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb
deleted file mode 100644
index 23e8be4a9ab..00000000000
--- a/lib/gitlab/background_migration/populate_untracked_uploads_dependencies.rb
+++ /dev/null
@@ -1,190 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
- module BackgroundMigration
- module PopulateUntrackedUploadsDependencies
- # This class is responsible for producing the attributes necessary to
- # track an uploaded file in the `uploads` table.
- class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength
- self.table_name = 'untracked_files_for_uploads'
-
- # Ends with /:random_hex/:filename
- FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}.freeze
- FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/.freeze
-
- # These regex patterns are tested against a relative path, relative to
- # the upload directory.
- # For convenience, if there exists a capture group in the pattern, then
- # it indicates the model_id.
- PATH_PATTERNS = [
- {
- pattern: %r{\A-/system/appearance/logo/(\d+)/},
- uploader: 'AttachmentUploader',
- model_type: 'Appearance'
- },
- {
- pattern: %r{\A-/system/appearance/header_logo/(\d+)/},
- uploader: 'AttachmentUploader',
- model_type: 'Appearance'
- },
- {
- pattern: %r{\A-/system/note/attachment/(\d+)/},
- uploader: 'AttachmentUploader',
- model_type: 'Note'
- },
- {
- pattern: %r{\A-/system/user/avatar/(\d+)/},
- uploader: 'AvatarUploader',
- model_type: 'User'
- },
- {
- pattern: %r{\A-/system/group/avatar/(\d+)/},
- uploader: 'AvatarUploader',
- model_type: 'Namespace'
- },
- {
- pattern: %r{\A-/system/project/avatar/(\d+)/},
- uploader: 'AvatarUploader',
- model_type: 'Project'
- },
- {
- pattern: FILE_UPLOADER_PATH,
- uploader: 'FileUploader',
- model_type: 'Project'
- }
- ].freeze
-
- def to_h
- @upload_hash ||= {
- path: upload_path,
- uploader: uploader,
- model_type: model_type,
- model_id: model_id,
- size: file_size,
- checksum: checksum
- }
- end
-
- def upload_path
- # UntrackedFile#path is absolute, but Upload#path depends on uploader
- @upload_path ||=
- if uploader == 'FileUploader'
- # Path relative to project directory in uploads
- matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH)
- matchd[0].sub(%r{\A/}, '') # remove leading slash
- else
- path
- end
- end
-
- def uploader
- matching_pattern_map[:uploader]
- end
-
- def model_type
- matching_pattern_map[:model_type]
- end
-
- def model_id
- return @model_id if defined?(@model_id)
-
- pattern = matching_pattern_map[:pattern]
- matchd = path_relative_to_upload_dir.match(pattern)
-
- # If something is captured (matchd[1] is not nil), it is a model_id
- # Only the FileUploader pattern will not match an ID
- @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id
- end
-
- def file_size
- File.size(absolute_path)
- end
-
- def checksum
- Digest::SHA256.file(absolute_path).hexdigest
- end
-
- private
-
- def matching_pattern_map
- @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map|
- path_relative_to_upload_dir.match(path_pattern_map[:pattern])
- end
-
- unless @matching_pattern_map
- raise "Unknown upload path pattern \"#{path}\""
- end
-
- @matching_pattern_map
- end
-
- def file_uploader_model_id
- matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE)
- not_found_msg = <<~MSG
- Could not capture project full_path from a FileUploader path:
- "#{path_relative_to_upload_dir}"
- MSG
- raise not_found_msg unless matchd
-
- full_path = matchd[1]
- project = Gitlab::BackgroundMigration::PopulateUntrackedUploadsDependencies::Project.find_by_full_path(full_path)
- return unless project
-
- project.id
- end
-
- # Not including a leading slash
- def path_relative_to_upload_dir
- upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR
- base = %r{\A#{Regexp.escape(upload_dir)}/}
- @path_relative_to_upload_dir ||= path.sub(base, '')
- end
-
- def absolute_path
- File.join(Gitlab.config.uploads.storage_path, path)
- end
- end
-
- # Avoid using application code
- class Upload < ActiveRecord::Base
- self.table_name = 'uploads'
- end
-
- # Avoid using application code
- class Appearance < ActiveRecord::Base
- self.table_name = 'appearances'
- end
-
- # Avoid using application code
- class Namespace < ActiveRecord::Base
- self.table_name = 'namespaces'
- end
-
- # Avoid using application code
- class Note < ActiveRecord::Base
- self.table_name = 'notes'
- end
-
- # Avoid using application code
- class User < ActiveRecord::Base
- self.table_name = 'users'
- end
-
- # Since project Markdown upload paths don't contain the project ID, we have to find the
- # project by its full_path. Due to MySQL/PostgreSQL differences, and historical reasons,
- # the logic is somewhat complex, so I've mostly copied it in here.
- class Project < ActiveRecord::Base
- self.table_name = 'projects'
-
- def self.find_by_full_path(path)
- order_sql = Arel.sql("(CASE WHEN routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)")
- where_full_path_in(path).reorder(order_sql).take
- end
-
- def self.where_full_path_in(path)
- where = "(LOWER(routes.path) = LOWER(#{connection.quote(path)}))"
- joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
deleted file mode 100644
index 3d943205783..00000000000
--- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb
+++ /dev/null
@@ -1,173 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module BackgroundMigration
- # This class finds all non-hashed uploaded file paths and saves them to a
- # `untracked_files_for_uploads` table.
- class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength
- # For bulk_queue_background_migration_jobs_by_range
- include Database::MigrationHelpers
- include ::Gitlab::Utils::StrongMemoize
-
- FIND_BATCH_SIZE = 500
- RELATIVE_UPLOAD_DIR = "uploads"
- ABSOLUTE_UPLOAD_DIR = File.join(
- Gitlab.config.uploads.storage_path,
- RELATIVE_UPLOAD_DIR
- )
- FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'
- START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/}.freeze
- EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*"
- EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*"
-
- # This class is used to iterate over batches of
- # `untracked_files_for_uploads` rows.
- class UntrackedFile < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'untracked_files_for_uploads'
- end
-
- def perform
- ensure_temporary_tracking_table_exists
-
- # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since
- # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be
- # slow, start with an empty table for Postgres < 9.5.
- # That way we can do bulk inserts at ~30x the speed of individual
- # inserts (~20 minutes worth of inserts at GitLab.com scale instead of
- # ~10 hours).
- # In all other cases, installations will get both bulk inserts and the
- # ability for these jobs to retry without having to clear and reinsert.
- clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates?
-
- store_untracked_file_paths
-
- if UntrackedFile.all.empty?
- drop_temp_table
- else
- schedule_populate_untracked_uploads_jobs
- end
- end
-
- private
-
- def ensure_temporary_tracking_table_exists
- table_name = :untracked_files_for_uploads
-
- unless ActiveRecord::Base.connection.table_exists?(table_name)
- UntrackedFile.connection.create_table table_name do |t|
- t.string :path, limit: 600, null: false
- t.index :path, unique: true
- end
- end
- end
-
- def clear_untracked_file_paths
- UntrackedFile.delete_all
- end
-
- def store_untracked_file_paths
- return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR)
-
- each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths|
- insert_file_paths(file_paths)
- end
- end
-
- def each_file_batch(search_dir, batch_size, &block)
- cmd = build_find_command(search_dir)
-
- Open3.popen2(*cmd) do |stdin, stdout, status_thread|
- yield_paths_in_batches(stdout, batch_size, &block)
-
- raise "Find command failed" unless status_thread.value.success?
- end
- end
-
- def yield_paths_in_batches(stdout, batch_size, &block)
- paths = []
-
- stdout.each_line("\0") do |line|
- paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '')
-
- if paths.size >= batch_size
- yield(paths)
- paths = []
- end
- end
-
- yield(paths) if paths.any?
- end
-
- def build_find_command(search_dir)
- cmd = %W[find -L #{search_dir}
- -type f
- ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune )
- ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune )
- -print0]
-
- ionice = which_ionice
- cmd = %W[#{ionice} -c Idle] + cmd if ionice
-
- log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\""
- Rails.logger.info log_msg # rubocop:disable Gitlab/RailsLogger
-
- cmd
- end
-
- def which_ionice
- Gitlab::Utils.which('ionice')
- rescue StandardError
- # In this case, returning false is relatively safe,
- # even though it isn't very nice
- false
- end
-
- def insert_file_paths(file_paths)
- sql = insert_sql(file_paths)
-
- ActiveRecord::Base.connection.execute(sql)
- end
-
- def insert_sql(file_paths)
- if postgresql_pre_9_5?
- "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};"
- else
- "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\
- " ON CONFLICT DO NOTHING;"
- end
- end
-
- def table_columns_and_values_for_insert(file_paths)
- values = file_paths.map do |file_path|
- ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend
- end.join(', ')
-
- "#{UntrackedFile.table_name} (path) VALUES #{values}"
- end
-
- def can_bulk_insert_and_ignore_duplicates?
- !postgresql_pre_9_5?
- end
-
- def postgresql_pre_9_5?
- strong_memoize(:postgresql_pre_9_5) do
- Gitlab::Database.version.to_f < 9.5
- end
- end
-
- def schedule_populate_untracked_uploads_jobs
- bulk_queue_background_migration_jobs_by_range(
- UntrackedFile, FOLLOW_UP_MIGRATION)
- end
-
- def drop_temp_table
- unless Rails.env.test? # Dropping a table intermittently breaks test cleanup
- UntrackedFile.connection.drop_table(:untracked_files_for_uploads,
- if_exists: true)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb
deleted file mode 100644
index 9ef6d8654ae..00000000000
--- a/lib/gitlab/background_migration/remove_restricted_todos.rb
+++ /dev/null
@@ -1,158 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-# rubocop:disable Metrics/ClassLength
-
-module Gitlab
- module BackgroundMigration
- class RemoveRestrictedTodos
- PRIVATE_FEATURE = 10
- PRIVATE_PROJECT = 0
-
- class Project < ActiveRecord::Base
- self.table_name = 'projects'
- end
-
- class ProjectAuthorization < ActiveRecord::Base
- self.table_name = 'project_authorizations'
- end
-
- class ProjectFeature < ActiveRecord::Base
- self.table_name = 'project_features'
- end
-
- class Todo < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'todos'
- end
-
- class Issue < ActiveRecord::Base
- include EachBatch
-
- self.table_name = 'issues'
- end
-
- def perform(start_id, stop_id)
- projects = Project.where('EXISTS (SELECT 1 FROM todos WHERE todos.project_id = projects.id)')
- .where(id: start_id..stop_id)
-
- projects.each do |project|
- remove_confidential_issue_todos(project.id)
-
- if project.visibility_level == PRIVATE_PROJECT
- remove_non_members_todos(project.id)
- else
- remove_restricted_features_todos(project.id)
- end
- end
- end
-
- private
-
- def remove_non_members_todos(project_id)
- batch_remove_todos_cte(project_id)
- end
-
- def remove_confidential_issue_todos(project_id)
- # min access level to access a confidential issue is reporter
- min_reporters = authorized_users(project_id)
- .select(:user_id)
- .where('access_level >= ?', 20)
-
- confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id)
- confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch|
- batch.each do |issue|
- assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id)
-
- todos = Todo.where(target_type: 'Issue', target_id: issue.id)
- .where('user_id NOT IN (?)', min_reporters)
- .where('user_id NOT IN (?)', assigned_users)
- todos = todos.where('user_id != ?', issue.author_id) if issue.author_id
-
- todos.delete_all
- end
- end
- end
-
- def remove_restricted_features_todos(project_id)
- ProjectFeature.where(project_id: project_id).each do |project_features|
- target_types = []
- target_types << 'Issue' if private?(project_features.issues_access_level)
- target_types << 'MergeRequest' if private?(project_features.merge_requests_access_level)
- target_types << 'Commit' if private?(project_features.repository_access_level)
-
- next if target_types.empty?
-
- batch_remove_todos_cte(project_id, target_types)
- end
- end
-
- def private?(feature_level)
- feature_level == PRIVATE_FEATURE
- end
-
- def authorized_users(project_id)
- ProjectAuthorization.select(:user_id).where(project_id: project_id)
- end
-
- def unauthorized_project_todos(project_id)
- Todo.where(project_id: project_id)
- .where('user_id NOT IN (?)', authorized_users(project_id))
- end
-
- def batch_remove_todos_cte(project_id, target_types = nil)
- loop do
- count = remove_todos_cte(project_id, target_types)
-
- break if count == 0
- end
- end
-
- def remove_todos_cte(project_id, target_types = nil)
- sql = []
- sql << with_all_todos_sql(project_id, target_types)
- sql << as_deleted_sql
- sql << "SELECT count(*) FROM deleted"
-
- result = Todo.connection.exec_query(sql.join(' '))
- result.rows[0][0].to_i
- end
-
- def with_all_todos_sql(project_id, target_types = nil)
- if target_types
- table = Arel::Table.new(:todos)
- in_target = table[:target_type].in(target_types)
- target_types_sql = " AND #{in_target.to_sql}"
- end
-
- <<-SQL
- WITH all_todos AS (
- SELECT id
- FROM "todos"
- WHERE "todos"."project_id" = #{project_id}
- AND (user_id NOT IN (
- SELECT "project_authorizations"."user_id"
- FROM "project_authorizations"
- WHERE "project_authorizations"."project_id" = #{project_id})
- #{target_types_sql}
- )
- ),
- SQL
- end
-
- def as_deleted_sql
- <<-SQL
- deleted AS (
- DELETE FROM todos
- WHERE id IN (
- SELECT id
- FROM all_todos
- LIMIT 5000
- )
- RETURNING id
- )
- SQL
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb
deleted file mode 100644
index bc434b0cb64..00000000000
--- a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- # Ensures services which previously received all notes events continue
- # to receive confidential ones.
- class SetConfidentialNoteEventsOnServices
- class Service < ActiveRecord::Base
- self.table_name = 'services'
-
- include ::EachBatch
-
- def self.services_to_update
- where(confidential_note_events: nil, note_events: true)
- end
- end
-
- def perform(start_id, stop_id)
- Service.services_to_update
- .where(id: start_id..stop_id)
- .update_all(confidential_note_events: true)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb
deleted file mode 100644
index 28d8d2c640b..00000000000
--- a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-# rubocop:disable Style/Documentation
-
-module Gitlab
- module BackgroundMigration
- # Ensures hooks which previously received all notes events continue
- # to receive confidential ones.
- class SetConfidentialNoteEventsOnWebhooks
- class WebHook < ActiveRecord::Base
- self.table_name = 'web_hooks'
-
- include ::EachBatch
-
- def self.hooks_to_update
- where(confidential_note_events: nil, note_events: true)
- end
- end
-
- def perform(start_id, stop_id)
- WebHook.hooks_to_update
- .where(id: start_id..stop_id)
- .update_all(confidential_note_events: true)
- end
- end
- end
-end
diff --git a/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb
new file mode 100644
index 00000000000..9f765d03d62
--- /dev/null
+++ b/lib/gitlab/background_migration/set_merge_request_diff_files_count.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Sets the MergeRequestDiff#files_count value for old rows
+ class SetMergeRequestDiffFilesCount
+ COUNT_SUBQUERY = <<~SQL
+ files_count = (
+ SELECT count(*)
+ FROM merge_request_diff_files
+ WHERE merge_request_diff_files.merge_request_diff_id = merge_request_diffs.id
+ )
+ SQL
+
+ class MergeRequestDiff < ActiveRecord::Base # rubocop:disable Style/Documentation
+ include EachBatch
+
+ self.table_name = 'merge_request_diffs'
+ end
+
+ def perform(start_id, end_id)
+ MergeRequestDiff.where(id: start_id..end_id).each_batch do |relation|
+ relation.update_all(COUNT_SUBQUERY)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb
new file mode 100644
index 00000000000..71f3483987e
--- /dev/null
+++ b/lib/gitlab/background_migration/set_null_external_diff_store_to_local_value.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This class is responsible for migrating a range of merge request diffs
+ # with external_diff_store == NULL to 1.
+ #
+ # The index `index_merge_request_diffs_external_diff_store_is_null` is
+ # expected to be used to find the rows here and in the migration scheduling
+ # the jobs that run this class.
+ class SetNullExternalDiffStoreToLocalValue
+ LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL
+
+ # Temporary AR class for merge request diffs
+ class MergeRequestDiff < ActiveRecord::Base
+ self.table_name = 'merge_request_diffs'
+ end
+
+ def perform(start_id, stop_id)
+ MergeRequestDiff.where(external_diff_store: nil, id: start_id..stop_id).update_all(external_diff_store: LOCAL_STORE)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb
new file mode 100644
index 00000000000..9ac92aab637
--- /dev/null
+++ b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This class is responsible for migrating a range of package files
+ # with file_store == NULL to 1.
+ #
+ # The index `index_packages_package_files_file_store_is_null` is
+ # expected to be used to find the rows here and in the migration scheduling
+ # the jobs that run this class.
+ class SetNullPackageFilesFileStoreToLocalValue
+ LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL
+
+ # Temporary AR class for package files
+ class PackageFile < ActiveRecord::Base
+ self.table_name = 'packages_package_files'
+ end
+
+ def perform(start_id, stop_id)
+ Packages::PackageFile.where(file_store: nil, id: start_id..stop_id).update_all(file_store: LOCAL_STORE)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb
index d71a50a0af6..b3876018553 100644
--- a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb
@@ -12,26 +12,22 @@ module Gitlab
ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models'
def perform(resource_model, join, conditions, with_notes, start_id, end_id)
+ return unless Feature.enabled?(:migrate_user_mentions, default_enabled: true)
+
resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String)
model = with_notes ? Gitlab::BackgroundMigration::UserMentions::Models::Note : resource_model
resource_user_mention_model = resource_model.user_mention_model
records = model.joins(join).where(conditions).where(id: start_id..end_id)
- records.in_groups_of(BULK_INSERT_SIZE, false).each do |records|
+ records.each_batch(of: BULK_INSERT_SIZE) do |records|
mentions = []
records.each do |record|
mention_record = record.build_mention_values(resource_user_mention_model.resource_foreign_key)
mentions << mention_record unless mention_record.blank?
end
- Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert
- resource_user_mention_model.table_name,
- mentions,
- return_ids: true,
- disable_quote: resource_model.no_quote_columns,
- on_conflict: :do_nothing
- )
+ resource_user_mention_model.insert_all(mentions) unless mentions.empty?
end
end
end
diff --git a/lib/gitlab/background_migration/user_mentions/models/commit.rb b/lib/gitlab/background_migration/user_mentions/models/commit.rb
index 279e93dbf0d..65f4a7a25b6 100644
--- a/lib/gitlab/background_migration/user_mentions/models/commit.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/commit.rb
@@ -6,6 +6,7 @@ module Gitlab
module UserMentions
module Models
class Commit
+ include EachBatch
include Concerns::IsolatedMentionable
include Concerns::MentionableMigrationMethods
diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb
index 0cdfc6447c7..bdb90b5d2b9 100644
--- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb
@@ -7,6 +7,7 @@ module Gitlab
module Models
module DesignManagement
class Design < ActiveRecord::Base
+ include EachBatch
include Concerns::MentionableMigrationMethods
def self.user_mention_model
diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb
index dc2b7819800..61d9244a4c9 100644
--- a/lib/gitlab/background_migration/user_mentions/models/epic.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb
@@ -6,6 +6,7 @@ module Gitlab
module UserMentions
module Models
class Epic < ActiveRecord::Base
+ include EachBatch
include Concerns::IsolatedMentionable
include Concerns::MentionableMigrationMethods
include CacheMarkdownField
diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb
index 655c1db71ae..6b52afea17c 100644
--- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb
@@ -6,6 +6,7 @@ module Gitlab
module UserMentions
module Models
class MergeRequest < ActiveRecord::Base
+ include EachBatch
include Concerns::IsolatedMentionable
include CacheMarkdownField
include Concerns::MentionableMigrationMethods
diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb
index c32292ad704..a3224c8c456 100644
--- a/lib/gitlab/background_migration/user_mentions/models/note.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/note.rb
@@ -6,6 +6,7 @@ module Gitlab
module UserMentions
module Models
class Note < ActiveRecord::Base
+ include EachBatch
include Concerns::IsolatedMentionable
include CacheMarkdownField
diff --git a/lib/gitlab/backtrace_cleaner.rb b/lib/gitlab/backtrace_cleaner.rb
index 30ec99808f7..d04f0983d12 100644
--- a/lib/gitlab/backtrace_cleaner.rb
+++ b/lib/gitlab/backtrace_cleaner.rb
@@ -1,5 +1,7 @@
# frozen_string_literal: true
+# Remove some GitLab code from backtraces. Do not use this for logging errors in
+# production environments, as the error may be thrown by our middleware.
module Gitlab
module BacktraceCleaner
IGNORE_BACKTRACES = %w[
diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb
index 144ba2ec031..ab7a08ffef9 100644
--- a/lib/gitlab/bare_repository_import/importer.rb
+++ b/lib/gitlab/bare_repository_import/importer.rb
@@ -123,7 +123,7 @@ module Gitlab
cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)
output, status = Gitlab::Popen.popen(cmd)
- raise output unless status.zero?
+ raise output unless status == 0
bundle_path
end
diff --git a/lib/gitlab/build_access.rb b/lib/gitlab/build_access.rb
index 37e79413541..81759693749 100644
--- a/lib/gitlab/build_access.rb
+++ b/lib/gitlab/build_access.rb
@@ -2,8 +2,6 @@
module Gitlab
class BuildAccess < UserAccess
- attr_accessor :user, :project
-
# This bypasses the `can?(:access_git)`-check we normally do in `UserAccess`
# for CI. That way if a user was able to trigger a pipeline, then the
# build is allowed to clone the project.
diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb
index 8bb5ac94e45..67c777f67a7 100644
--- a/lib/gitlab/checks/change_access.rb
+++ b/lib/gitlab/checks/change_access.rb
@@ -25,7 +25,7 @@ module Gitlab
@logger.append_message("Running checks for ref: #{@branch_name || @tag_name}")
end
- def exec
+ def validate!
ref_level_checks
# Check of commits should happen as the last step
# given they're expensive in terms of performance
diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb
index f81c215d847..b70a6a69b93 100644
--- a/lib/gitlab/checks/lfs_check.rb
+++ b/lib/gitlab/checks/lfs_check.rb
@@ -7,7 +7,10 @@ module Gitlab
ERROR_MESSAGE = 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".'
def validate!
+ # This feature flag is used for disabling integrify check on some envs
+ # because these costy calculations may cause performance issues
return unless Feature.enabled?(:lfs_check, default_enabled: true)
+
return unless project.lfs_enabled?
return if skip_lfs_integrity_check
diff --git a/lib/gitlab/ci/build/artifacts/expire_in_parser.rb b/lib/gitlab/ci/build/artifacts/expire_in_parser.rb
new file mode 100644
index 00000000000..3e8a1fb86fc
--- /dev/null
+++ b/lib/gitlab/ci/build/artifacts/expire_in_parser.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Build
+ module Artifacts
+ class ExpireInParser
+ def self.validate_duration(value)
+ new(value).validate_duration
+ end
+
+ def initialize(value)
+ @value = value
+ end
+
+ def validate_duration
+ return true if never?
+
+ parse
+ rescue ChronicDuration::DurationParseError
+ false
+ end
+
+ def seconds_from_now
+ parse&.seconds&.from_now
+ end
+
+ private
+
+ attr_reader :value
+
+ def parse
+ return if never?
+
+ ChronicDuration.parse(value)
+ end
+
+ def never?
+ value.to_s.casecmp('never') == 0
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
index ef354832e8e..355fffbf9c6 100644
--- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -16,7 +16,7 @@ module Gitlab
#
class Entry
attr_reader :entries
- attr_accessor :name
+ attr_writer :name
def initialize(path, entries)
@entries = entries
diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb
new file mode 100644
index 00000000000..e6ef12975c2
--- /dev/null
+++ b/lib/gitlab/ci/build/auto_retry.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+class Gitlab::Ci::Build::AutoRetry
+ include Gitlab::Utils::StrongMemoize
+
+ DEFAULT_RETRIES = {
+ scheduler_failure: 2
+ }.freeze
+
+ def initialize(build)
+ @build = build
+ end
+
+ def allowed?
+ return false unless @build.retryable?
+
+ within_max_retry_limit?
+ end
+
+ private
+
+ def within_max_retry_limit?
+ max_allowed_retries > 0 && max_allowed_retries > @build.retries_count
+ end
+
+ def max_allowed_retries
+ strong_memoize(:max_allowed_retries) do
+ options_retry_max || DEFAULT_RETRIES.fetch(@build.failure_reason.to_sym, 0)
+ end
+ end
+
+ def options_retry_max
+ Integer(options_retry[:max], exception: false) if retry_on_reason_or_always?
+ end
+
+ def options_retry_when
+ options_retry.fetch(:when, ['always'])
+ end
+
+ def retry_on_reason_or_always?
+ options_retry_when.include?(@build.failure_reason.to_s) ||
+ options_retry_when.include?('always')
+ end
+
+ # The format of the retry option changed in GitLab 11.5: Before it was
+ # integer only, after it is a hash. New builds are created with the new
+ # format, but builds created before GitLab 11.5 and saved in database still
+ # have the old integer only format. This method returns the retry option
+ # normalized as a hash in 11.5+ format.
+ def options_retry
+ strong_memoize(:options_retry) do
+ value = @build.options&.dig(:retry)
+ value = value.is_a?(Integer) ? { max: value } : value.to_h
+ value.with_indifferent_access
+ end
+ end
+end
diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb
index f8550b50905..3f0ccefa9e5 100644
--- a/lib/gitlab/ci/build/step.rb
+++ b/lib/gitlab/ci/build/step.rb
@@ -21,8 +21,6 @@ module Gitlab
end
def from_release(job)
- return unless Gitlab::Ci::Features.release_generation_enabled?
-
release = job.options[:release]
return unless release
diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb
index a9a9636637f..206dbaea272 100644
--- a/lib/gitlab/ci/config/entry/artifacts.rb
+++ b/lib/gitlab/ci/config/entry/artifacts.rb
@@ -42,7 +42,7 @@ module Gitlab
inclusion: { in: %w[on_success on_failure always],
message: 'should be on_success, on_failure ' \
'or always' }
- validates :expire_in, duration: true
+ validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::Artifacts::ExpireInParser }
end
end
diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb
index f4362d3b0ce..a8b67a1db4f 100644
--- a/lib/gitlab/ci/config/entry/bridge.rb
+++ b/lib/gitlab/ci/config/entry/bridge.rb
@@ -11,7 +11,7 @@ module Gitlab
class Bridge < ::Gitlab::Config::Entry::Node
include ::Gitlab::Ci::Config::Entry::Processable
- ALLOWED_KEYS = %i[trigger allow_failure when needs].freeze
+ ALLOWED_KEYS = %i[trigger].freeze
validations do
validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index a615cab1a80..f960cec1f26 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -11,9 +11,8 @@ module Gitlab
include ::Gitlab::Ci::Config::Entry::Processable
ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze
- ALLOWED_KEYS = %i[tags script type image services
- allow_failure type when start_in artifacts cache
- dependencies before_script needs after_script
+ ALLOWED_KEYS = %i[tags script type image services start_in artifacts
+ cache dependencies before_script after_script
environment coverage retry parallel interruptible timeout
resource_group release secrets].freeze
@@ -23,18 +22,9 @@ module Gitlab
validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS
validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs?
validates :script, presence: true
- validates :config,
- disallowed_keys: {
- in: %i[release],
- message: 'release features are not enabled'
- },
- unless: -> { Gitlab::Ci::Features.release_generation_enabled? }
with_options allow_nil: true do
validates :allow_failure, boolean: true
- validates :parallel, numericality: { only_integer: true,
- greater_than_or_equal_to: 2,
- less_than_or_equal_to: 50 }
validates :when, inclusion: {
in: ALLOWED_WHEN,
message: "should be one of: #{ALLOWED_WHEN.join(', ')}"
@@ -124,13 +114,47 @@ module Gitlab
description: 'This job will produce a release.',
inherit: false
+ entry :parallel, Entry::Product::Parallel,
+ description: 'Parallel configuration for this job.',
+ inherit: false
+
attributes :script, :tags, :allow_failure, :when, :dependencies,
:needs, :retry, :parallel, :start_in,
:interruptible, :timeout, :resource_group, :release
+ Matcher = Struct.new(:name, :config) do
+ def applies?
+ job_is_not_hidden? &&
+ config_is_a_hash? &&
+ has_job_keys?
+ end
+
+ private
+
+ def job_is_not_hidden?
+ !name.to_s.start_with?('.')
+ end
+
+ def config_is_a_hash?
+ config.is_a?(Hash)
+ end
+
+ def has_job_keys?
+ if name == :default
+ config.key?(:script)
+ else
+ (ALLOWED_KEYS & config.keys).any?
+ end
+ end
+ end
+
def self.matching?(name, config)
- !name.to_s.start_with?('.') &&
- config.is_a?(Hash) && config.key?(:script)
+ if Gitlab::Ci::Features.job_entry_matches_all_keys?
+ Matcher.new(name, config).applies?
+ else
+ !name.to_s.start_with?('.') &&
+ config.is_a?(Hash) && config.key?(:script)
+ end
end
def self.visible?
@@ -174,7 +198,7 @@ module Gitlab
environment_name: environment_defined? ? environment_value[:name] : nil,
coverage: coverage_defined? ? coverage_value : nil,
retry: retry_defined? ? retry_value : nil,
- parallel: has_parallel? ? parallel.to_i : nil,
+ parallel: has_parallel? ? parallel_value : nil,
interruptible: interruptible_defined? ? interruptible_value : nil,
timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil,
artifacts: artifacts_value,
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index b4539475d88..f10c509d0cc 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -14,7 +14,8 @@ module Gitlab
include ::Gitlab::Config::Entry::Attributable
include ::Gitlab::Config::Entry::Inheritable
- PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables inherit].freeze
+ PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables
+ inherit allow_failure when needs].freeze
included do
validations do
@@ -82,8 +83,8 @@ module Gitlab
@entries.delete(:except) unless except_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables
end
- if has_rules? && !has_workflow_rules && Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning?
- add_warning('uses `rules` without defining `workflow:rules`')
+ unless has_workflow_rules
+ validate_against_warnings
end
# inherit root variables
@@ -93,6 +94,19 @@ module Gitlab
end
end
+ def validate_against_warnings
+ # If rules are valid format and workflow rules are not specified
+ return unless rules_value
+ return unless Gitlab::Ci::Features.raise_job_rules_without_workflow_rules_warning?
+
+ last_rule = rules_value.last
+
+ if last_rule&.keys == [:when] && last_rule[:when] != 'never'
+ docs_url = 'read more: https://docs.gitlab.com/ee/ci/troubleshooting.html#pipeline-warnings'
+ add_warning("may allow multiple pipelines to run for a single action due to `rules:when` clause with no `workflow:rules` - #{docs_url}")
+ end
+ end
+
def name
metadata[:name]
end
diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb
new file mode 100644
index 00000000000..6af809d46c1
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/product/matrix.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Entry
+ ##
+ # Entry that represents matrix style parallel builds.
+ #
+ module Product
+ class Matrix < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Utils::StrongMemoize
+ include ::Gitlab::Config::Entry::Validatable
+ include ::Gitlab::Config::Entry::Attributable
+
+ validations do
+ validates :config, array_of_hashes: true
+
+ validate on: :composed do
+ limit = Entry::Product::Parallel::PARALLEL_LIMIT
+
+ if number_of_generated_jobs > limit
+ errors.add(:config, "generates too many jobs (maximum is #{limit})")
+ end
+ end
+ end
+
+ def compose!(deps = nil)
+ super(deps) do
+ @config.each_with_index do |variables, index|
+ @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Product::Variables)
+ .value(variables)
+ .with(parent: self, description: 'matrix variables definition.') # rubocop:disable CodeReuse/ActiveRecord
+ .create!
+ end
+
+ @entries.each_value do |entry|
+ entry.compose!(deps)
+ end
+ end
+ end
+
+ def value
+ strong_memoize(:value) do
+ @entries.values.map(&:value)
+ end
+ end
+
+ # rubocop:disable CodeReuse/ActiveRecord
+ def number_of_generated_jobs
+ value.sum do |config|
+ config.values.reduce(1) { |acc, values| acc * values.size }
+ end
+ end
+ # rubocop:enable CodeReuse/ActiveRecord
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb
new file mode 100644
index 00000000000..cd9eabbbc66
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/product/parallel.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Entry
+ ##
+ # Entry that represents a parallel job config.
+ #
+ module Product
+ class Parallel < ::Gitlab::Config::Entry::Simplifiable
+ strategy :ParallelBuilds, if: -> (config) { config.is_a?(Numeric) }
+ strategy :MatrixBuilds, if: -> (config) { config.is_a?(Hash) }
+
+ PARALLEL_LIMIT = 50
+
+ class ParallelBuilds < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+
+ validations do
+ validates :config, numericality: { only_integer: true,
+ greater_than_or_equal_to: 2,
+ less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT },
+ allow_nil: true
+ end
+
+ def value
+ { number: super.to_i }
+ end
+ end
+
+ class MatrixBuilds < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Attributable
+ include ::Gitlab::Config::Entry::Configurable
+
+ PERMITTED_KEYS = %i[matrix].freeze
+
+ validations do
+ validates :config, allowed_keys: PERMITTED_KEYS
+ validates :config, required_keys: PERMITTED_KEYS
+ end
+
+ entry :matrix, Entry::Product::Matrix,
+ description: 'Variables definition for matrix builds'
+ end
+
+ class UnknownStrategy < ::Gitlab::Config::Entry::Node
+ def errors
+ ["#{location} should be an integer or a hash"]
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb
new file mode 100644
index 00000000000..ac4f70fb69e
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/product/variables.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Entry
+ ##
+ # Entry that represents variables for parallel matrix builds.
+ #
+ module Product
+ class Variables < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+
+ validations do
+ validates :config, variables: { array_values: true }
+ validates :config, length: {
+ minimum: 2,
+ too_short: 'requires at least %{count} items'
+ }
+ end
+
+ def self.default(**)
+ {}
+ end
+
+ def value
+ @config
+ .map { |key, value| [key.to_s, Array(value).map(&:to_s)] }
+ .to_h
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
index 814dcc66362..cf6c2961ee7 100644
--- a/lib/gitlab/ci/config/external/context.rb
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -54,7 +54,7 @@ module Gitlab
end
def execution_expired?
- return false if execution_deadline.zero?
+ return false if execution_deadline == 0
current_monotonic_time > execution_deadline
end
diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb
index 1139efee9e8..451ba14bb89 100644
--- a/lib/gitlab/ci/config/normalizer.rb
+++ b/lib/gitlab/ci/config/normalizer.rb
@@ -32,7 +32,7 @@ module Gitlab
return unless job_names
job_names.flat_map do |job_name|
- parallelized_jobs[job_name.to_sym] || job_name
+ parallelized_jobs[job_name.to_sym]&.map(&:name) || job_name
end
end
@@ -42,10 +42,8 @@ module Gitlab
job_needs.flat_map do |job_need|
job_need_name = job_need[:name].to_sym
- if all_job_names = parallelized_jobs[job_need_name]
- all_job_names.map do |job_name|
- job_need.merge(name: job_name)
- end
+ if all_jobs = parallelized_jobs[job_need_name]
+ all_jobs.map { |job| job_need.merge(name: job.name) }
else
job_need
end
@@ -57,7 +55,7 @@ module Gitlab
@jobs_config.each_with_object({}) do |(job_name, config), hash|
next unless config[:parallel]
- hash[job_name] = self.class.parallelize_job_names(job_name, config[:parallel])
+ hash[job_name] = parallelize_job_config(job_name, config[:parallel])
end
end
end
@@ -65,9 +63,9 @@ module Gitlab
def expand_parallelize_jobs
@jobs_config.each_with_object({}) do |(job_name, config), hash|
if parallelized_jobs.key?(job_name)
- parallelized_jobs[job_name].each_with_index do |name, index|
- hash[name.to_sym] =
- yield(name, config.merge(name: name, instance: index + 1))
+ parallelized_jobs[job_name].each do |job|
+ hash[job.name.to_sym] =
+ yield(job.name, config.deep_merge(job.attributes))
end
else
hash[job_name] = yield(job_name, config)
@@ -75,8 +73,8 @@ module Gitlab
end
end
- def self.parallelize_job_names(name, total)
- Array.new(total) { |index| "#{name} #{index + 1}/#{total}" }
+ def parallelize_job_config(name, config)
+ Normalizer::Factory.new(name, config).create
end
end
end
diff --git a/lib/gitlab/ci/config/normalizer/factory.rb b/lib/gitlab/ci/config/normalizer/factory.rb
new file mode 100644
index 00000000000..bf813f8e878
--- /dev/null
+++ b/lib/gitlab/ci/config/normalizer/factory.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ class Normalizer
+ class Factory
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(name, config)
+ @name = name
+ @config = config
+ end
+
+ def create
+ return [] unless strategy
+
+ strategy.build_from(@name, @config)
+ end
+
+ private
+
+ def strategy
+ strong_memoize(:strategy) do
+ strategies.find do |strategy|
+ strategy.applies_to?(@config)
+ end
+ end
+ end
+
+ def strategies
+ [NumberStrategy, MatrixStrategy]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
new file mode 100644
index 00000000000..db21274a9ed
--- /dev/null
+++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ class Normalizer
+ class MatrixStrategy
+ class << self
+ def applies_to?(config)
+ config.is_a?(Hash) && config.key?(:matrix)
+ end
+
+ def build_from(job_name, initial_config)
+ config = expand(initial_config[:matrix])
+ total = config.size
+
+ config.map.with_index do |vars, index|
+ new(job_name, index.next, vars, total)
+ end
+ end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def expand(config)
+ config.flat_map do |config|
+ values = config.values
+
+ values[0]
+ .product(*values.from(1))
+ .map { |vals| config.keys.zip(vals).to_h }
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+ end
+
+ def initialize(job_name, instance, variables, total)
+ @job_name = job_name
+ @instance = instance
+ @variables = variables.to_h
+ @total = total
+ end
+
+ def attributes
+ {
+ name: name,
+ instance: instance,
+ variables: variables,
+ parallel: { total: total }
+ }
+ end
+
+ def name_with_details
+ vars = variables.map { |key, value| "#{key}=#{value}"}.join('; ')
+
+ "#{job_name} (#{vars})"
+ end
+
+ def name
+ "#{job_name} #{instance}/#{total}"
+ end
+
+ private
+
+ attr_reader :job_name, :instance, :variables, :total
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/normalizer/number_strategy.rb b/lib/gitlab/ci/config/normalizer/number_strategy.rb
new file mode 100644
index 00000000000..4754e7b46d4
--- /dev/null
+++ b/lib/gitlab/ci/config/normalizer/number_strategy.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ class Normalizer
+ class NumberStrategy
+ class << self
+ def applies_to?(config)
+ config.is_a?(Integer) || config.is_a?(Hash) && config.key?(:number)
+ end
+
+ def build_from(job_name, config)
+ total = config.is_a?(Hash) ? config[:number] : config
+
+ Array.new(total) do |index|
+ new(job_name, index.next, total)
+ end
+ end
+ end
+
+ def initialize(job_name, instance, total)
+ @job_name = job_name
+ @instance = instance
+ @total = total
+ end
+
+ def attributes
+ {
+ name: name,
+ instance: instance,
+ parallel: { total: total }
+ }
+ end
+
+ def name
+ "#{job_name} #{instance}/#{total}"
+ end
+
+ private
+
+ attr_reader :job_name, :instance, :total
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index 6130baeb9d5..2f6667d3600 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -14,32 +14,16 @@ module Gitlab
::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true)
end
- def self.pipeline_fixed_notifications?
- ::Feature.enabled?(:ci_pipeline_fixed_notifications, default_enabled: true)
- end
-
def self.instance_variables_ui_enabled?
::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true)
end
- def self.composite_status?(project)
- ::Feature.enabled?(:ci_composite_status, project, default_enabled: true)
- end
-
- def self.atomic_processing?(project)
- ::Feature.enabled?(:ci_atomic_processing, project, default_enabled: true)
- end
-
def self.pipeline_latest?
::Feature.enabled?(:ci_pipeline_latest, default_enabled: true)
end
def self.pipeline_status_omit_commit_sha_in_cache_key?(project)
- Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project)
- end
-
- def self.release_generation_enabled?
- ::Feature.enabled?(:ci_release_generation, default_enabled: true)
+ Feature.enabled?(:ci_pipeline_status_omit_commit_sha_in_cache_key, project, default_enabled: true)
end
# Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/224199
@@ -49,13 +33,11 @@ module Gitlab
# Remove in https://gitlab.com/gitlab-org/gitlab/-/issues/227052
def self.variables_api_filter_environment_scope?
- ::Feature.enabled?(:ci_variables_api_filter_environment_scope, default_enabled: false)
+ ::Feature.enabled?(:ci_variables_api_filter_environment_scope, default_enabled: true)
end
- # This FF is only used for development purpose to test that warnings can be
- # raised and propagated to the UI.
def self.raise_job_rules_without_workflow_rules_warning?
- ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning)
+ ::Feature.enabled?(:ci_raise_job_rules_without_workflow_rules_warning, default_enabled: true)
end
def self.keep_latest_artifacts_for_ref_enabled?(project)
@@ -70,8 +52,32 @@ module Gitlab
::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true)
end
+ def self.ci_if_parenthesis_enabled?
+ ::Feature.enabled?(:ci_if_parenthesis_enabled, default_enabled: true)
+ end
+
def self.allow_to_create_merge_request_pipelines_in_target_project?(target_project)
- ::Feature.enabled?(:ci_allow_to_create_merge_request_pipelines_in_target_project, target_project)
+ ::Feature.enabled?(:ci_allow_to_create_merge_request_pipelines_in_target_project, target_project, default_enabled: true)
+ end
+
+ def self.ci_plan_needs_size_limit?(project)
+ ::Feature.enabled?(:ci_plan_needs_size_limit, project, default_enabled: true)
+ end
+
+ def self.job_entry_matches_all_keys?
+ ::Feature.enabled?(:ci_job_entry_matches_all_keys)
+ end
+
+ def self.lint_creates_pipeline_with_dry_run?(project)
+ ::Feature.enabled?(:ci_lint_creates_pipeline_with_dry_run, project, default_enabled: true)
+ end
+
+ def self.reset_ci_minutes_for_all_namespaces?
+ ::Feature.enabled?(:reset_ci_minutes_for_all_namespaces, default_enabled: false)
+ end
+
+ def self.expand_names_for_cross_pipeline_artifacts?(project)
+ ::Feature.enabled?(:ci_expand_names_for_cross_pipeline_artifacts, project)
end
end
end
diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb
index 006d5097148..934c797580c 100644
--- a/lib/gitlab/ci/parsers/coverage/cobertura.rb
+++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb
@@ -28,6 +28,8 @@ module Gitlab
end
def parse_node(key, value, coverage_report)
+ return if key == 'sources'
+
if key == 'class'
Array.wrap(value).each do |item|
parse_class(item, coverage_report)
diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
new file mode 100644
index 00000000000..468f3bc4689
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Chain
+ class CancelPendingPipelines < Chain::Base
+ include Chain::Helpers
+
+ def perform!
+ return unless project.auto_cancel_pending_pipelines?
+
+ Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines) do |cancelables|
+ cancelables.find_each do |cancelable|
+ cancelable.auto_cancel_running(pipeline)
+ end
+ end
+ end
+
+ def break?
+ false
+ end
+
+ private
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def auto_cancelable_pipelines
+ project.ci_pipelines
+ .where(ref: pipeline.ref)
+ .where.not(id: pipeline.same_family_pipeline_ids)
+ .where.not(sha: project.commit(pipeline.ref).try(:id))
+ .alive_or_scheduled
+ .with_only_interruptible_builds
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 74b28b181bc..dbaa6951e64 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -10,7 +10,7 @@ module Gitlab
:trigger_request, :schedule, :merge_request, :external_pull_request,
:ignore_skip_ci, :save_incompleted,
:seeds_block, :variables_attributes, :push_options,
- :chat_data, :allow_mirror_update, :bridge, :content,
+ :chat_data, :allow_mirror_update, :bridge, :content, :dry_run,
# These attributes are set by Chains during processing:
:config_content, :config_processor, :stage_seeds
) do
@@ -22,6 +22,8 @@ module Gitlab
end
end
+ alias_method :dry_run?, :dry_run
+
def branch_exists?
strong_memoize(:is_branch) do
project.repository.branch_exists?(ref)
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb
index 3dd216b33d1..9954aedc4b7 100644
--- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb
@@ -12,7 +12,6 @@ module Gitlab
def content
strong_memoize(:content) do
next unless command.content.present?
- raise UnsupportedSourceError, "#{command.source} not a dangling build" unless command.dangling_build?
command.content
end
diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb
index aba7dab508d..d7271df1694 100644
--- a/lib/gitlab/ci/pipeline/chain/helpers.rb
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@@ -6,13 +6,13 @@ module Gitlab
module Chain
module Helpers
def error(message, config_error: false, drop_reason: nil)
- if config_error && command.save_incompleted
+ if config_error
drop_reason = :config_error
pipeline.yaml_errors = message
end
pipeline.add_error_message(message)
- pipeline.drop!(drop_reason) if drop_reason
+ pipeline.drop!(drop_reason) if drop_reason && persist_pipeline?
# TODO: consider not to rely on AR errors directly as they can be
# polluted with other unrelated errors (e.g. state machine)
@@ -23,6 +23,10 @@ module Gitlab
def warning(message)
pipeline.add_warning_message(message)
end
+
+ def persist_pipeline?
+ command.save_incompleted && !pipeline.readonly?
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb
new file mode 100644
index 00000000000..0d7449813b4
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/metrics.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Chain
+ class Metrics < Chain::Base
+ def perform!
+ counter.increment(source: @pipeline.source)
+ end
+
+ def break?
+ false
+ end
+
+ def counter
+ ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
new file mode 100644
index 00000000000..1eb7474e915
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Chain
+ module Pipeline
+ # After pipeline has been successfully created we can start processing it.
+ class Process < Chain::Base
+ def perform!
+ ::Ci::ProcessPipelineService
+ .new(@pipeline)
+ .execute
+ end
+
+ def break?
+ false
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index 204c7725214..dc648568129 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -9,30 +9,21 @@ module Gitlab
@pipeline = pipeline
@command = command
@sequence = sequence
- @completed = []
@start = Time.now
end
def build!
- @sequence.each do |chain|
- step = chain.new(@pipeline, @command)
+ @sequence.each do |step_class|
+ step = step_class.new(@pipeline, @command)
step.perform!
break if step.break?
-
- @completed.push(step)
end
- @pipeline.tap do
- yield @pipeline, self if block_given?
-
- @command.observe_creation_duration(Time.now - @start)
- @command.observe_pipeline_size(@pipeline)
- end
- end
+ @command.observe_creation_duration(Time.now - @start)
+ @command.observe_pipeline_size(@pipeline)
- def complete?
- @completed.size == @sequence.size
+ @pipeline
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb
new file mode 100644
index 00000000000..0e9add4ee74
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Chain
+ # During the dry run we don't want to persist the pipeline and skip
+ # all the other steps that operate on a persisted context.
+ # This causes the chain to break at this point.
+ class StopDryRun < Chain::Base
+ def perform!
+ # no-op
+ end
+
+ def break?
+ @command.dry_run?
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
index 769d0dffd0b..8f1e690c081 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
@@ -34,7 +34,7 @@ module Gitlab
end
def allowed_to_write_ref?
- access = Gitlab::UserAccess.new(current_user, project: project)
+ access = Gitlab::UserAccess.new(current_user, container: project)
if @command.branch_exists?
access.can_update_branch?(@command.ref)
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb
index 54a0e2ad9dd..422735bd104 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb
@@ -5,7 +5,7 @@ module Gitlab
module Pipeline
module Expression
module Lexeme
- class And < Lexeme::Operator
+ class And < Lexeme::LogicalOperator
PATTERN = /&&/.freeze
def evaluate(variables = {})
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb
index 7ebd2e25398..676857183cf 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb
@@ -10,6 +10,10 @@ module Gitlab
raise NotImplementedError
end
+ def name
+ self.class.name.demodulize.underscore
+ end
+
def self.build(token)
raise NotImplementedError
end
@@ -23,6 +27,10 @@ module Gitlab
def self.pattern
self::PATTERN
end
+
+ def self.consume?(lexeme)
+ lexeme && precedence >= lexeme.precedence
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb
index 62f4c14f597..d35be12c996 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb
@@ -5,7 +5,7 @@ module Gitlab
module Pipeline
module Expression
module Lexeme
- class Equals < Lexeme::Operator
+ class Equals < Lexeme::LogicalOperator
PATTERN = /==/.freeze
def evaluate(variables = {})
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb
new file mode 100644
index 00000000000..05d5043c06e
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Expression
+ module Lexeme
+ class LogicalOperator < Lexeme::Operator
+ # This operator class is design to handle single operators that take two
+ # arguments. Expression::Parser was originally designed to read infix operators,
+ # and so the two operands are called "left" and "right" here. If we wish to
+ # implement an Operator that takes a greater or lesser number of arguments, a
+ # structural change or additional Operator superclass will likely be needed.
+
+ def initialize(left, right)
+ raise OperatorError, 'Invalid left operand' unless left.respond_to? :evaluate
+ raise OperatorError, 'Invalid right operand' unless right.respond_to? :evaluate
+
+ @left = left
+ @right = right
+ end
+
+ def inspect
+ "#{name}(#{@left.inspect}, #{@right.inspect})"
+ end
+
+ def self.type
+ :logical_operator
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
index f7b0720d4a9..4d65b914d8d 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb
@@ -5,7 +5,7 @@ module Gitlab
module Pipeline
module Expression
module Lexeme
- class Matches < Lexeme::Operator
+ class Matches < Lexeme::LogicalOperator
PATTERN = /=~/.freeze
def evaluate(variables = {})
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb
index 8166bcd5730..64485a7e6b3 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb
@@ -5,7 +5,7 @@ module Gitlab
module Pipeline
module Expression
module Lexeme
- class NotEquals < Lexeme::Operator
+ class NotEquals < Lexeme::LogicalOperator
PATTERN = /!=/.freeze
def evaluate(variables = {})
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb
index 02479ed28a4..29c5aa5d753 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb
@@ -5,7 +5,7 @@ module Gitlab
module Pipeline
module Expression
module Lexeme
- class NotMatches < Lexeme::Operator
+ class NotMatches < Lexeme::LogicalOperator
PATTERN = /\!~/.freeze
def evaluate(variables = {})
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb
index be7258c201a..e7f7945532b 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb
@@ -9,13 +9,17 @@ module Gitlab
PATTERN = /null/.freeze
def initialize(value = nil)
- @value = nil
+ super
end
def evaluate(variables = {})
nil
end
+ def inspect
+ 'null'
+ end
+
def self.build(_value)
self.new
end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb
index 3ddab7800c8..a740c50c900 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb
@@ -6,24 +6,10 @@ module Gitlab
module Expression
module Lexeme
class Operator < Lexeme::Base
- # This operator class is design to handle single operators that take two
- # arguments. Expression::Parser was originally designed to read infix operators,
- # and so the two operands are called "left" and "right" here. If we wish to
- # implement an Operator that takes a greater or lesser number of arguments, a
- # structural change or additional Operator superclass will likely be needed.
-
OperatorError = Class.new(Expression::ExpressionError)
- def initialize(left, right)
- raise OperatorError, 'Invalid left operand' unless left.respond_to? :evaluate
- raise OperatorError, 'Invalid right operand' unless right.respond_to? :evaluate
-
- @left = left
- @right = right
- end
-
def self.type
- :operator
+ raise NotImplementedError
end
def self.precedence
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb
index 807876f905a..c7d653ac859 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb
@@ -5,7 +5,7 @@ module Gitlab
module Pipeline
module Expression
module Lexeme
- class Or < Lexeme::Operator
+ class Or < Lexeme::LogicalOperator
PATTERN = /\|\|/.freeze
def evaluate(variables = {})
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb
new file mode 100644
index 00000000000..b0ca26c9f5d
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Expression
+ module Lexeme
+ class ParenthesisClose < Lexeme::Operator
+ PATTERN = /\)/.freeze
+
+ def self.type
+ :parenthesis_close
+ end
+
+ def self.precedence
+ 900
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb
new file mode 100644
index 00000000000..924fe0663ab
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Expression
+ module Lexeme
+ class ParenthesisOpen < Lexeme::Operator
+ PATTERN = /\(/.freeze
+
+ def self.type
+ :parenthesis_open
+ end
+
+ def self.precedence
+ # Needs to be higher than `ParenthesisClose` and all other Lexemes
+ 901
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
index 0212fa9d661..514241e8ae2 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
@@ -11,7 +11,7 @@ module Gitlab
PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze
def initialize(regexp)
- @value = regexp.gsub(/\\\//, '/')
+ super(regexp.gsub(/\\\//, '/'))
unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value)
raise Lexer::SyntaxError, 'Invalid regular expression!'
@@ -24,6 +24,10 @@ module Gitlab
raise Expression::RuntimeError, 'Invalid regular expression!'
end
+ def inspect
+ "/#{value}/"
+ end
+
def self.pattern
PATTERN
end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb
index 2db2bf011f1..e90e764bcd9 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb
@@ -9,13 +9,17 @@ module Gitlab
PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze
def initialize(value)
- @value = value
+ super(value)
end
def evaluate(variables = {})
@value.to_s
end
+ def inspect
+ @value.inspect
+ end
+
def self.build(string)
new(string.match(PATTERN)[:string])
end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb
index ef9ddb6cae9..6d872fee39d 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb
@@ -9,6 +9,10 @@ module Gitlab
def self.type
:value
end
+
+ def initialize(value)
+ @value = value
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
index 85c0899e4f6..11d2010909f 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
@@ -8,12 +8,12 @@ module Gitlab
class Variable < Lexeme::Value
PATTERN = /\$(?<name>\w+)/.freeze
- def initialize(name)
- @name = name
+ def evaluate(variables = {})
+ variables.with_indifferent_access.fetch(@value, nil)
end
- def evaluate(variables = {})
- variables.with_indifferent_access.fetch(@name, nil)
+ def inspect
+ "$#{@value}"
end
def self.build(string)
diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb
index 7d7582612f9..5b7365cb33b 100644
--- a/lib/gitlab/ci/pipeline/expression/lexer.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexer.rb
@@ -10,6 +10,8 @@ module Gitlab
SyntaxError = Class.new(Expression::ExpressionError)
LEXEMES = [
+ Expression::Lexeme::ParenthesisOpen,
+ Expression::Lexeme::ParenthesisClose,
Expression::Lexeme::Variable,
Expression::Lexeme::String,
Expression::Lexeme::Pattern,
@@ -22,6 +24,28 @@ module Gitlab
Expression::Lexeme::Or
].freeze
+ # To be removed with `ci_if_parenthesis_enabled`
+ LEGACY_LEXEMES = [
+ Expression::Lexeme::Variable,
+ Expression::Lexeme::String,
+ Expression::Lexeme::Pattern,
+ Expression::Lexeme::Null,
+ Expression::Lexeme::Equals,
+ Expression::Lexeme::Matches,
+ Expression::Lexeme::NotEquals,
+ Expression::Lexeme::NotMatches,
+ Expression::Lexeme::And,
+ Expression::Lexeme::Or
+ ].freeze
+
+ def self.lexemes
+ if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled?
+ LEXEMES
+ else
+ LEGACY_LEXEMES
+ end
+ end
+
MAX_TOKENS = 100
def initialize(statement, max_tokens: MAX_TOKENS)
@@ -47,7 +71,7 @@ module Gitlab
return tokens if @scanner.eos?
- lexeme = LEXEMES.find do |type|
+ lexeme = self.class.lexemes.find do |type|
type.scan(@scanner).tap do |token|
tokens.push(token) if token.present?
end
diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb
index edb55edf356..27d7aa2f37e 100644
--- a/lib/gitlab/ci/pipeline/expression/parser.rb
+++ b/lib/gitlab/ci/pipeline/expression/parser.rb
@@ -15,11 +15,18 @@ module Gitlab
def tree
results = []
- tokens_rpn.each do |token|
+ tokens =
+ if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled?
+ tokens_rpn
+ else
+ legacy_tokens_rpn
+ end
+
+ tokens.each do |token|
case token.type
when :value
results.push(token.build)
- when :operator
+ when :logical_operator
right_operand = results.pop
left_operand = results.pop
@@ -27,7 +34,7 @@ module Gitlab
results.push(res)
end
else
- raise ParseError, 'Unprocessable token found in parse tree'
+ raise ParseError, "Unprocessable token found in parse tree: #{token.type}"
end
end
@@ -45,6 +52,7 @@ module Gitlab
# Parse the expression into Reverse Polish Notation
# (See: Shunting-yard algorithm)
+ # Taken from: https://en.wikipedia.org/wiki/Shunting-yard_algorithm#The_algorithm_in_detail
def tokens_rpn
output = []
operators = []
@@ -53,7 +61,34 @@ module Gitlab
case token.type
when :value
output.push(token)
- when :operator
+ when :logical_operator
+ output.push(operators.pop) while token.lexeme.consume?(operators.last&.lexeme)
+
+ operators.push(token)
+ when :parenthesis_open
+ operators.push(token)
+ when :parenthesis_close
+ output.push(operators.pop) while token.lexeme.consume?(operators.last&.lexeme)
+
+ raise ParseError, 'Unmatched parenthesis' unless operators.last
+
+ operators.pop if operators.last.lexeme.type == :parenthesis_open
+ end
+ end
+
+ output.concat(operators.reverse)
+ end
+
+ # To be removed with `ci_if_parenthesis_enabled`
+ def legacy_tokens_rpn
+ output = []
+ operators = []
+
+ @tokens.each do |token|
+ case token.type
+ when :value
+ output.push(token)
+ when :logical_operator
if operators.any? && token.lexeme.precedence >= operators.last.lexeme.precedence
output.push(operators.pop)
end
diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb
index 649da745eea..db6cca27f1c 100644
--- a/lib/gitlab/ci/pipeline/metrics.rb
+++ b/lib/gitlab/ci/pipeline/metrics.rb
@@ -36,6 +36,15 @@ module Gitlab
Gitlab::Metrics.counter(name, comment)
end
end
+
+ def pipelines_created_counter
+ strong_memoize(:pipelines_created_count) do
+ name = :pipelines_created_total
+ comment = 'Counter of pipelines created'
+
+ Gitlab::Metrics.counter(name, comment)
+ end
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 114a46ca9f6..3be3fa63b92 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -11,9 +11,7 @@ module Gitlab
delegate :dig, to: :@seed_attributes
- # When the `ci_dag_limit_needs` is enabled it uses the lower limit
- LOW_NEEDS_LIMIT = 10
- HARD_NEEDS_LIMIT = 50
+ DEFAULT_NEEDS_LIMIT = 10
def initialize(pipeline, attributes, previous_stages)
@pipeline = pipeline
@@ -142,10 +140,10 @@ module Gitlab
end
def max_needs_allowed
- if Feature.enabled?(:ci_dag_limit_needs, @project, default_enabled: true)
- LOW_NEEDS_LIMIT
+ if ::Gitlab::Ci::Features.ci_plan_needs_size_limit?(@pipeline.project)
+ @pipeline.project.actual_limits.ci_needs_size_limit
else
- HARD_NEEDS_LIMIT
+ DEFAULT_NEEDS_LIMIT
end
end
diff --git a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb
index fa6337166d5..210eb17f2d3 100644
--- a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb
+++ b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb
@@ -17,7 +17,7 @@ module Gitlab
end
def status
- head_reports.errors_count.positive? ? STATUS_FAILED : STATUS_SUCCESS
+ head_reports.errors_count > 0 ? STATUS_FAILED : STATUS_SUCCESS
end
def existing_errors
diff --git a/lib/gitlab/ci/reports/test_report_summary.rb b/lib/gitlab/ci/reports/test_report_summary.rb
index 85b83b790e7..3e7227b7223 100644
--- a/lib/gitlab/ci/reports/test_report_summary.rb
+++ b/lib/gitlab/ci/reports/test_report_summary.rb
@@ -4,42 +4,17 @@ module Gitlab
module Ci
module Reports
class TestReportSummary
- attr_reader :all_results
-
- def initialize(all_results)
- @all_results = all_results
+ def initialize(build_report_results)
+ @build_report_results = build_report_results
+ @suite_summary = TestSuiteSummary.new(@build_report_results)
end
def total
- TestSuiteSummary.new(all_results)
- end
-
- def total_time
- total.total_time
- end
-
- def total_count
- total.total_count
- end
-
- def success_count
- total.success_count
- end
-
- def failed_count
- total.failed_count
- end
-
- def skipped_count
- total.skipped_count
- end
-
- def error_count
- total.error_count
+ @suite_summary.to_h
end
def test_suites
- all_results
+ @build_report_results
.group_by(&:tests_name)
.transform_values { |results| TestSuiteSummary.new(results) }
end
diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb
index 86ba725c71e..a5a630642e5 100644
--- a/lib/gitlab/ci/reports/test_reports.rb
+++ b/lib/gitlab/ci/reports/test_reports.rb
@@ -43,9 +43,7 @@ module Gitlab
end
def suite_errors
- test_suites.each_with_object({}) do |(name, suite), errors|
- errors[suite.name] = suite.suite_error if suite.suite_error
- end
+ test_suites.transform_values(&:suite_error).compact
end
TestCase::STATUS_TYPES.each do |status_type|
diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb
index 28b81e7a471..5ee779227ec 100644
--- a/lib/gitlab/ci/reports/test_suite.rb
+++ b/lib/gitlab/ci/reports/test_suite.rb
@@ -28,7 +28,7 @@ module Gitlab
def total_count
return 0 if suite_error
- test_cases.values.sum(&:count)
+ [success_count, failed_count, skipped_count, error_count].sum
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/ci/reports/test_suite_summary.rb b/lib/gitlab/ci/reports/test_suite_summary.rb
index f9b0bedb712..32b06d0ad49 100644
--- a/lib/gitlab/ci/reports/test_suite_summary.rb
+++ b/lib/gitlab/ci/reports/test_suite_summary.rb
@@ -4,45 +4,54 @@ module Gitlab
module Ci
module Reports
class TestSuiteSummary
- attr_reader :results
-
- def initialize(results)
- @results = results
+ def initialize(build_report_results)
+ @build_report_results = build_report_results
end
def name
- @name ||= results.first.tests_name
+ @name ||= @build_report_results.first.tests_name
end
# rubocop: disable CodeReuse/ActiveRecord
def build_ids
- results.pluck(:build_id)
+ @build_report_results.pluck(:build_id)
end
def total_time
- @total_time ||= results.sum(&:tests_duration)
+ @total_time ||= @build_report_results.sum(&:tests_duration)
end
def success_count
- @success_count ||= results.sum(&:tests_success)
+ @success_count ||= @build_report_results.sum(&:tests_success)
end
def failed_count
- @failed_count ||= results.sum(&:tests_failed)
+ @failed_count ||= @build_report_results.sum(&:tests_failed)
end
def skipped_count
- @skipped_count ||= results.sum(&:tests_skipped)
+ @skipped_count ||= @build_report_results.sum(&:tests_skipped)
end
def error_count
- @error_count ||= results.sum(&:tests_errored)
+ @error_count ||= @build_report_results.sum(&:tests_errored)
end
def total_count
@total_count ||= [success_count, failed_count, skipped_count, error_count].sum
end
# rubocop: disable CodeReuse/ActiveRecord
+
+ def to_h
+ {
+ time: total_time,
+ count: total_count,
+ success: success_count,
+ failed: failed_count,
+ skipped: skipped_count,
+ error: error_count
+ }
+ end
end
end
end
diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb
new file mode 100644
index 00000000000..2171637687f
--- /dev/null
+++ b/lib/gitlab/ci/runner_instructions.rb
@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class RunnerInstructions
+ class ArgumentError < ::ArgumentError; end
+
+ include Gitlab::Allowable
+
+ OS = {
+ linux: {
+ human_readable_name: "Linux",
+ download_locations: {
+ amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64",
+ '386': "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-386",
+ arm: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm",
+ arm64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-arm64"
+ },
+ install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/linux/install.sh",
+ runner_executable: "sudo gitlab-runner"
+ },
+ osx: {
+ human_readable_name: "macOS",
+ download_locations: {
+ amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-darwin-amd64"
+ },
+ install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/osx/install.sh",
+ runner_executable: "sudo gitlab-runner"
+ },
+ windows: {
+ human_readable_name: "Windows",
+ download_locations: {
+ amd64: "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-windows-amd64.exe",
+ '386': "https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-windows-386.exe"
+ },
+ install_script_template_path: "lib/gitlab/ci/runner_instructions/templates/windows/install.ps1",
+ runner_executable: "./gitlab-runner.exe"
+ }
+ }.freeze
+
+ OTHER_ENVIRONMENTS = {
+ docker: {
+ human_readable_name: "Docker",
+ installation_instructions_url: "https://docs.gitlab.com/runner/install/docker.html"
+ },
+ kubernetes: {
+ human_readable_name: "Kubernetes",
+ installation_instructions_url: "https://docs.gitlab.com/runner/install/kubernetes.html"
+ }
+ }.freeze
+
+ attr_reader :errors
+
+ def initialize(current_user:, group: nil, project: nil, os:, arch:)
+ @current_user = current_user
+ @group = group
+ @project = project
+ @os = os
+ @arch = arch
+ @errors = []
+
+ validate_params
+ end
+
+ def install_script
+ with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError] do
+ raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Architecture not found for OS') unless environment[:download_locations].key?(@arch.to_sym)
+
+ replace_variables(get_file(environment[:install_script_template_path]))
+ end
+ end
+
+ def register_command
+ with_error_handling [Gitlab::Ci::RunnerInstructions::ArgumentError, Gitlab::Access::AccessDeniedError] do
+ raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('No runner executable') unless environment[:runner_executable]
+
+ server_url = Gitlab::Routing.url_helpers.root_url(only_path: false)
+ runner_executable = environment[:runner_executable]
+
+ "#{runner_executable} register --url #{server_url} --registration-token #{registration_token}"
+ end
+ end
+
+ private
+
+ def with_error_handling(exceptions)
+ return if errors.present?
+
+ yield
+ rescue *exceptions => e
+ @errors << e.message
+ nil
+ end
+
+ def environment
+ @environment ||= OS[@os.to_sym] || ( raise Gitlab::Ci::RunnerInstructions::ArgumentError, s_('Invalid OS') )
+ end
+
+ def validate_params
+ @errors << s_('Missing OS') unless @os.present?
+ @errors << s_('Missing arch') unless @arch.present?
+ end
+
+ def replace_variables(expression)
+ expression.sub('${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION}', "#{environment[:download_locations][@arch.to_sym]}")
+ end
+
+ def get_file(path)
+ File.read(path)
+ end
+
+ def registration_token
+ project_token || group_token || instance_token
+ end
+
+ def project_token
+ return unless @project
+ raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_pipeline, @project)
+
+ @project.runners_token
+ end
+
+ def group_token
+ return unless @group
+ raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_group, @group)
+
+ @group.runners_token
+ end
+
+ def instance_token
+ raise Gitlab::Access::AccessDeniedError unless @current_user&.admin?
+
+ Gitlab::CurrentSettings.runners_registration_token
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/runner_instructions/templates/linux/install.sh b/lib/gitlab/ci/runner_instructions/templates/linux/install.sh
new file mode 100644
index 00000000000..6c8a0796d23
--- /dev/null
+++ b/lib/gitlab/ci/runner_instructions/templates/linux/install.sh
@@ -0,0 +1,12 @@
+# Download the binary for your system
+sudo curl -L --output /usr/local/bin/gitlab-runner ${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION}
+
+# Give it permissions to execute
+sudo chmod +x /usr/local/bin/gitlab-runner
+
+# Create a GitLab CI user
+sudo useradd --comment 'GitLab Runner' --create-home gitlab-runner --shell /bin/bash
+
+# Install and run as service
+sudo gitlab-runner install --user=gitlab-runner --working-directory=/home/gitlab-runner
+sudo gitlab-runner start
diff --git a/lib/gitlab/ci/runner_instructions/templates/osx/install.sh b/lib/gitlab/ci/runner_instructions/templates/osx/install.sh
new file mode 100644
index 00000000000..de4ee3e52fc
--- /dev/null
+++ b/lib/gitlab/ci/runner_instructions/templates/osx/install.sh
@@ -0,0 +1,11 @@
+# Download the binary for your system
+sudo curl --output /usr/local/bin/gitlab-runner ${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION}
+
+# Give it permissions to execute
+sudo chmod +x /usr/local/bin/gitlab-runner
+
+# The rest of commands execute as the user who will run the Runner
+# Register the Runner (steps below), then run
+cd ~
+gitlab-runner install
+gitlab-runner start
diff --git a/lib/gitlab/ci/runner_instructions/templates/windows/install.ps1 b/lib/gitlab/ci/runner_instructions/templates/windows/install.ps1
new file mode 100644
index 00000000000..dc37f88543c
--- /dev/null
+++ b/lib/gitlab/ci/runner_instructions/templates/windows/install.ps1
@@ -0,0 +1,13 @@
+# Run PowerShell: https://docs.microsoft.com/en-us/powershell/scripting/windows-powershell/starting-windows-powershell?view=powershell-7#with-administrative-privileges-run-as-administrator
+# Create a folder somewhere in your system ex.: C:\GitLab-Runner
+New-Item -Path 'C:\GitLab-Runner' -ItemType Directory
+
+# Enter the folder
+cd 'C:\GitLab-Runner'
+
+# Dowload binary
+Invoke-WebRequest -Uri "${GITLAB_CI_RUNNER_DOWNLOAD_LOCATION}" -OutFile "gitlab-runner.exe"
+
+# Register the Runner (steps below), then run
+.\gitlab-runner.exe install
+.\gitlab-runner.exe start
diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb
index 76ad113aad9..88846f724e7 100644
--- a/lib/gitlab/ci/status/build/failed.rb
+++ b/lib/gitlab/ci/status/build/failed.rb
@@ -24,7 +24,8 @@ module Gitlab
downstream_bridge_project_not_found: 'downstream project could not be found',
insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline',
bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline',
- downstream_pipeline_creation_failed: 'downstream pipeline can not be created'
+ downstream_pipeline_creation_failed: 'downstream pipeline can not be created',
+ secrets_provider_not_found: 'secrets provider can not be found'
}.freeze
private_constant :REASONS
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index c10d87a537b..968ff0fce89 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -162,4 +162,4 @@ include:
- template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
- template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
- - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+ - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml
index 5f4bd631db6..c1815baf7e6 100644
--- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml
@@ -1,4 +1,4 @@
-# This template is deprecated and will be removed as part of GitLab 13.2!
+# This template is deprecated.
#
# If you have referenced this template in your CI pipeline, please
# update your CI configuration by replacing the following occurrence(s):
@@ -20,12 +20,8 @@ stages:
- deploy
- production
-before_script:
- - printf '\nWARNING!\nThis job includes "Deploy-ECS.gitlab-ci.yml". Please rename this to "AWS/Deploy-ECS.gitlab-ci.yml".\n'
-
-variables:
- AUTO_DEVOPS_PLATFORM_TARGET: ECS
-
-include:
- - template: Jobs/Build.gitlab-ci.yml
- - template: Jobs/Deploy/ECS.gitlab-ci.yml
+"error: Template has moved":
+ stage: deploy
+ script:
+ - echo "Deploy-ECS.gitlab-ci.yml has been moved to AWS/Deploy-ECS.gitlab-ci.yml, see https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-the-aws-elastic-container-service-ecs for more details."
+ - exit 1
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index dbe870953ae..0c3598a61a7 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -1,6 +1,6 @@
build:
stage: build
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.3.1"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0"
variables:
DOCKER_TLS_CERTDIR: ""
services:
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 6b76d7e0c9b..cf851c875ee 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -7,7 +7,7 @@ code_quality:
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
- CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10"
+ CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.10-gitlab.1"
needs: []
script:
- |
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index d7d927ac8ee..f234008dad4 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.dast-auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.2"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.0"
dast_environment_deploy:
extends: .dast-auto-deploy
@@ -23,7 +23,7 @@ dast_environment_deploy:
when: never
- if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH
when: never
- - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given
+ - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given
when: never
- if: $CI_COMMIT_BRANCH &&
$CI_KUBERNETES_ACTIVE &&
@@ -46,7 +46,7 @@ stop_dast_environment:
when: never
- if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH
when: never
- - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given
+ - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given
when: never
- if: $CI_COMMIT_BRANCH &&
$CI_KUBERNETES_ACTIVE &&
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 66c60e85892..76fb2948144 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.2"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v1.0.0"
dependencies: []
include:
diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml
index b437ddbd734..4a9849c85c9 100644
--- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml
@@ -5,7 +5,7 @@ load_performance:
variables:
DOCKER_TLS_CERTDIR: ""
K6_IMAGE: loadimpact/k6
- K6_VERSION: 0.26.2
+ K6_VERSION: 0.27.0
K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js
K6_OPTIONS: ''
services:
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
new file mode 100644
index 00000000000..e87f0f28d01
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
@@ -0,0 +1,146 @@
+stages:
+ - build
+ - test
+ - deploy
+ - fuzz
+
+variables:
+ FUZZAPI_PROFILE: Quick
+ FUZZAPI_VERSION: latest
+ FUZZAPI_CONFIG: "/app/.gitlab-api-fuzzing.yml"
+ FUZZAPI_TIMEOUT: 30
+ FUZZAPI_REPORT: gl-api-fuzzing-report.xml
+ #
+ FUZZAPI_D_NETWORK: testing-net
+ #
+ # Wait up to 5 minutes for API Fuzzer and target url to become
+ # available (non 500 response to HTTP(s))
+ FUZZAPI_SERVICE_START_TIMEOUT: "300"
+ #
+
+apifuzzer_fuzz:
+ stage: fuzz
+ image: docker:19.03.12
+ variables:
+ DOCKER_DRIVER: overlay2
+ DOCKER_TLS_CERTDIR: ""
+ FUZZAPI_PROJECT: $CI_PROJECT_PATH
+ FUZZAPI_API: http://apifuzzer:80
+ allow_failure: true
+ rules:
+ - if: $API_FUZZING_DISABLED
+ when: never
+ - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $FUZZAPI_HAR == null &&
+ $FUZZAPI_OPENAPI == null &&
+ $FUZZAPI_D_WORKER_IMAGE == null
+ when: never
+ - if: $FUZZAPI_D_WORKER_IMAGE == null &&
+ $FUZZAPI_TARGET_URL == null
+ when: never
+ - if: $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+ services:
+ - docker:19.03.12-dind
+ script:
+ #
+ - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+ #
+ - docker network create --driver bridge $FUZZAPI_D_NETWORK
+ #
+ # Run user provided pre-script
+ - sh -c "$FUZZAPI_PRE_SCRIPT"
+ #
+ # Start peach testing engine container
+ - |
+ docker run -d \
+ --name apifuzzer \
+ --network $FUZZAPI_D_NETWORK \
+ -e Proxy:Port=8000 \
+ -e TZ=America/Los_Angeles \
+ -e FUZZAPI_API=http://127.0.0.1:80 \
+ -e FUZZAPI_PROJECT \
+ -e FUZZAPI_PROFILE \
+ -e FUZZAPI_CONFIG \
+ -e FUZZAPI_REPORT \
+ -e FUZZAPI_HAR \
+ -e FUZZAPI_OPENAPI \
+ -e FUZZAPI_TARGET_URL \
+ -e FUZZAPI_OVERRIDES_FILE \
+ -e FUZZAPI_OVERRIDES_ENV \
+ -e FUZZAPI_OVERRIDES_CMD \
+ -e FUZZAPI_OVERRIDES_INTERVAL \
+ -e FUZZAPI_TIMEOUT \
+ -e FUZZAPI_VERBOSE \
+ -e FUZZAPI_SERVICE_START_TIMEOUT \
+ -e GITLAB_FEATURES \
+ -v $CI_PROJECT_DIR:/app \
+ -p 80:80 \
+ -p 8000:8000 \
+ -p 514:514 \
+ --restart=no \
+ registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing-src:${FUZZAPI_VERSION}-engine
+ #
+ # Start target container
+ - |
+ if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \
+ docker run -d \
+ --name target \
+ --network $FUZZAPI_D_NETWORK \
+ $FUZZAPI_D_TARGET_ENV \
+ $FUZZAPI_D_TARGET_PORTS \
+ $FUZZAPI_D_TARGET_VOLUME \
+ --restart=no \
+ $FUZZAPI_D_TARGET_IMAGE \
+ ; fi
+ #
+ # Start worker container
+ - |
+ if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \
+ echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE" \
+ docker run \
+ --name worker \
+ --network $FUZZAPI_D_NETWORK \
+ -e FUZZAPI_API=http://apifuzzer:80 \
+ -e FUZZAPI_PROJECT \
+ -e FUZZAPI_PROFILE \
+ -e FUZZAPI_AUTOMATION_CMD \
+ -e FUZZAPI_CONFIG \
+ -e FUZZAPI_REPORT \
+ -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \
+ $FUZZAPI_D_WORKER_ENV \
+ $FUZZAPI_D_WORKER_PORTS \
+ $FUZZAPI_D_WORKER_VOLUME \
+ --restart=no \
+ $FUZZAPI_D_WORKER_IMAGE \
+ ; fi
+ #
+ # Wait for testing to complete if api fuzzer is scanning
+ - if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI" != "" ]; then echo "Waiting for API Fuzzer to exit"; docker wait apifuzzer; fi
+ #
+ # Run user provided pre-script
+ - sh -c "$FUZZAPI_POST_SCRIPT"
+ #
+ after_script:
+ #
+ # Shutdown all containers
+ - echo "Stopping all containers"
+ - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi
+ - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker stop worker; fi
+ - docker stop apifuzzer
+ #
+ # Save docker logs
+ - docker logs apifuzzer &> gl-api_fuzzing-logs.log
+ - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi
+ - if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then docker logs worker &> gl-api_fuzzing-worker-logs.log; fi
+ #
+ artifacts:
+ when: always
+ paths:
+ - ./gl-api_fuzzing*.log
+ - ./gl-api_fuzzing*.zip
+ reports:
+ junit: $FUZZAPI_REPORT
+
+# end
diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
index 2fab8b95a3d..3f47e575afd 100644
--- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
@@ -3,22 +3,26 @@
variables:
# Which branch we want to run full fledged long running fuzzing jobs.
# All others will run fuzzing regression
- COVERAGE_FUZZING_BRANCH: "$CI_DEFAULT_BRANCH"
- # This is using semantic version and will always download latest v1 gitlab-cov-fuzz release
- COVERAGE_FUZZING_VERSION: v1
+ COVFUZZ_BRANCH: "$CI_DEFAULT_BRANCH"
+ # This is using semantic version and will always download latest v2 gitlab-cov-fuzz release
+ COVFUZZ_VERSION: v2
# This is for users who have an offline environment and will have to replicate gitlab-cov-fuzz release binaries
# to their own servers
- COVERAGE_FUZZING_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw"
+ COVFUZZ_URL_PREFIX: "https://gitlab.com/gitlab-org/security-products/analyzers/gitlab-cov-fuzz/-/raw"
+
.fuzz_base:
stage: fuzz
allow_failure: true
before_script:
+ - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN
+ - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN
+ - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID
- if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi
- - wget -O gitlab-cov-fuzz "${COVERAGE_FUZZING_URL_PREFIX}"/"${COVERAGE_FUZZING_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64
+ - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64
- chmod a+x gitlab-cov-fuzz
- export REGRESSION=true
- - if [[ $CI_COMMIT_BRANCH = $COVERAGE_FUZZING_BRANCH ]]; then REGRESSION=false; fi;
+ - if [[ $CI_COMMIT_BRANCH = $COVFUZZ_BRANCH ]]; then REGRESSION=false; fi;
artifacts:
paths:
- corpus
@@ -28,7 +32,7 @@ variables:
coverage_fuzzing: gl-coverage-fuzzing-report.json
when: always
rules:
- - if: $COVERAGE_FUZZING_DISABLED
+ - if: $COVFUZZ_DISABLED
when: never
- if: $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/
- if: $CI_RUNNER_EXECUTABLE_ARCH == "linux"
diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
index 37f6cd216ca..d5275c57ef8 100644
--- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
@@ -111,6 +111,7 @@ gemnasium-dependency_scanning:
- '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}'
- '{package-lock.json,*/package-lock.json,*/*/package-lock.json}'
- '{yarn.lock,*/yarn.lock,*/*/yarn.lock}'
+ - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}'
gemnasium-maven-dependency_scanning:
extends: .ds-analyzer
@@ -144,8 +145,8 @@ gemnasium-python-dependency_scanning:
- '{Pipfile,*/Pipfile,*/*/Pipfile}'
- '{requires.txt,*/requires.txt,*/*/requires.txt}'
- '{setup.py,*/setup.py,*/*/setup.py}'
- # Support passing of $PIP_REQUIREMENTS_FILE
- # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning
+ # Support passing of $PIP_REQUIREMENTS_FILE
+ # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning
- if: $CI_COMMIT_BRANCH &&
$GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
$DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ &&
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index f0e2f48dd5c..6eb17341472 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -52,8 +52,7 @@ sast:
rules:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- - if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/
+ - if: $CI_COMMIT_BRANCH
script:
- /analyzer run
@@ -65,7 +64,6 @@ bandit-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /bandit/
exists:
- '**/*.py'
@@ -106,7 +104,6 @@ flawfinder-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /flawfinder/
exists:
- '**/*.c'
@@ -120,7 +117,6 @@ kubesec-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /kubesec/ &&
$SCAN_KUBERNETES_MANIFESTS == 'true'
@@ -132,7 +128,6 @@ gosec-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /gosec/
exists:
- '**/*.go'
@@ -145,7 +140,6 @@ nodejs-scan-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/
exists:
- 'package.json'
@@ -158,7 +152,6 @@ phpcs-security-audit-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/
exists:
- '**/*.php'
@@ -171,7 +164,6 @@ pmd-apex-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /pmd-apex/
exists:
- '**/*.cls'
@@ -184,7 +176,6 @@ secrets-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /secrets/
security-code-scan-sast:
@@ -195,7 +186,6 @@ security-code-scan-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /security-code-scan/
exists:
- '**/*.csproj'
@@ -209,7 +199,6 @@ sobelow-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /sobelow/
exists:
- 'mix.exs'
@@ -222,7 +211,6 @@ spotbugs-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /spotbugs/
exists:
- '**/*.groovy'
diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
index 441a57048e1..b897c7b482f 100644
--- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
@@ -7,6 +7,8 @@
variables:
SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
SECRETS_ANALYZER_VERSION: "3"
+ SECRET_DETECTION_EXCLUDED_PATHS: ""
+
.secret-analyzer:
stage: test
@@ -21,8 +23,7 @@ secret_detection_default_branch:
rules:
- if: $SECRET_DETECTION_DISABLED
when: never
- - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsecret_detection\b/
+ - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH
script:
- /analyzer run
@@ -31,8 +32,7 @@ secret_detection:
rules:
- if: $SECRET_DETECTION_DISABLED
when: never
- - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsecret_detection\b/
+ - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
script:
- git fetch origin $CI_DEFAULT_BRANCH $CI_BUILD_REF_NAME
- export SECRET_DETECTION_COMMIT_TO=$(git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_BUILD_REF_NAME | tail -n 1)
diff --git a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml
index 77a1b57d92f..584e6966180 100644
--- a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml
@@ -1,4 +1,5 @@
rspec-rails-modified-path-specs:
+ image: ruby:2.6
stage: .pre
rules:
- if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "merge_train"
diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml
index d39bd234020..f964b3b2caf 100644
--- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml
@@ -11,7 +11,7 @@ load_performance:
image: docker:git
variables:
K6_IMAGE: loadimpact/k6
- K6_VERSION: 0.26.2
+ K6_VERSION: 0.27.0
K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js
K6_OPTIONS: ''
services:
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 8cf355bbfc1..b7046064f44 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -115,7 +115,7 @@ module Gitlab
end
def release(job)
- job[:release] if Gitlab::Ci::Features.release_generation_enabled?
+ job[:release]
end
def stage_builds_attributes(stage)
diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb
index 0a629075302..415f6f77214 100644
--- a/lib/gitlab/config/entry/legacy_validation_helpers.rb
+++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb
@@ -6,17 +6,27 @@ module Gitlab
module LegacyValidationHelpers
private
- def validate_duration(value)
- value.is_a?(String) && ChronicDuration.parse(value)
+ def validate_duration(value, parser = nil)
+ return false unless value.is_a?(String)
+
+ if parser && parser.respond_to?(:validate_duration)
+ parser.validate_duration(value)
+ else
+ ChronicDuration.parse(value)
+ end
rescue ChronicDuration::DurationParseError
false
end
- def validate_duration_limit(value, limit)
+ def validate_duration_limit(value, limit, parser = nil)
return false unless value.is_a?(String)
- ChronicDuration.parse(value).second.from_now <
- ChronicDuration.parse(limit).second.from_now
+ if parser && parser.respond_to?(:validate_duration_limit)
+ parser.validate_duration_limit(value, limit)
+ else
+ ChronicDuration.parse(value).second.from_now <
+ ChronicDuration.parse(limit).second.from_now
+ end
rescue ChronicDuration::DurationParseError
false
end
@@ -30,10 +40,18 @@ module Gitlab
end
def validate_variables(variables)
+ variables.is_a?(Hash) && variables.flatten.all?(&method(:validate_alphanumeric))
+ end
+
+ def validate_array_value_variables(variables)
variables.is_a?(Hash) &&
- variables.flatten.all? do |value|
- validate_string(value) || validate_integer(value)
- end
+ variables.keys.all?(&method(:validate_alphanumeric)) &&
+ variables.values.all?(&:present?) &&
+ variables.values.flatten(1).all?(&method(:validate_alphanumeric))
+ end
+
+ def validate_alphanumeric(value)
+ validate_string(value) || validate_integer(value)
end
def validate_integer(value)
diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb
index d1c23c41d35..a7ec98ace6e 100644
--- a/lib/gitlab/config/entry/validators.rb
+++ b/lib/gitlab/config/entry/validators.rb
@@ -106,12 +106,12 @@ module Gitlab
include LegacyValidationHelpers
def validate_each(record, attribute, value)
- unless validate_duration(value)
+ unless validate_duration(value, options[:parser])
record.errors.add(attribute, 'should be a duration')
end
if options[:limit]
- unless validate_duration_limit(value, options[:limit])
+ unless validate_duration_limit(value, options[:limit], options[:parser])
record.errors.add(attribute, 'should not exceed the limit')
end
end
@@ -272,10 +272,24 @@ module Gitlab
include LegacyValidationHelpers
def validate_each(record, attribute, value)
+ if options[:array_values]
+ validate_key_array_values(record, attribute, value)
+ else
+ validate_key_values(record, attribute, value)
+ end
+ end
+
+ def validate_key_values(record, attribute, value)
unless validate_variables(value)
record.errors.add(attribute, 'should be a hash of key value pairs')
end
end
+
+ def validate_key_array_values(record, attribute, value)
+ unless validate_array_value_variables(value)
+ record.errors.add(attribute, 'should be a hash of key value pairs, value can be an array')
+ end
+ end
end
class ExpressionValidator < ActiveModel::EachValidator
diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb
index af828acb9c0..dfcdbdf39e0 100644
--- a/lib/gitlab/config_checker/external_database_checker.rb
+++ b/lib/gitlab/config_checker/external_database_checker.rb
@@ -9,35 +9,41 @@ module Gitlab
notices = []
unless Gitlab::Database.postgresql_minimum_supported_version?
+ string_args = {
+ pg_version_current: Gitlab::Database.version,
+ pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION,
+ pg_requirements_url_open: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">'.html_safe,
+ pg_requirements_url_close: '</a>'.html_safe
+ }
+
notices <<
{
type: 'warning',
- message: _('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \
+ message: html_escape(_('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \
'%{pg_version_minimum} is required for this version of GitLab. ' \
'Please upgrade your environment to a supported PostgreSQL version, ' \
- 'see %{pg_requirements_url} for details.') % {
- pg_version_current: Gitlab::Database.version,
- pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION,
- pg_requirements_url: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>'
- }
+ 'see %{pg_requirements_url_open}database requirements%{pg_requirements_url_close} for details.')) % string_args
}
end
if Gitlab::Database.postgresql_upcoming_deprecation? && Gitlab::Database.within_deprecation_notice_window?
upcoming_deprecation = Gitlab::Database::UPCOMING_POSTGRES_VERSION_DETAILS
+ string_args = {
+ pg_version_upcoming: upcoming_deprecation[:pg_version_minimum],
+ gl_version_upcoming: upcoming_deprecation[:gl_version],
+ gl_version_upcoming_date: upcoming_deprecation[:gl_version_date],
+ pg_version_upcoming_url_open: "<a href=\"#{upcoming_deprecation[:url]}\">".html_safe,
+ pg_version_upcoming_url_close: '</a>'.html_safe
+ }
+
notices <<
{
type: 'warning',
- message: _('Note that PostgreSQL %{pg_version_upcoming} will become the minimum required ' \
+ message: html_escape(_('Note that PostgreSQL %{pg_version_upcoming} will become the minimum required ' \
'version in GitLab %{gl_version_upcoming} (%{gl_version_upcoming_date}). Please ' \
'consider upgrading your environment to a supported PostgreSQL version soon, ' \
- 'see <a href="%{pg_version_upcoming_url}">the related epic</a> for details.') % {
- pg_version_upcoming: upcoming_deprecation[:pg_version_minimum],
- gl_version_upcoming: upcoming_deprecation[:gl_version],
- gl_version_upcoming_date: upcoming_deprecation[:gl_version_date],
- pg_version_upcoming_url: upcoming_deprecation[:url]
- }
+ 'see %{pg_version_upcoming_url_open}the related epic%{pg_version_upcoming_url_close} for details.')) % string_args
}
end
diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb
index 07ae430c45e..6c6dd90e450 100644
--- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb
+++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb
@@ -6,7 +6,7 @@ module Gitlab
include BaseQuery
include GroupProjectsProvider
- attr_reader :projections, :query, :stage, :order, :options
+ attr_reader :projections, :query, :stage, :options
MAX_EVENTS = 50
diff --git a/lib/gitlab/cycle_analytics/summary/value.rb b/lib/gitlab/cycle_analytics/summary/value.rb
index e443e037517..36306fa7c45 100644
--- a/lib/gitlab/cycle_analytics/summary/value.rb
+++ b/lib/gitlab/cycle_analytics/summary/value.rb
@@ -34,7 +34,7 @@ module Gitlab
end
def to_s
- value.zero? ? '0' : value.to_s
+ value == 0 ? '0' : value.to_s
end
def to_i
diff --git a/lib/gitlab/cycle_analytics/summary_helper.rb b/lib/gitlab/cycle_analytics/summary_helper.rb
index 3cf9f463024..11e48679a40 100644
--- a/lib/gitlab/cycle_analytics/summary_helper.rb
+++ b/lib/gitlab/cycle_analytics/summary_helper.rb
@@ -4,7 +4,7 @@ module Gitlab
module CycleAnalytics
module SummaryHelper
def frequency(count, from, to)
- return Summary::Value::None.new if count.zero?
+ return Summary::Value::None.new if count == 0
freq = (count / days(from, to)).round(1)
diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb
index db799c094b2..077c71f1233 100644
--- a/lib/gitlab/danger/helper.rb
+++ b/lib/gitlab/danger/helper.rb
@@ -53,7 +53,7 @@ module Gitlab
def ee?
# Support former project name for `dev` and support local Danger run
- %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?('../../ee')
+ %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../../ee', __dir__))
end
def gitlab_helper
@@ -124,7 +124,7 @@ module Gitlab
}.freeze
# First-match win, so be sure to put more specific regex at the top...
CATEGORIES = {
- [%r{usage_data}, %r{^(\+|-).*(count|distinct_count)\(.*\)(.*)$}] => [:database, :backend],
+ [%r{usage_data\.rb}, %r{^(\+|-).*(count|distinct_count)\(.*\)(.*)$}] => [:database, :backend],
%r{\Adoc/.*(\.(md|png|gif|jpg))\z} => :docs,
%r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs,
@@ -170,10 +170,15 @@ module Gitlab
%r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity,
%r{\A(ee/)?scripts/} => :engineering_productivity,
%r{\Atooling/} => :engineering_productivity,
+ %r{(CODEOWNERS)} => :engineering_productivity,
+
+ %r{\A(ee/)?spec/features/} => :test,
+ %r{\A(ee/)?spec/support/shared_examples/features/} => :test,
+ %r{\A(ee/)?spec/support/shared_contexts/features/} => :test,
+ %r{\A(ee/)?spec/support/helpers/features/} => :test,
%r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend,
%r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend,
- %r{\A(ee/)?spec/features/} => :test,
%r{\A(ee/)?spec/} => :backend,
%r{\A(ee/)?vendor/} => :backend,
%r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend,
@@ -249,6 +254,10 @@ module Gitlab
"/label #{labels_list(labels, sep: ' ')}"
end
+ def changed_files(regex)
+ all_changed_files.grep(regex)
+ end
+
private
def has_database_scoped_labels?(current_mr_labels)
diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb
index ed4af3f4a43..2e6181d1cab 100644
--- a/lib/gitlab/danger/roulette.rb
+++ b/lib/gitlab/danger/roulette.rb
@@ -1,6 +1,7 @@
# frozen_string_literal: true
require_relative 'teammate'
+require_relative 'request_helper'
module Gitlab
module Danger
@@ -12,45 +13,49 @@ module Gitlab
database: false
}.freeze
- Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role)
+ Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role, :timezone_experiment)
+
+ def team_mr_author
+ team.find { |person| person.username == mr_author_username }
+ end
# Assigns GitLab team members to be reviewer and maintainer
# for each change category that a Merge Request contains.
#
# @return [Array<Spin>]
- def spin(project, categories, branch_name, timezone_experiment: false)
- team =
- begin
- project_team(project)
- rescue => err
- warn("Reviewer roulette failed to load team data: #{err.message}")
- []
- end
-
- canonical_branch_name = canonical_branch_name(branch_name)
-
- spin_per_category = categories.each_with_object({}) do |category, memo|
+ def spin(project, categories, timezone_experiment: false)
+ spins = categories.map do |category|
including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(category, timezone_experiment)
- memo[category] = spin_for_category(team, project, category, canonical_branch_name, timezone_experiment: including_timezone)
+ spin_for_category(project, category, timezone_experiment: including_timezone)
end
- spin_per_category.map do |category, spin|
- case category
+ backend_spin = spins.find { |spin| spin.category == :backend }
+
+ spins.each do |spin|
+ including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(spin.category, timezone_experiment)
+ case spin.category
+ when :qa
+ # MR includes QA changes, but also other changes, and author isn't an SET
+ if categories.size > 1 && !team_mr_author&.reviewer?(project, spin.category, [])
+ spin.optional_role = :maintainer
+ end
when :test
+ spin.optional_role = :maintainer
+
if spin.reviewer.nil?
# Fetch an already picked backend reviewer, or pick one otherwise
- spin.reviewer = spin_per_category[:backend]&.reviewer || spin_for_category(team, project, :backend, canonical_branch_name).reviewer
+ spin.reviewer = backend_spin&.reviewer || spin_for_category(project, :backend, timezone_experiment: including_timezone).reviewer
end
when :engineering_productivity
if spin.maintainer.nil?
# Fetch an already picked backend maintainer, or pick one otherwise
- spin.maintainer = spin_per_category[:backend]&.maintainer || spin_for_category(team, project, :backend, canonical_branch_name).maintainer
+ spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer
end
end
-
- spin
end
+
+ spins
end
# Looks up the current list of GitLab team members and parses it into a
@@ -73,14 +78,9 @@ module Gitlab
# @return [Array<Teammate>]
def project_team(project_name)
team.select { |member| member.in_project?(project_name) }
- end
-
- def canonical_branch_name(branch_name)
- branch_name.gsub(/^[ce]e-|-[ce]e$/, '')
- end
-
- def new_random(seed)
- Random.new(Digest::MD5.hexdigest(seed).to_i(16))
+ rescue => err
+ warn("Reviewer roulette failed to load team data: #{err.message}")
+ []
end
# Known issue: If someone is rejected due to OOO, and then becomes not OOO, the
@@ -113,16 +113,35 @@ module Gitlab
# @param [Teammate] person
# @return [Boolean]
def mr_author?(person)
- person.username == gitlab.mr_author
+ person.username == mr_author_username
+ end
+
+ def mr_author_username
+ helper.gitlab_helper&.mr_author || `whoami`
+ end
+
+ def mr_source_branch
+ return `git rev-parse --abbrev-ref HEAD` unless helper.gitlab_helper&.mr_json
+
+ helper.gitlab_helper.mr_json['source_branch']
+ end
+
+ def mr_labels
+ helper.gitlab_helper&.mr_labels || []
+ end
+
+ def new_random(seed)
+ Random.new(Digest::MD5.hexdigest(seed).to_i(16))
end
def spin_role_for_category(team, role, project, category)
team.select do |member|
- member.public_send("#{role}?", project, category, gitlab.mr_labels) # rubocop:disable GitlabSecurity/PublicSend
+ member.public_send("#{role}?", project, category, mr_labels) # rubocop:disable GitlabSecurity/PublicSend
end
end
- def spin_for_category(team, project, category, branch_name, timezone_experiment: false)
+ def spin_for_category(project, category, timezone_experiment: false)
+ team = project_team(project)
reviewers, traintainers, maintainers =
%i[reviewer traintainer maintainer].map do |role|
spin_role_for_category(team, role, project, category)
@@ -132,11 +151,11 @@ module Gitlab
# https://gitlab.com/gitlab-org/gitlab/issues/26723
# Make traintainers have triple the chance to be picked as a reviewer
- random = new_random(branch_name)
+ random = new_random(mr_source_branch)
reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random, timezone_experiment: timezone_experiment)
maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment)
- Spin.new(category, reviewer, maintainer)
+ Spin.new(category, reviewer, maintainer, false, timezone_experiment)
end
end
end
diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb
index f7da66e77cd..9b389907090 100644
--- a/lib/gitlab/danger/teammate.rb
+++ b/lib/gitlab/danger/teammate.rb
@@ -3,10 +3,11 @@
module Gitlab
module Danger
class Teammate
- attr_reader :username, :name, :role, :projects, :available, :tz_offset_hours
+ attr_reader :options, :username, :name, :role, :projects, :available, :tz_offset_hours
# The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb
def initialize(options = {})
+ @options = options
@username = options['username']
@name = options['name']
@markdown_name = options['markdown_name']
@@ -16,6 +17,16 @@ module Gitlab
@tz_offset_hours = options['tz_offset_hours']
end
+ def to_h
+ options
+ end
+
+ def ==(other)
+ return false unless other.respond_to?(:username)
+
+ other.username == username
+ end
+
def in_project?(name)
projects&.has_key?(name)
end
@@ -69,9 +80,9 @@ module Gitlab
def offset_diff_compared_to_author(author)
diff = floored_offset_hours - author.floored_offset_hours
- return "same timezone as `@#{author.username}`" if diff.zero?
+ return "same timezone as `@#{author.username}`" if diff == 0
- ahead_or_behind = diff < 0 ? 'behind' : 'ahead'
+ ahead_or_behind = diff < 0 ? 'behind' : 'ahead of'
pluralized_hours = pluralize(diff.abs, 'hour', 'hours')
"#{pluralized_hours} #{ahead_or_behind} `@#{author.username}`"
diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb
index d88ca6d7fe3..e7df9fd27f0 100644
--- a/lib/gitlab/database.rb
+++ b/lib/gitlab/database.rb
@@ -2,8 +2,6 @@
module Gitlab
module Database
- include Gitlab::Metrics::Methods
-
# Minimum PostgreSQL version requirement per documentation:
# https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements
MINIMUM_POSTGRES_VERSION = 11
@@ -24,6 +22,7 @@ module Gitlab
# https://www.postgresql.org/docs/9.2/static/datatype-numeric.html
MAX_INT_VALUE = 2147483647
+ MIN_INT_VALUE = -2147483648
# The max value between MySQL's TIMESTAMP and PostgreSQL's timestampz:
# https://www.postgresql.org/docs/9.1/static/datatype-datetime.html
@@ -50,10 +49,6 @@ module Gitlab
# It does not include the default public schema
EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze
- define_histogram :gitlab_database_transaction_seconds do
- docstring "Time spent in database transactions, in seconds"
- end
-
def self.config
ActiveRecord::Base.configurations[Rails.env]
end
@@ -80,7 +75,7 @@ module Gitlab
# @deprecated
def self.postgresql?
- adapter_name.casecmp('postgresql').zero?
+ adapter_name.casecmp('postgresql') == 0
end
def self.read_only?
@@ -363,8 +358,11 @@ module Gitlab
# observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to
# record transaction durations.
def self.observe_transaction_duration(duration_seconds)
- labels = Gitlab::Metrics::Transaction.current&.labels || {}
- gitlab_database_transaction_seconds.observe(labels, duration_seconds)
+ if current_transaction = ::Gitlab::Metrics::Transaction.current
+ current_transaction.observe(:gitlab_database_transaction_seconds, duration_seconds) do
+ docstring "Time spent in database transactions, in seconds"
+ end
+ end
rescue Prometheus::Client::LabelSetValidator::LabelSetError => err
# Ensure that errors in recording these metrics don't affect the operation of the application
Rails.logger.error("Unable to observe database transaction duration: #{err}") # rubocop:disable Gitlab/RailsLogger
diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb
index ab069ce1da1..1762b81b7d8 100644
--- a/lib/gitlab/database/batch_count.rb
+++ b/lib/gitlab/database/batch_count.rb
@@ -16,6 +16,7 @@
# batch_count(::Clusters::Cluster.aws_installed.enabled, :cluster_id)
# batch_distinct_count(::Project, :creator_id)
# batch_distinct_count(::Project.with_active_services.service_desk_enabled.where(time_period), start: ::User.minimum(:id), finish: ::User.maximum(:id))
+# batch_sum(User, :sign_in_count)
module Gitlab
module Database
module BatchCount
@@ -27,6 +28,10 @@ module Gitlab
BatchCounter.new(relation, column: column).count(mode: :distinct, batch_size: batch_size, start: start, finish: finish)
end
+ def batch_sum(relation, column, batch_size: nil, start: nil, finish: nil)
+ BatchCounter.new(relation, column: nil, operation: :sum, operation_args: [column]).count(batch_size: batch_size, start: start, finish: finish)
+ end
+
class << self
include BatchCount
end
@@ -35,6 +40,7 @@ module Gitlab
class BatchCounter
FALLBACK = -1
MIN_REQUIRED_BATCH_SIZE = 1_250
+ DEFAULT_SUM_BATCH_SIZE = 1_000
MAX_ALLOWED_LOOPS = 10_000
SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep
ALLOWED_MODES = [:itself, :distinct].freeze
@@ -43,13 +49,16 @@ module Gitlab
DEFAULT_DISTINCT_BATCH_SIZE = 10_000
DEFAULT_BATCH_SIZE = 100_000
- def initialize(relation, column: nil)
+ def initialize(relation, column: nil, operation: :count, operation_args: nil)
@relation = relation
@column = column || relation.primary_key
+ @operation = operation
+ @operation_args = operation_args
end
def unwanted_configuration?(finish, batch_size, start)
- batch_size <= MIN_REQUIRED_BATCH_SIZE ||
+ (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) ||
+ (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) ||
(finish - start) / batch_size >= MAX_ALLOWED_LOOPS ||
start > finish
end
@@ -60,7 +69,7 @@ module Gitlab
check_mode!(mode)
# non-distinct have better performance
- batch_size ||= mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE
+ batch_size ||= batch_size_for_mode_and_operation(mode, @operation)
start = actual_start(start)
finish = actual_finish(finish)
@@ -91,11 +100,17 @@ module Gitlab
def batch_fetch(start, finish, mode)
# rubocop:disable GitlabSecurity/PublicSend
- @relation.select(@column).public_send(mode).where(between_condition(start, finish)).count
+ @relation.select(@column).public_send(mode).where(between_condition(start, finish)).send(@operation, *@operation_args)
end
private
+ def batch_size_for_mode_and_operation(mode, operation)
+ return DEFAULT_SUM_BATCH_SIZE if operation == :sum
+
+ mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE
+ end
+
def between_condition(start, finish)
return @column.between(start..(finish - 1)) if @column.is_a?(Arel::Attributes::Attribute)
diff --git a/lib/gitlab/database/connection_timer.rb b/lib/gitlab/database/connection_timer.rb
index ef8d52ba71c..f9b893ffd0f 100644
--- a/lib/gitlab/database/connection_timer.rb
+++ b/lib/gitlab/database/connection_timer.rb
@@ -23,7 +23,7 @@ module Gitlab
end
def interval_with_randomization
- interval + rand(RANDOMIZATION_INTERVAL) if interval.positive?
+ interval + rand(RANDOMIZATION_INTERVAL) if interval > 0
end
def current_clock_value
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 006a24da8fe..a618a3017b2 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -815,7 +815,7 @@ module Gitlab
BEFORE INSERT OR UPDATE
ON #{table}
FOR EACH ROW
- EXECUTE PROCEDURE #{trigger}()
+ EXECUTE FUNCTION #{trigger}()
EOF
end
@@ -1062,7 +1062,7 @@ into similar problems in the future (e.g. when new tables are created).
AND pg_class.relname = '#{table}'
SQL
- connection.select_value(check_sql).positive?
+ connection.select_value(check_sql) > 0
end
# Adds a check constraint to a table
diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_creator.rb
index 348dd1ba660..4c1b13fe3b5 100644
--- a/lib/gitlab/database/partitioning/partition_creator.rb
+++ b/lib/gitlab/database/partitioning/partition_creator.rb
@@ -24,7 +24,7 @@ module Gitlab
end
def create_partitions
- return unless Feature.enabled?(:postgres_dynamic_partition_creation, default_enabled: true)
+ Gitlab::AppLogger.info("Checking state of dynamic postgres partitions")
models.each do |model|
# Double-checking before getting the lease:
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index b676767f41d..e6d8ec55319 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -16,7 +16,9 @@ module Gitlab
BATCH_SIZE = 50_000
# Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column.
- # One partition is created per month between the given `min_date` and `max_date`.
+ # One partition is created per month between the given `min_date` and `max_date`. Also installs a trigger on
+ # the original table to copy writes into the partitioned table. To copy over historic data from before creation
+ # of the partitioned table, use the `enqueue_partitioning_data_migration` helper in a post-deploy migration.
#
# A copy of the original table is required as PG currently does not support partitioning existing tables.
#
@@ -56,10 +58,10 @@ module Gitlab
create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key)
create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date)
create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key)
- enqueue_background_migration(table_name, partitioned_table_name, primary_key)
end
- # Clean up a partitioned copy of an existing table. This deletes the partitioned table and all partitions.
+ # Clean up a partitioned copy of an existing table. First, deletes the database function and trigger that were
+ # used to copy writes to the partitioned table, then removes the partitioned table (also removing partitions).
#
# Example:
#
@@ -69,8 +71,6 @@ module Gitlab
assert_table_is_allowed(table_name)
assert_not_in_transaction_block(scope: ERROR_SCOPE)
- cleanup_migration_jobs(table_name)
-
with_lock_retries do
trigger_name = make_sync_trigger_name(table_name)
drop_trigger(table_name, trigger_name)
@@ -83,6 +83,38 @@ module Gitlab
drop_table(partitioned_table_name)
end
+ # Enqueue the background jobs that will backfill data in the partitioned table, by batch-copying records from
+ # original table. This helper should be called from a post-deploy migration.
+ #
+ # Example:
+ #
+ # enqueue_partitioning_data_migration :audit_events
+ #
+ def enqueue_partitioning_data_migration(table_name)
+ assert_table_is_allowed(table_name)
+
+ assert_not_in_transaction_block(scope: ERROR_SCOPE)
+
+ partitioned_table_name = make_partitioned_table_name(table_name)
+ primary_key = connection.primary_key(table_name)
+ enqueue_background_migration(table_name, partitioned_table_name, primary_key)
+ end
+
+ # Cleanup a previously enqueued background migration to copy data into a partitioned table. This will not
+ # prevent the enqueued jobs from executing, but instead cleans up information in the database used to track the
+ # state of the background migration. It should be safe to also remove the partitioned table even if the
+ # background jobs are still in-progress, as the absence of the table will cause them to safely exit.
+ #
+ # Example:
+ #
+ # cleanup_partitioning_data_migration :audit_events
+ #
+ def cleanup_partitioning_data_migration(table_name)
+ assert_table_is_allowed(table_name)
+
+ cleanup_migration_jobs(table_name)
+ end
+
def create_hash_partitions(table_name, number_of_partitions)
transaction do
(0..number_of_partitions - 1).each do |partition|
diff --git a/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb
new file mode 100644
index 00000000000..59bd24d3c37
--- /dev/null
+++ b/lib/gitlab/database/postgresql_adapter/dump_schema_versions_mixin.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module PostgresqlAdapter
+ module DumpSchemaVersionsMixin
+ extend ActiveSupport::Concern
+
+ def dump_schema_information # :nodoc:
+ versions = schema_migration.all_versions
+ Gitlab::Database::SchemaVersionFiles.touch_all(versions) if versions.any?
+
+ nil
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin.rb b/lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin.rb
deleted file mode 100644
index d8f96643dcb..00000000000
--- a/lib/gitlab/database/postgresql_adapter/schema_versions_copy_mixin.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Database
- module PostgresqlAdapter
- module SchemaVersionsCopyMixin
- extend ActiveSupport::Concern
-
- def dump_schema_information # :nodoc:
- versions = schema_migration.all_versions
- copy_versions_sql(versions) if versions.any?
- end
-
- private
-
- def copy_versions_sql(versions)
- sm_table = quote_table_name(schema_migration.table_name)
-
- sql = +"COPY #{sm_table} (version) FROM STDIN;\n"
- sql << versions.map { |v| Integer(v) }.sort.join("\n")
- sql << "\n\\.\n"
-
- sql
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb b/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb
new file mode 100644
index 00000000000..cf8342941c4
--- /dev/null
+++ b/lib/gitlab/database/postgresql_database_tasks/load_schema_versions_mixin.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module PostgresqlDatabaseTasks
+ module LoadSchemaVersionsMixin
+ extend ActiveSupport::Concern
+
+ def structure_load(*args)
+ super(*args)
+ Gitlab::Database::SchemaVersionFiles.load_all
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
index 6b9af51a6ab..4fbbfdc4914 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb
@@ -45,7 +45,7 @@ module Gitlab
reverts_for_type('namespace') do |path_before_rename, current_path|
matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path)
namespace = MigrationClasses::Namespace.joins(:route)
- .find_by(matches_path)&.becomes(MigrationClasses::Namespace)
+ .find_by(matches_path)&.becomes(MigrationClasses::Namespace) # rubocop: disable Cop/AvoidBecomes
if namespace
perform_rename(namespace, current_path, path_before_rename)
diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb
index ae9d77e635e..7c415287878 100644
--- a/lib/gitlab/database/schema_cleaner.rb
+++ b/lib/gitlab/database/schema_cleaner.rb
@@ -23,6 +23,11 @@ module Gitlab
structure.gsub!(/\n{3,}/, "\n\n")
io << structure
+ io << <<~MSG
+ -- schema_migrations.version information is no longer stored in this file,
+ -- but instead tracked in the db/schema_migrations directory
+ -- see https://gitlab.com/gitlab-org/gitlab/-/issues/218590 for details
+ MSG
nil
end
diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb
index 34daafd06de..dda4d8eecdb 100644
--- a/lib/gitlab/database/schema_helpers.rb
+++ b/lib/gitlab/database/schema_helpers.rb
@@ -25,7 +25,7 @@ module Gitlab
CREATE TRIGGER #{name}
#{fires} ON #{table_name}
FOR EACH ROW
- EXECUTE PROCEDURE #{function_name}()
+ EXECUTE FUNCTION #{function_name}()
SQL
end
diff --git a/lib/gitlab/database/schema_version_files.rb b/lib/gitlab/database/schema_version_files.rb
new file mode 100644
index 00000000000..27a942404ef
--- /dev/null
+++ b/lib/gitlab/database/schema_version_files.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class SchemaVersionFiles
+ SCHEMA_DIRECTORY = 'db/schema_migrations'
+ MIGRATION_DIRECTORIES = %w[db/migrate db/post_migrate].freeze
+ MIGRATION_VERSION_GLOB = '20[0-9][0-9]*'
+
+ def self.touch_all(versions_from_database)
+ versions_from_migration_files = find_versions_from_migration_files
+
+ version_filepaths = find_version_filenames.map { |f| schema_directory.join(f) }
+ FileUtils.rm(version_filepaths)
+
+ versions_to_create = versions_from_database & versions_from_migration_files
+ versions_to_create.each do |version|
+ version_filepath = schema_directory.join(version)
+
+ File.open(version_filepath, 'w') do |file|
+ file << Digest::SHA256.hexdigest(version)
+ end
+ end
+ end
+
+ def self.load_all
+ version_filenames = find_version_filenames
+ return if version_filenames.empty?
+
+ values = version_filenames.map { |vf| "('#{connection.quote_string(vf)}')" }
+ connection.execute(<<~SQL)
+ INSERT INTO schema_migrations (version)
+ VALUES #{values.join(',')}
+ ON CONFLICT DO NOTHING
+ SQL
+ end
+
+ def self.schema_directory
+ @schema_directory ||= Rails.root.join(SCHEMA_DIRECTORY)
+ end
+
+ def self.migration_directories
+ @migration_directories ||= MIGRATION_DIRECTORIES.map { |dir| Rails.root.join(dir) }
+ end
+
+ def self.find_version_filenames
+ Dir.glob(MIGRATION_VERSION_GLOB, base: schema_directory)
+ end
+
+ def self.find_versions_from_migration_files
+ migration_directories.each_with_object([]) do |directory, migration_versions|
+ directory_migrations = Dir.glob(MIGRATION_VERSION_GLOB, base: directory)
+ directory_versions = directory_migrations.map! { |m| m.split('_').first }
+
+ migration_versions.concat(directory_versions)
+ end
+ end
+
+ def self.connection
+ ActiveRecord::Base.connection
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb
new file mode 100644
index 00000000000..2633c29438a
--- /dev/null
+++ b/lib/gitlab/database/similarity_score.rb
@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class SimilarityScore
+ EMPTY_STRING = Arel.sql("''").freeze
+ EXPRESSION_ON_INVALID_INPUT = Arel::Nodes::NamedFunction.new('CAST', [Arel.sql('0').as('integer')]).freeze
+ DEFAULT_MULTIPLIER = 1
+
+ # This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity.
+ #
+ # Note: Calculating similarity score for large volume of records is inefficient. use SimilarityScore only for smaller
+ # resultset which is already filtered by other conditions (< 10_000 records).
+ #
+ # ==== Parameters
+ # * +search+ - [String] the user provided search string
+ # * +rules+ - [{ column: COLUMN, multiplier: 1 }, { column: COLUMN_2, multiplier: 0.5 }] rules for the scoring.
+ # * +column+ - Arel column expression, example: Project.arel_table["name"]
+ # * +multiplier+ - Integer or Float to increase or decrease the score (optional, defaults to 1)
+ #
+ # ==== Use case
+ #
+ # We'd like to search for projects by path, name and description. We want to rank higher the path and name matches, since
+ # it's more likely that the user was remembering the path or the name of the project.
+ #
+ # Rules:
+ # [
+ # { column: Project.arel_table['path'], multiplier: 1 },
+ # { column: Project.arel_table['name'], multiplier: 1 },
+ # { column: Project.arel_table['description'], multiplier: 0.5 }
+ # ]
+ #
+ # ==== Examples
+ #
+ # Similarity calculation based on one column:
+ #
+ # Gitlab::Database::SimilarityScore.build_expession(search: 'my input', rules: [{ column: Project.arel_table['name'] }])
+ #
+ # Similarity calculation based on two column, where the second column has lower priority:
+ #
+ # Gitlab::Database::SimilarityScore.build_expession(search: 'my input', rules: [
+ # { column: Project.arel_table['name'], multiplier: 1 },
+ # { column: Project.arel_table['description'], multiplier: 0.5 }
+ # ])
+ #
+ # Integration with an ActiveRecord query:
+ #
+ # table = Project.arel_table
+ #
+ # order_expression = Gitlab::Database::SimilarityScore.build_expession(search: 'input', rules: [
+ # { column: table['name'], multiplier: 1 },
+ # { column: table['description'], multiplier: 0.5 }
+ # ])
+ #
+ # Project.where("name LIKE ?", '%' + 'input' + '%').order(order_expression.desc)
+ #
+ # The expression can be also used in SELECT:
+ #
+ # results = Project.select(order_expression.as('similarity')).where("name LIKE ?", '%' + 'input' + '%').order(similarity: :desc)
+ # puts results.map(&:similarity)
+ #
+ def self.build_expression(search:, rules:)
+ return EXPRESSION_ON_INVALID_INPUT if search.blank? || rules.empty?
+
+ quoted_search = ActiveRecord::Base.connection.quote(search.to_s)
+
+ first_expression, *expressions = rules.map do |rule|
+ rule_to_arel(quoted_search, rule)
+ end
+
+ # (SIMILARITY ...) + (SIMILARITY ...)
+ expressions.inject(first_expression) do |expression1, expression2|
+ Arel::Nodes::Addition.new(expression1, expression2)
+ end
+ end
+
+ # (SIMILARITY(COALESCE(column, ''), 'search_string') * CAST(multiplier AS numeric))
+ def self.rule_to_arel(search, rule)
+ Arel::Nodes::Grouping.new(
+ Arel::Nodes::Multiplication.new(
+ similarity_function_call(search, column_expression(rule)),
+ multiplier_expression(rule)
+ )
+ )
+ end
+
+ # COALESCE(column, '')
+ def self.column_expression(rule)
+ Arel::Nodes::NamedFunction.new('COALESCE', [rule.fetch(:column), EMPTY_STRING])
+ end
+
+ # SIMILARITY(COALESCE(column, ''), 'search_string')
+ def self.similarity_function_call(search, column)
+ Arel::Nodes::NamedFunction.new('SIMILARITY', [column, Arel.sql(search)])
+ end
+
+ # CAST(multiplier AS numeric)
+ def self.multiplier_expression(rule)
+ quoted_multiplier = ActiveRecord::Base.connection.quote(rule.fetch(:multiplier, DEFAULT_MULTIPLIER).to_s)
+
+ Arel::Nodes::NamedFunction.new('CAST', [Arel.sql(quoted_multiplier).as('numeric')])
+ end
+
+ private_class_method :rule_to_arel
+ private_class_method :column_expression
+ private_class_method :similarity_function_call
+ private_class_method :multiplier_expression
+ end
+ end
+end
diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb
index bebcba6f42e..a9c86e4e267 100644
--- a/lib/gitlab/database/with_lock_retries.rb
+++ b/lib/gitlab/database/with_lock_retries.rb
@@ -2,7 +2,14 @@
module Gitlab
module Database
+ # This class provides a way to automatically execute code that relies on acquiring a database lock in a way
+ # designed to minimize impact on a busy production database.
+ #
+ # A default timing configuration is provided that makes repeated attempts to acquire the necessary lock, with
+ # varying lock_timeout settings, and also serves to limit the maximum number of attempts.
class WithLockRetries
+ AttemptsExhaustedError = Class.new(StandardError)
+
NULL_LOGGER = Gitlab::JsonLogger.new('/dev/null')
# Each element of the array represents a retry iteration.
@@ -63,7 +70,17 @@ module Gitlab
@log_params = { method: 'with_lock_retries', class: klass.to_s }
end
- def run(&block)
+ # Executes a block of code, retrying it whenever a database lock can't be acquired in time
+ #
+ # When a database lock can't be acquired, ActiveRecord throws ActiveRecord::LockWaitTimeout
+ # exception which we intercept to re-execute the block of code, until it finishes or we reach the
+ # max attempt limit. The default behavior when max attempts have been reached is to make a final attempt with the
+ # lock_timeout disabled, but this can be altered with the raise_on_exhaustion parameter.
+ #
+ # @see DEFAULT_TIMING_CONFIGURATION for the timings used when attempting a retry
+ # @param [Boolean] raise_on_exhaustion whether to raise `AttemptsExhaustedError` when exhausting max attempts
+ # @param [Proc] block of code that will be executed
+ def run(raise_on_exhaustion: false, &block)
raise 'no block given' unless block_given?
@block = block
@@ -85,6 +102,9 @@ module Gitlab
retry
else
reset_db_settings
+
+ raise AttemptsExhaustedError, 'configured attempts to obtain locks are exhausted' if raise_on_exhaustion
+
run_block_without_lock_timeout
end
diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb
index 59a7c4a6660..eb475307f27 100644
--- a/lib/gitlab/devise_failure.rb
+++ b/lib/gitlab/devise_failure.rb
@@ -7,8 +7,6 @@ module Gitlab
# If the request format is not known, send a redirect instead of a 401
# response, since this is the outcome we're most likely to want
def http_auth?
- return super unless Feature.enabled?(:devise_redirect_unknown_formats, default_enabled: true)
-
request_format && super
end
diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb
index 31eeadc45f7..e24150a2330 100644
--- a/lib/gitlab/diff/formatters/base_formatter.rb
+++ b/lib/gitlab/diff/formatters/base_formatter.rb
@@ -10,7 +10,6 @@ module Gitlab
attr_reader :base_sha
attr_reader :start_sha
attr_reader :head_sha
- attr_reader :position_type
def initialize(attrs)
if diff_file = attrs[:diff_file]
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index ccf09b37b9b..0c3b6b72313 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -3,7 +3,6 @@
module Gitlab
module Diff
class HighlightCache
- include Gitlab::Metrics::Methods
include Gitlab::Utils::StrongMemoize
EXPIRATION = 1.week
@@ -12,19 +11,6 @@ module Gitlab
delegate :diffable, to: :@diff_collection
delegate :diff_options, to: :@diff_collection
- define_histogram :gitlab_redis_diff_caching_memory_usage_bytes do
- docstring 'Redis diff caching memory usage by key'
- buckets [100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000]
- end
-
- define_counter :gitlab_redis_diff_caching_hit do
- docstring 'Redis diff caching hits'
- end
-
- define_counter :gitlab_redis_diff_caching_miss do
- docstring 'Redis diff caching misses'
- end
-
def initialize(diff_collection)
@diff_collection = diff_collection
end
@@ -117,7 +103,10 @@ module Gitlab
def record_memory_usage(memory_usage)
if memory_usage
- self.class.gitlab_redis_diff_caching_memory_usage_bytes.observe({}, memory_usage)
+ current_transaction&.observe(:gitlab_redis_diff_caching_memory_usage_bytes, memory_usage) do
+ docstring 'Redis diff caching memory usage by key'
+ buckets [100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000]
+ end
end
end
@@ -163,34 +152,24 @@ module Gitlab
end
def compose_data(json_data)
- if ::Feature.enabled?(:gzip_diff_cache, default_enabled: true)
- # #compress returns ASCII-8BIT, so we need to force the encoding to
- # UTF-8 before caching it in redis, else we risk encoding mismatch
- # errors.
- #
- ActiveSupport::Gzip.compress(json_data).force_encoding("UTF-8")
- else
- json_data
- end
+ # #compress returns ASCII-8BIT, so we need to force the encoding to
+ # UTF-8 before caching it in redis, else we risk encoding mismatch
+ # errors.
+ #
+ ActiveSupport::Gzip.compress(json_data).force_encoding("UTF-8")
rescue Zlib::GzipFile::Error
json_data
end
def extract_data(data)
- # Since when we deploy this code, we'll be dealing with an already
- # populated cache full of data that isn't gzipped, we want to also
- # check to see if the data is gzipped before we attempt to #decompress
- # it, thus we check the first 2 bytes for "\x1F\x8B" to confirm it is
- # a gzipped string. While a non-gzipped string will raise a
- # Zlib::GzipFile::Error, which we're rescuing, we don't want to count
- # on rescue for control flow. This check can be removed in the release
- # after this change is released.
+ # Since we could be dealing with an already populated cache full of data
+ # that isn't gzipped, we want to also check to see if the data is
+ # gzipped before we attempt to #decompress it, thus we check the first
+ # 2 bytes for "\x1F\x8B" to confirm it is a gzipped string. While a
+ # non-gzipped string will raise a Zlib::GzipFile::Error, which we're
+ # rescuing, we don't want to count on rescue for control flow.
#
- if ::Feature.enabled?(:gzip_diff_cache, default_enabled: true) && data[0..1] == "\x1F\x8B"
- ActiveSupport::Gzip.decompress(data)
- else
- data
- end
+ data[0..1] == "\x1F\x8B" ? ActiveSupport::Gzip.decompress(data) : data
rescue Zlib::GzipFile::Error
data
end
@@ -206,6 +185,10 @@ module Gitlab
#
@diff_collection.raw_diff_files
end
+
+ def current_transaction
+ ::Gitlab::Metrics::Transaction.current
+ end
end
end
end
diff --git a/lib/gitlab/diff/stats_cache.rb b/lib/gitlab/diff/stats_cache.rb
index f38fb21d497..eb0ef4200dc 100644
--- a/lib/gitlab/diff/stats_cache.rb
+++ b/lib/gitlab/diff/stats_cache.rb
@@ -3,11 +3,11 @@
module Gitlab
module Diff
class StatsCache
- include Gitlab::Metrics::Methods
include Gitlab::Utils::StrongMemoize
EXPIRATION = 1.week
- VERSION = 1
+ # The DiffStats#as_json representation is tied to the Gitaly protobuf version
+ VERSION = Gem.loaded_specs['gitaly'].version.to_s
def initialize(cachable_key:)
@cachable_key = cachable_key
@@ -29,7 +29,8 @@ module Gitlab
return if cache.exist?(key)
return unless stats
- cache.write(key, stats.as_json, expires_in: EXPIRATION)
+ cache.write(key, stats.map(&:to_h).as_json, expires_in: EXPIRATION)
+ clear_memoization(:cached_values)
end
def clear
diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb
index 2da1b8915e4..6aff8f909f3 100644
--- a/lib/gitlab/exception_log_formatter.rb
+++ b/lib/gitlab/exception_log_formatter.rb
@@ -23,7 +23,7 @@ module Gitlab
end
if exception.backtrace
- payload['exception.backtrace'] = Gitlab::BacktraceCleaner.clean_backtrace(exception.backtrace)
+ payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace)
end
end
end
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index 1d2c1c69423..b602393b59e 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -102,7 +102,7 @@ module Gitlab
Gitlab::Redis::SharedState.with do |redis|
ttl = redis.ttl(@redis_shared_state_key)
- ttl if ttl.positive?
+ ttl if ttl > 0
end
end
diff --git a/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb b/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb
index 8213c9bc042..52035220a71 100644
--- a/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb
+++ b/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb
@@ -39,7 +39,7 @@ module Gitlab
end
def first_attempt?
- attempts.zero?
+ attempts == 0
end
def sleep_sec
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
index d3df9be0d63..9908369426a 100644
--- a/lib/gitlab/experimentation.rb
+++ b/lib/gitlab/experimentation.rb
@@ -53,9 +53,21 @@ module Gitlab
},
new_create_project_ui: {
tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi'
+ },
+ terms_opt_in: {
+ tracking_category: 'Growth::Acquisition::Experiment::TermsOptIn'
+ },
+ contact_sales_btn_in_app: {
+ tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp'
+ },
+ customize_homepage: {
+ tracking_category: 'Growth::Expansion::Experiment::CustomizeHomepage'
}
}.freeze
+ GROUP_CONTROL = :control
+ GROUP_EXPERIMENTAL = :experimental
+
# Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent.
# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method
# to controllers and views. It returns true when the experiment is enabled and the user is selected as part
@@ -100,6 +112,12 @@ module Gitlab
end
end
+ def record_experiment_user(experiment_key)
+ return unless Experimentation.enabled?(experiment_key) && current_user
+
+ ::Experiment.add_user(experiment_key, tracking_group(experiment_key), current_user)
+ end
+
private
def dnt_enabled?
@@ -126,7 +144,7 @@ module Gitlab
{
category: tracking_category(experiment_key),
action: action,
- property: tracking_group(experiment_key),
+ property: "#{tracking_group(experiment_key)}_group",
label: experimentation_subject_id,
value: value
}.compact
@@ -139,7 +157,7 @@ module Gitlab
def tracking_group(experiment_key)
return unless Experimentation.enabled?(experiment_key)
- experiment_enabled?(experiment_key) ? 'experimental_group' : 'control_group'
+ experiment_enabled?(experiment_key) ? GROUP_EXPERIMENTAL : GROUP_CONTROL
end
def forced_enabled?(experiment_key)
@@ -167,7 +185,7 @@ module Gitlab
Experiment = Struct.new(:key, :environment, :tracking_category, keyword_init: true) do
def enabled?
- experiment_percentage.positive?
+ experiment_percentage > 0
end
def enabled_for_environment?
diff --git a/lib/gitlab/external_authorization/client.rb b/lib/gitlab/external_authorization/client.rb
index 7985e6dcf7b..fc859304eab 100644
--- a/lib/gitlab/external_authorization/client.rb
+++ b/lib/gitlab/external_authorization/client.rb
@@ -17,23 +17,28 @@ module Gitlab
end
def request_access
- response = Excon.post(
+ response = Gitlab::HTTP.post(
service_url,
post_params
)
::Gitlab::ExternalAuthorization::Response.new(response)
- rescue Excon::Error => e
+ rescue *Gitlab::HTTP::HTTP_ERRORS => e
raise ::Gitlab::ExternalAuthorization::RequestFailed.new(e)
end
private
+ def allow_local_requests?
+ Gitlab::CurrentSettings.allow_local_requests_from_system_hooks?
+ end
+
def post_params
params = { headers: REQUEST_HEADERS,
body: body.to_json,
connect_timeout: timeout,
read_timeout: timeout,
- write_timeout: timeout }
+ write_timeout: timeout,
+ allow_local_requests: allow_local_requests? }
if has_tls?
params[:client_cert_data] = client_cert
diff --git a/lib/gitlab/external_authorization/response.rb b/lib/gitlab/external_authorization/response.rb
index 04f9688fad0..8656065303d 100644
--- a/lib/gitlab/external_authorization/response.rb
+++ b/lib/gitlab/external_authorization/response.rb
@@ -5,16 +5,16 @@ module Gitlab
class Response
include ::Gitlab::Utils::StrongMemoize
- def initialize(excon_response)
- @excon_response = excon_response
+ def initialize(response)
+ @response = response
end
def valid?
- @excon_response && [200, 401, 403].include?(@excon_response.status)
+ @response && [200, 401, 403].include?(@response.code)
end
def successful?
- valid? && @excon_response.status == 200
+ valid? && @response.code == 200
end
def reason
@@ -28,7 +28,7 @@ module Gitlab
end
def parse_response!
- Gitlab::Json.parse(@excon_response.body)
+ Gitlab::Json.parse(@response.body)
rescue JSON::JSONError
# The JSON response is optional, so don't fail when it's missing
nil
diff --git a/lib/gitlab/file_hook.rb b/lib/gitlab/file_hook.rb
index f23ef2921d7..55eba2858fb 100644
--- a/lib/gitlab/file_hook.rb
+++ b/lib/gitlab/file_hook.rb
@@ -27,7 +27,7 @@ module Gitlab
end
exit_status = result.status&.exitstatus
- [exit_status.zero?, result.stderr]
+ [exit_status == 0, result.stderr]
rescue => e
[false, e.message]
end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 1b49d356d29..5d91eb605e8 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -5,7 +5,6 @@ module Gitlab
class Blob
include Gitlab::BlobHelper
include Gitlab::EncodingHelper
- include Gitlab::Metrics::Methods
extend Gitlab::Git::WrapsGitalyErrors
# This number is the maximum amount of data that we want to display to
@@ -25,19 +24,24 @@ module Gitlab
LFS_POINTER_MIN_SIZE = 120.bytes
LFS_POINTER_MAX_SIZE = 200.bytes
- attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary
+ attr_accessor :size, :mode, :id, :commit_id, :loaded_size, :binary
+ attr_writer :name, :path, :data
- define_counter :gitlab_blob_truncated_true do
- docstring 'blob.truncated? == true'
+ def self.gitlab_blob_truncated_true
+ @gitlab_blob_truncated_true ||= ::Gitlab::Metrics.counter(:gitlab_blob_truncated_true, 'blob.truncated? == true')
end
- define_counter :gitlab_blob_truncated_false do
- docstring 'blob.truncated? == false'
+ def self.gitlab_blob_truncated_false
+ @gitlab_blob_truncated_false ||= ::Gitlab::Metrics.counter(:gitlab_blob_truncated_false, 'blob.truncated? == false')
end
- define_histogram :gitlab_blob_size do
- docstring 'Gitlab::Git::Blob size'
- buckets [1_000, 5_000, 10_000, 50_000, 100_000, 500_000, 1_000_000]
+ def self.gitlab_blob_size
+ @gitlab_blob_size ||= ::Gitlab::Metrics.histogram(
+ :gitlab_blob_size,
+ 'Gitlab::Git::Blob size',
+ {},
+ [1_000, 5_000, 10_000, 50_000, 100_000, 500_000, 1_000_000]
+ )
end
class << self
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 8db73ecc480..0bc7ecccf5e 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -261,7 +261,7 @@ module Gitlab
end
def has_zero_stats?
- stats.total.zero?
+ stats.total == 0
rescue
true
end
@@ -423,7 +423,7 @@ module Gitlab
end
def message_from_gitaly_body
- return @raw_commit.subject.dup if @raw_commit.body_size.zero?
+ return @raw_commit.subject.dup if @raw_commit.body_size == 0
return @raw_commit.body.dup if full_body_fetched_from_gitaly?
if @raw_commit.body_size > MAX_COMMIT_MESSAGE_DISPLAY_SIZE
diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb
index ef9b1bf5224..7a6f27179f0 100644
--- a/lib/gitlab/git/pre_receive_error.rb
+++ b/lib/gitlab/git/pre_receive_error.rb
@@ -16,8 +16,16 @@ module Gitlab
SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/.freeze
- def initialize(message = '')
- super(sanitize(message))
+ attr_reader :raw_message
+
+ def initialize(message = '', user_message = '')
+ @raw_message = message
+
+ if user_message.present?
+ super(sanitize(user_message))
+ else
+ super(sanitize(message))
+ end
end
private
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index ea7a6e84195..596b4e9f692 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -44,7 +44,7 @@ module Gitlab
# Relative path of repo
attr_reader :relative_path
- attr_reader :storage, :gl_repository, :relative_path, :gl_project_path
+ attr_reader :storage, :gl_repository, :gl_project_path
# This remote name has to be stable for all types of repositories that
# can join an object pool. If it's structure ever changes, a migration
@@ -598,14 +598,15 @@ module Gitlab
end
end
- def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+ def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false)
args = {
user: user,
commit: commit,
branch_name: branch_name,
message: message,
start_branch_name: start_branch_name,
- start_repository: start_repository
+ start_repository: start_repository,
+ dry_run: dry_run
}
wrapped_gitaly_errors do
@@ -613,14 +614,15 @@ module Gitlab
end
end
- def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+ def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false)
args = {
user: user,
commit: commit,
branch_name: branch_name,
message: message,
start_branch_name: start_branch_name,
- start_repository: start_repository
+ start_repository: start_repository,
+ dry_run: dry_run
}
wrapped_gitaly_errors do
@@ -813,7 +815,7 @@ module Gitlab
def fsck
msg, status = gitaly_repository_client.fsck
- raise GitError.new("Could not fsck repository: #{msg}") unless status.zero?
+ raise GitError.new("Could not fsck repository: #{msg}") unless status == 0
end
def create_from_bundle(bundle_path)
diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb
index 5c73c0c66a9..dc869ff5279 100644
--- a/lib/gitlab/git/rugged_impl/blob.rb
+++ b/lib/gitlab/git/rugged_impl/blob.rb
@@ -48,7 +48,7 @@ module Gitlab
name: blob_entry[:name],
size: blob.size,
# Rugged::Blob#content is expensive; don't call it if we don't have to.
- data: limit.zero? ? '' : blob.content(limit),
+ data: limit == 0 ? '' : blob.content(limit),
mode: blob_entry[:filemode].to_s(8),
path: path,
commit_id: sha,
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index 7e072c5db50..ed02f2e92ec 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -6,8 +6,8 @@ module Gitlab
include Gitlab::EncodingHelper
extend Gitlab::Git::WrapsGitalyErrors
- attr_accessor :id, :root_id, :name, :path, :flat_path, :type,
- :mode, :commit_id, :submodule_url
+ attr_accessor :id, :root_id, :type, :mode, :commit_id, :submodule_url
+ attr_writer :name, :path, :flat_path
class << self
# Get list of tree objects
diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb
index f6cac398548..a1f3d64ccde 100644
--- a/lib/gitlab/git/wiki_page.rb
+++ b/lib/gitlab/git/wiki_page.rb
@@ -3,7 +3,7 @@
module Gitlab
module Git
class WikiPage
- attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :text_data, :historical, :formatted_data
+ attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :historical, :formatted_data
# This class abstracts away Gitlab::GitalyClient::WikiPage
def initialize(gitaly_page, version)
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 37e3da984d6..f3b53a2ba0b 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -23,7 +23,6 @@ module Gitlab
deploy_key_upload: 'This deploy key does not have write access to this project.',
no_repo: 'A repository for this project does not exist yet.',
project_not_found: 'The project you were looking for could not be found.',
- namespace_not_found: 'The namespace you were looking for could not be found.',
command_not_allowed: "The command you're trying to execute is not allowed.",
upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.',
receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.',
@@ -43,38 +42,42 @@ module Gitlab
PUSH_COMMANDS = %w{git-receive-pack}.freeze
ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS
- attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :repository_path, :redirected_path, :auth_result_type, :changes, :logger
+ attr_reader :actor, :protocol, :authentication_abilities,
+ :namespace_path, :redirected_path, :auth_result_type,
+ :cmd, :changes
+ attr_accessor :container
- alias_method :container, :project
-
- def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, repository_path: nil, redirected_path: nil, auth_result_type: nil)
- @actor = actor
- @project = project
- @protocol = protocol
+ def initialize(actor, container, protocol, authentication_abilities:, namespace_path: nil, repository_path: nil, redirected_path: nil, auth_result_type: nil)
+ @actor = actor
+ @container = container
+ @protocol = protocol
@authentication_abilities = Array(authentication_abilities)
- @namespace_path = namespace_path || project&.namespace&.full_path
- @repository_path = repository_path || project&.path
+ @namespace_path = namespace_path
+ @repository_path = repository_path
@redirected_path = redirected_path
@auth_result_type = auth_result_type
end
+ def repository_path
+ @repository_path ||= project&.path
+ end
+
def check(cmd, changes)
- @logger = Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER)
@changes = changes
+ @cmd = cmd
check_protocol!
check_valid_actor!
check_active_user!
- check_authentication_abilities!(cmd)
- check_command_disabled!(cmd)
- check_command_existence!(cmd)
+ check_authentication_abilities!
+ check_command_disabled!
+ check_command_existence!
- custom_action = check_custom_action(cmd)
+ custom_action = check_custom_action
return custom_action if custom_action
- check_db_accessibility!(cmd)
- check_namespace!
- check_project!(cmd)
+ check_db_accessibility!
+ check_container!
check_repository_existence!
case cmd
@@ -87,12 +90,27 @@ module Gitlab
success_result
end
+ def logger
+ @logger ||= Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER)
+ end
+
def guest_can_download_code?
- Guest.can?(:download_code, project)
+ Guest.can?(download_ability, container)
end
def user_can_download_code?
- authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code)
+ authentication_abilities.include?(:download_code) &&
+ user_access.can_do_action?(download_ability)
+ end
+
+ # @return [Symbol] the name of a Declarative Policy ability to check
+ def download_ability
+ raise NotImplementedError
+ end
+
+ # @return [Symbol] the name of a Declarative Policy ability to check
+ def push_ability
+ raise NotImplementedError
end
def build_can_download_code?
@@ -111,13 +129,17 @@ module Gitlab
private
- def check_project!(_cmd)
+ def check_container!
+ check_project! if project?
+ end
+
+ def check_project!
check_project_accessibility!
add_project_moved_message!
end
- def check_custom_action(cmd)
- nil
+ def check_custom_action
+ # no-op: Overridden in EE
end
def check_for_console_messages
@@ -152,12 +174,6 @@ module Gitlab
end
end
- def check_namespace!
- return if namespace_path.present?
-
- raise NotFoundError, ERROR_MESSAGES[:namespace_not_found]
- end
-
def check_active_user!
return unless user
@@ -167,25 +183,29 @@ module Gitlab
end
end
- def check_authentication_abilities!(cmd)
+ def check_authentication_abilities!
case cmd
when *DOWNLOAD_COMMANDS
unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code)
- raise ForbiddenError, ERROR_MESSAGES[:auth_download]
+ raise ForbiddenError, error_message(:auth_download)
end
when *PUSH_COMMANDS
unless authentication_abilities.include?(:push_code)
- raise ForbiddenError, ERROR_MESSAGES[:auth_upload]
+ raise ForbiddenError, error_message(:auth_upload)
end
end
end
def check_project_accessibility!
if project.blank? || !can_read_project?
- raise NotFoundError, ERROR_MESSAGES[:project_not_found]
+ raise NotFoundError, not_found_message
end
end
+ def not_found_message
+ error_message(:project_not_found)
+ end
+
def add_project_moved_message!
return if redirected_path.nil?
@@ -194,34 +214,34 @@ module Gitlab
project_moved.add_message
end
- def check_command_disabled!(cmd)
- if upload_pack?(cmd)
+ def check_command_disabled!
+ if upload_pack?
check_upload_pack_disabled!
- elsif receive_pack?(cmd)
+ elsif receive_pack?
check_receive_pack_disabled!
end
end
def check_upload_pack_disabled!
if http? && upload_pack_disabled_over_http?
- raise ForbiddenError, ERROR_MESSAGES[:upload_pack_disabled_over_http]
+ raise ForbiddenError, error_message(:upload_pack_disabled_over_http)
end
end
def check_receive_pack_disabled!
if http? && receive_pack_disabled_over_http?
- raise ForbiddenError, ERROR_MESSAGES[:receive_pack_disabled_over_http]
+ raise ForbiddenError, error_message(:receive_pack_disabled_over_http)
end
end
- def check_command_existence!(cmd)
+ def check_command_existence!
unless ALL_COMMANDS.include?(cmd)
- raise ForbiddenError, ERROR_MESSAGES[:command_not_allowed]
+ raise ForbiddenError, error_message(:command_not_allowed)
end
end
- def check_db_accessibility!(cmd)
- return unless receive_pack?(cmd)
+ def check_db_accessibility!
+ return unless receive_pack?
if Gitlab::Database.read_only?
raise ForbiddenError, push_to_read_only_message
@@ -229,9 +249,11 @@ module Gitlab
end
def check_repository_existence!
- unless repository.exists?
- raise NotFoundError, ERROR_MESSAGES[:no_repo]
- end
+ raise NotFoundError, no_repo_message unless repository.exists?
+ end
+
+ def no_repo_message
+ error_message(:no_repo)
end
def check_download_access!
@@ -242,44 +264,62 @@ module Gitlab
guest_can_download_code?
unless passed
- raise ForbiddenError, ERROR_MESSAGES[:download]
+ raise ForbiddenError, download_forbidden_message
end
end
+ def download_forbidden_message
+ error_message(:download)
+ end
+
+ # We assume that all git-access classes are in project context by default.
+ # Override this method to be more specific.
+ def project?
+ true
+ end
+
+ def project
+ container if container.is_a?(::Project)
+ end
+
def check_push_access!
- if project.repository_read_only?
- raise ForbiddenError, ERROR_MESSAGES[:read_only]
+ if container.repository_read_only?
+ raise ForbiddenError, error_message(:read_only)
end
if deploy_key?
unless deploy_key.can_push_to?(project)
- raise ForbiddenError, ERROR_MESSAGES[:deploy_key_upload]
+ raise ForbiddenError, error_message(:deploy_key_upload)
end
elsif user
# User access is verified in check_change_access!
else
- raise ForbiddenError, ERROR_MESSAGES[:upload]
+ raise ForbiddenError, error_message(:upload)
end
check_change_access!
end
+ def user_can_push?
+ user_access.can_do_action?(push_ability)
+ end
+
def check_change_access!
# Deploy keys with write access can push anything
return if deploy_key?
if changes == ANY
- can_push = user_access.can_do_action?(:push_code) ||
- project.any_branch_allows_collaboration?(user_access.user)
+ can_push = user_can_push? ||
+ project&.any_branch_allows_collaboration?(user_access.user)
unless can_push
- raise ForbiddenError, ERROR_MESSAGES[:push_code]
+ raise ForbiddenError, error_message(:push_code)
end
else
# If there are worktrees with a HEAD pointing to a non-existent object,
# calls to `git rev-list --all` will fail in git 2.15+. This should also
# clear stale lock files.
- project.repository.clean_stale_repository_files
+ project.repository.clean_stale_repository_files if project.present?
# Iterate over all changes to find if user allowed all of them to be applied
changes_list.each.with_index do |change, index|
@@ -293,16 +333,14 @@ module Gitlab
end
def check_single_change_access(change, skip_lfs_integrity_check: false)
- change_access = Checks::ChangeAccess.new(
+ Checks::ChangeAccess.new(
change,
user_access: user_access,
project: project,
skip_lfs_integrity_check: skip_lfs_integrity_check,
protocol: protocol,
logger: logger
- )
-
- change_access.exec
+ ).validate!
rescue Checks::TimedLogger::TimeoutError
raise TimeoutError, logger.full_message
end
@@ -347,12 +385,12 @@ module Gitlab
protocol == 'http'
end
- def upload_pack?(command)
- command == 'git-upload-pack'
+ def upload_pack?
+ cmd == 'git-upload-pack'
end
- def receive_pack?(command)
- command == 'git-receive-pack'
+ def receive_pack?
+ cmd == 'git-receive-pack'
end
def upload_pack_disabled_over_http?
@@ -365,6 +403,16 @@ module Gitlab
protected
+ def error_message(key)
+ self.class.ancestors.each do |cls|
+ return cls.const_get('ERROR_MESSAGES', false).fetch(key)
+ rescue NameError, KeyError
+ next
+ end
+
+ raise ArgumentError, "No error message defined for #{key}"
+ end
+
def success_result
::Gitlab::GitAccessResult::Success.new(console_messages: check_for_console_messages)
end
@@ -374,9 +422,7 @@ module Gitlab
end
def user
- return @user if defined?(@user)
-
- @user =
+ strong_memoize(:user) do
case actor
when User
actor
@@ -387,20 +433,21 @@ module Gitlab
when :ci
nil
end
+ end
end
def user_access
@user_access ||= if ci?
CiAccess.new
elsif user && request_from_ci_build?
- BuildAccess.new(user, project: project)
+ BuildAccess.new(user, container: container)
else
- UserAccess.new(user, project: project)
+ UserAccess.new(user, container: container)
end
end
def push_to_read_only_message
- ERROR_MESSAGES[:cannot_push_to_read_only]
+ error_message(:cannot_push_to_read_only)
end
def repository
diff --git a/lib/gitlab/git_access_design.rb b/lib/gitlab/git_access_design.rb
index 36604bd0b3b..6bea9fe53b3 100644
--- a/lib/gitlab/git_access_design.rb
+++ b/lib/gitlab/git_access_design.rb
@@ -2,6 +2,8 @@
module Gitlab
class GitAccessDesign < GitAccess
+ extend ::Gitlab::Utils::Override
+
def check(_cmd, _changes)
check_protocol!
check_can_create_design!
@@ -9,6 +11,11 @@ module Gitlab
success_result
end
+ override :push_ability
+ def push_ability
+ :create_design
+ end
+
private
def check_protocol!
@@ -18,7 +25,7 @@ module Gitlab
end
def check_can_create_design!
- unless user&.can?(:create_design, project)
+ unless user_can_push?
raise ::Gitlab::GitAccess::ForbiddenError, "You are not allowed to manage designs of this project"
end
end
diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb
index c79a61c263e..cdefcc84f7d 100644
--- a/lib/gitlab/git_access_project.rb
+++ b/lib/gitlab/git_access_project.rb
@@ -6,21 +6,41 @@ module Gitlab
CreationError = Class.new(StandardError)
+ ERROR_MESSAGES = {
+ namespace_not_found: 'The namespace you were looking for could not be found.'
+ }.freeze
+
+ override :download_ability
+ def download_ability
+ :download_code
+ end
+
+ override :push_ability
+ def push_ability
+ :push_code
+ end
+
private
- override :check_project!
- def check_project!(cmd)
- ensure_project_on_push!(cmd)
+ override :check_container!
+ def check_container!
+ check_namespace!
+ ensure_project_on_push!
super
end
- def ensure_project_on_push!(cmd)
- return if project || deploy_key?
- return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code)
+ def check_namespace!
+ raise NotFoundError, ERROR_MESSAGES[:namespace_not_found] unless namespace_path.present?
+ end
- namespace = Namespace.find_by_full_path(namespace_path)
+ def namespace
+ @namespace ||= Namespace.find_by_full_path(namespace_path)
+ end
+ def ensure_project_on_push!
+ return if project || deploy_key?
+ return unless receive_pack? && changes == ANY && authentication_abilities.include?(:push_code)
return unless user&.can?(:create_projects, namespace)
project_params = {
@@ -35,8 +55,8 @@ module Gitlab
raise CreationError, "Could not create project: #{project.errors.full_messages.join(', ')}"
end
- @project = project
- user_access.project = @project
+ self.container = project
+ user_access.container = project
Checks::ProjectCreated.new(repository, user, protocol).add_message
end
diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb
index 3de6c9ee30a..f2b4e930707 100644
--- a/lib/gitlab/git_access_snippet.rb
+++ b/lib/gitlab/git_access_snippet.rb
@@ -9,50 +9,68 @@ module Gitlab
read_snippet: 'You are not allowed to read this snippet.',
update_snippet: 'You are not allowed to update this snippet.',
snippet_not_found: 'The snippet you were looking for could not be found.',
- repository_not_found: 'The snippet repository you were looking for could not be found.'
+ no_repo: 'The snippet repository you were looking for could not be found.'
}.freeze
- attr_reader :snippet
-
- alias_method :container, :snippet
+ alias_method :snippet, :container
def initialize(actor, snippet, protocol, **kwargs)
- @snippet = snippet
-
- super(actor, snippet&.project, protocol, **kwargs)
+ super(actor, snippet, protocol, **kwargs)
@auth_result_type = nil
@authentication_abilities &= [:download_code, :push_code]
end
+ override :check
def check(cmd, changes)
- # TODO: Investigate if expanding actor/authentication types are needed.
- # https://gitlab.com/gitlab-org/gitlab/issues/202190
- if actor && !actor.is_a?(User) && !actor.instance_of?(Key)
- raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism]
- end
-
check_snippet_accessibility!
super
end
+ override :download_ability
+ def download_ability
+ :read_snippet
+ end
+
+ override :push_ability
+ def push_ability
+ :update_snippet
+ end
+
private
- override :check_namespace!
- def check_namespace!
- return unless snippet.is_a?(ProjectSnippet)
+ # TODO: Implement EE/Geo https://gitlab.com/gitlab-org/gitlab/issues/205629
+ override :check_custom_action
+ def check_custom_action
+ # snippets never return custom actions, such as geo replication.
+ end
- super
+ override :project?
+ def project?
+ project_snippet?
+ end
+
+ override :project
+ def project
+ snippet&.project
end
- override :check_project!
- def check_project!(cmd)
- return unless snippet.is_a?(ProjectSnippet)
+ override :check_valid_actor!
+ def check_valid_actor!
+ # TODO: Investigate if expanding actor/authentication types are needed.
+ # https://gitlab.com/gitlab-org/gitlab/issues/202190
+ if actor && !actor.is_a?(User) && !actor.instance_of?(Key)
+ raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism]
+ end
super
end
+ def project_snippet?
+ snippet.is_a?(ProjectSnippet)
+ end
+
override :check_push_access!
def check_push_access!
raise ForbiddenError, ERROR_MESSAGES[:update_snippet] unless user
@@ -82,19 +100,9 @@ module Gitlab
end
end
- override :guest_can_download_code?
- def guest_can_download_code?
- Guest.can?(:read_snippet, snippet)
- end
-
- override :user_can_download_code?
- def user_can_download_code?
- authentication_abilities.include?(:download_code) && user_access.can_do_action?(:read_snippet)
- end
-
override :check_change_access!
def check_change_access!
- unless user_access.can_do_action?(:update_snippet)
+ unless user_can_push?
raise ForbiddenError, ERROR_MESSAGES[:update_snippet]
end
@@ -109,31 +117,19 @@ module Gitlab
check_push_size!
end
- def check_single_change_access(change)
+ override :check_single_change_access
+ def check_single_change_access(change, _skip_lfs_integrity_check: false)
Checks::SnippetCheck.new(change, logger: logger).validate!
Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit(user), logger: logger).validate!
rescue Checks::TimedLogger::TimeoutError
raise TimeoutError, logger.full_message
end
- override :check_repository_existence!
- def check_repository_existence!
- unless repository.exists?
- raise NotFoundError, ERROR_MESSAGES[:repository_not_found]
- end
- end
-
override :user_access
def user_access
@user_access ||= UserAccessSnippet.new(user, snippet: snippet)
end
- # TODO: Implement EE/Geo https://gitlab.com/gitlab-org/gitlab/issues/205629
- override :check_custom_action
- def check_custom_action(cmd)
- nil
- end
-
override :check_size_limit?
def check_size_limit?
return false if user&.migration_bot?
diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb
index aad46937c32..a941282e713 100644
--- a/lib/gitlab/git_access_wiki.rb
+++ b/lib/gitlab/git_access_wiki.rb
@@ -2,41 +2,50 @@
module Gitlab
class GitAccessWiki < GitAccess
- prepend_if_ee('EE::Gitlab::GitAccessWiki') # rubocop: disable Cop/InjectEnterpriseEditionModule
+ extend ::Gitlab::Utils::Override
ERROR_MESSAGES = {
- read_only: "You can't push code to a read-only GitLab instance.",
+ download: 'You are not allowed to download files from this wiki.',
+ not_found: 'The wiki you were looking for could not be found.',
+ no_repo: 'A repository for this wiki does not exist yet.',
+ read_only: "You can't push code to a read-only GitLab instance.",
write_to_wiki: "You are not allowed to write to this project's wiki."
}.freeze
- def guest_can_download_code?
- Guest.can?(:download_wiki_code, project)
+ override :download_ability
+ def download_ability
+ :download_wiki_code
end
- def user_can_download_code?
- authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code)
+ override :push_ability
+ def push_ability
+ :create_wiki
end
+ override :check_change_access!
def check_change_access!
- unless user_access.can_do_action?(:create_wiki)
- raise ForbiddenError, ERROR_MESSAGES[:write_to_wiki]
- end
-
- if Gitlab::Database.read_only?
- raise ForbiddenError, push_to_read_only_message
- end
+ raise ForbiddenError, write_to_wiki_message unless user_can_push?
true
end
def push_to_read_only_message
- ERROR_MESSAGES[:read_only]
+ error_message(:read_only)
end
- private
+ def write_to_wiki_message
+ error_message(:write_to_wiki)
+ end
+ def not_found_message
+ error_message(:not_found)
+ end
+
+ override :repository
def repository
- project.wiki.repository
+ container.wiki.repository
end
end
end
+
+Gitlab::GitAccessWiki.prepend_if_ee('EE::Gitlab::GitAccessWiki')
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index b284aadc107..131c00db612 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -8,8 +8,6 @@ require 'grpc/health/v1/health_services_pb'
module Gitlab
module GitalyClient
- include Gitlab::Metrics::Methods
-
class TooManyInvocationsError < StandardError
attr_reader :call_site, :invocation_count, :max_call_stack
@@ -191,11 +189,6 @@ module Gitlab
Gitlab::SafeRequestStore[:gitaly_query_time] += duration
end
- def self.current_transaction_labels
- Gitlab::Metrics::Transaction.current&.labels || {}
- end
- private_class_method :current_transaction_labels
-
# For some time related tasks we can't rely on `Time.now` since it will be
# affected by Timecop in some tests, and the clock of some gitaly-related
# components (grpc's c-core and gitaly server) use system time instead of
@@ -483,7 +476,7 @@ module Gitlab
return unless stack_counter
max = max_call_count
- return if max.zero?
+ return if max == 0
stack_counter.select { |_, v| v == max }.keys
end
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 87505418ae9..513063c60d2 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -179,7 +179,7 @@ module Gitlab
)
if response.pre_receive_error.present?
- raise Gitlab::Git::PreReceiveError.new("GL-HOOK-ERR: pre-receive hook failed.")
+ raise Gitlab::Git::PreReceiveError.new(response.pre_receive_error, "GL-HOOK-ERR: pre-receive hook failed.")
end
Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update)
@@ -187,24 +187,26 @@ module Gitlab
raise Gitlab::Git::CommitError, e
end
- def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+ def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false)
call_cherry_pick_or_revert(:cherry_pick,
user: user,
commit: commit,
branch_name: branch_name,
message: message,
start_branch_name: start_branch_name,
- start_repository: start_repository)
+ start_repository: start_repository,
+ dry_run: dry_run)
end
- def user_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+ def user_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run: false)
call_cherry_pick_or_revert(:revert,
user: user,
commit: commit,
branch_name: branch_name,
message: message,
start_branch_name: start_branch_name,
- start_repository: start_repository)
+ start_repository: start_repository,
+ dry_run: dry_run)
end
def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: [])
@@ -390,7 +392,7 @@ module Gitlab
response
end
- def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:)
+ def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:, dry_run:)
request_class = "Gitaly::User#{rpc.to_s.camelcase}Request".constantize
request = request_class.new(
@@ -400,7 +402,8 @@ module Gitlab
branch_name: encode_binary(branch_name),
message: encode_binary(message),
start_branch_name: encode_binary(start_branch_name.to_s),
- start_repository: start_repository.gitaly_repository
+ start_repository: start_repository.gitaly_repository,
+ dry_run: dry_run
)
response = GitalyClient.call(
diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb
index 9da986ae921..34d1231b9a5 100644
--- a/lib/gitlab/github_import/user_finder.rb
+++ b/lib/gitlab/github_import/user_finder.rb
@@ -161,7 +161,7 @@ module Gitlab
# The cache key may be empty to indicate a previously looked up user for
# which we couldn't find an ID.
- [exists, number.positive? ? number : nil]
+ [exists, number > 0 ? number : nil]
end
end
end
diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb
index 2b482ee3d2d..2c0038b61e2 100644
--- a/lib/gitlab/gl_repository/repo_type.rb
+++ b/lib/gitlab/gl_repository/repo_type.rb
@@ -37,19 +37,19 @@ module Gitlab
end
def wiki?
- self == WIKI
+ name == :wiki
end
def project?
- self == PROJECT
+ name == :project
end
def snippet?
- self == SNIPPET
+ name == :snippet
end
def design?
- self == DESIGN
+ name == :design
end
def path_suffix
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index fbbfed7279d..dfba68ce899 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -43,10 +43,11 @@ module Gitlab
# Initialize gon.features with any flags that should be
# made globally available to the frontend
push_frontend_feature_flag(:snippets_vue, default_enabled: true)
- push_frontend_feature_flag(:monaco_blobs, default_enabled: false)
+ push_frontend_feature_flag(:monaco_blobs, default_enabled: true)
push_frontend_feature_flag(:monaco_ci, default_enabled: false)
push_frontend_feature_flag(:snippets_edit_vue, default_enabled: false)
push_frontend_feature_flag(:webperf_experiment, default_enabled: false)
+ push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false)
end
# Exposes the state of a feature flag to the frontend code.
diff --git a/lib/gitlab/hashed_path.rb b/lib/gitlab/hashed_path.rb
new file mode 100644
index 00000000000..2510c511e28
--- /dev/null
+++ b/lib/gitlab/hashed_path.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Class that returns the disk path for a model using hashed storage
+
+module Gitlab
+ class HashedPath
+ def initialize(*paths, root_hash:)
+ @paths = paths
+ @root_hash = root_hash
+ end
+
+ def to_s
+ File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, @paths.map(&:to_s))
+ end
+
+ alias_method :to_str, :to_s
+
+ private
+
+ def disk_hash
+ @disk_hash ||= Digest::SHA2.hexdigest(@root_hash.to_s)
+ end
+ end
+end
diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb
index 6a8e16f5a85..b72d08549fe 100644
--- a/lib/gitlab/hashed_storage/migrator.rb
+++ b/lib/gitlab/hashed_storage/migrator.rb
@@ -101,7 +101,7 @@ module Gitlab
def any_non_empty_queue?(*workers)
workers.any? do |worker|
- !Sidekiq::Queue.new(worker.queue).size.zero?
+ Sidekiq::Queue.new(worker.queue).size != 0 # rubocop:disable Style/ZeroLengthPredicate
end
end
diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb
index 911b71c3734..559e1828a70 100644
--- a/lib/gitlab/http.rb
+++ b/lib/gitlab/http.rb
@@ -9,32 +9,51 @@ module Gitlab
BlockedUrlError = Class.new(StandardError)
RedirectionTooDeep = Class.new(StandardError)
- HTTP_ERRORS = [
+ HTTP_TIMEOUT_ERRORS = [
+ Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout
+ ].freeze
+ HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [
SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError,
Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH,
- Net::OpenTimeout, Net::ReadTimeout, Gitlab::HTTP::BlockedUrlError,
- Gitlab::HTTP::RedirectionTooDeep
+ Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep
].freeze
+ DEFAULT_TIMEOUT_OPTIONS = {
+ open_timeout: 10,
+ read_timeout: 20,
+ write_timeout: 30
+ }.freeze
+
include HTTParty # rubocop:disable Gitlab/HTTParty
+ class << self
+ alias_method :httparty_perform_request, :perform_request
+ end
+
connection_adapter HTTPConnectionAdapter
def self.perform_request(http_method, path, options, &block)
- super
+ log_info = options.delete(:extra_log_info)
+ options_with_timeouts =
+ if !options.has_key?(:timeout) && Feature.enabled?(:http_default_timeouts)
+ options.with_defaults(DEFAULT_TIMEOUT_OPTIONS)
+ else
+ options
+ end
+
+ httparty_perform_request(http_method, path, options_with_timeouts, &block)
rescue HTTParty::RedirectionTooDeep
raise RedirectionTooDeep
- end
-
- def self.try_get(path, options = {}, &block)
- log_info = options.delete(:extra_log_info)
- self.get(path, options, &block)
-
rescue *HTTP_ERRORS => e
extra_info = log_info || {}
extra_info = log_info.call(e, path, options) if log_info.respond_to?(:call)
-
Gitlab::ErrorTracking.log_exception(e, extra_info)
+ raise e
+ end
+
+ def self.try_get(path, options = {}, &block)
+ self.get(path, options, &block)
+ rescue *HTTP_ERRORS
nil
end
end
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index 18f4cb559c5..3b19ae3d7ff 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -4,6 +4,20 @@ module Gitlab
module I18n
extend self
+ # Languages with less then 2% of available translations will not
+ # be available in the UI.
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/221012
+ NOT_AVAILABLE_IN_UI = %w[
+ fil_PH
+ pl_PL
+ nl_NL
+ id_ID
+ cs_CZ
+ bg
+ eo
+ gl_ES
+ ].freeze
+
AVAILABLE_LANGUAGES = {
'bg' => 'Bulgarian - български',
'cs_CZ' => 'Czech - čeština',
@@ -29,6 +43,10 @@ module Gitlab
'zh_TW' => 'Chinese, Traditional (Taiwan) - 繁體中文 (台灣)'
}.freeze
+ def selectable_locales
+ AVAILABLE_LANGUAGES.reject { |key, _value| NOT_AVAILABLE_IN_UI.include? key }
+ end
+
def available_locales
AVAILABLE_LANGUAGES.keys
end
diff --git a/lib/gitlab/i18n/html_todo.yml b/lib/gitlab/i18n/html_todo.yml
new file mode 100644
index 00000000000..bfd96ba8579
--- /dev/null
+++ b/lib/gitlab/i18n/html_todo.yml
@@ -0,0 +1,315 @@
+#
+# PLEASE DO NOT ADD NEW STRINGS TO THIS FILE.
+#
+# See https://docs.gitlab.com/ee/development/i18n/externalization.html#html
+# for information on how to handle HTML in translations.
+
+#
+# This file contains strings that need to be fixed to use the
+# updated HTML guidelines. Any strings in this file will no
+# longer be able to be translated until they have been updated.
+#
+# This file (and the functionality around it) will be removed
+# once https://gitlab.com/gitlab-org/gitlab/-/issues/217933 is complete.
+#
+# See https://gitlab.com/gitlab-org/gitlab/-/issues/19485 for more details
+# why this change has been made.
+#
+
+" or <!merge request id>":
+ translations:
+ - " ወይም <!merge request id>"
+ - " ou <!merge request id>"
+ - " または <!merge request id>"
+ - "或 <!合併請求 id>"
+ - " или <!merge request id>"
+ - "或<!merge request id>"
+ - " або <!merge request id>"
+ - " oder <!merge request id>"
+ - " o <!merge request id>"
+ - " 또는 <!merge request id>"
+ - " o <!merge request id>"
+ - " veya <!merge request id>"
+ - " neu <!merge request id>"
+ - " neu <#issue id>"
+" or <#issue id>":
+ translations:
+ - "或 <#issue id>"
+ - " ወይም ‹#issue id›"
+ - " ou <identificación #issue>"
+ - " ou <#issue id>"
+ - " または <#課題 ID>"
+ - " o <#issue id>"
+ - "或 <#議題 id>"
+ - " ou <#issue id>"
+ - " или <#issue id>"
+ - "或 <#issue id>"
+ - " або <#issue id>"
+ - " oder <#issue id>"
+ - " o <#issue id>"
+ - " 또는 <#issue id>"
+ - " ou <#issue id>"
+ - " o <#issue id>"
+ - " veya <#issue id>"
+ - " neu <#issue id>"
+" or <&epic id>":
+ translations:
+ - " ወይም <&epic id>"
+ - " または <&エピックID>"
+ - " 或 <#史詩 id>"
+ - " или <&epic id>"
+ - " 或<#epic id>"
+ - " або <&epic id>"
+ - " oder <&epic id>"
+ - " o <&epic id>"
+ - " veya <&epic id>"
+ - " neu <#epic id>"
+ - " 또는 <&epic id>"
+"< 1 hour":
+ translations:
+ - "1 時間未満"
+ - "< 1 小時"
+ - "< 1 часа"
+ - "< 1小时"
+ - "< 1 години"
+ - "< 1 hora"
+ - "< 1 saat"
+ - "< 1 Stunde"
+ - "< 1시간"
+
+#
+# Strings below are fixed in the source code but the translations are still present in CrowdIn so the
+# locale files will fail the linter. They can be deleted after next CrowdIn sync, likely in:
+# https://gitlab.com/gitlab-org/gitlab/-/issues/226008
+#
+
+"This commit was signed with an <strong>unverified</strong> signature.":
+ plural_id:
+ translations:
+ - "このコミットは<strong>検証されていない</strong> 署名でサインされています。"
+ - "Этот коммит был подписан <strong>непроверенной</strong> подписью."
+ - "此提交使用 <strong>未经验证的</strong> 签名进行签名。"
+ - "Цей коміт підписано <strong>неперевіреним</strong> підписом."
+ - "Esta commit fue firmado con una firma <strong>no verificada</strong>."
+"This commit was signed with a <strong>verified</strong> signature and the committer email is verified to belong to the same user.":
+ plural_id:
+ translations:
+ - "このコミットは <strong>検証済み</strong> の署名でサインされており、このコミッターのメールは同じユーザーのものであることが検証されています。"
+ - "Это коммит был подписан <strong>верифицированной</strong> подписью и коммитер подтвердил, что адрес почты принадлежит ему."
+ - "此提交使用 <strong>已验证</strong> 的签名进行签名,并且已验证提交者的电子邮件属于同一用户。"
+ - "Цей коміт підписано <strong>перевіреним</strong> підписом і адреса електронної пошти комітера гарантовано належить тому самому користувачу."
+ - "Este commit fue firmado con una firma verificada, y <strong>se ha verificado</strong> que la dirección de correo electrónico del committer y la firma pertenecen al mismo usuario."
+"Branch <strong>%{branch_name}</strong> was created. To set up auto deploy, choose a GitLab CI Yaml template and commit your changes. %{link_to_autodeploy_doc}":
+ plural_id:
+ translations:
+ - "分支 <strong>%{branch_name}</strong> 已創建。如需設置自動部署, 請選擇合適的 GitLab CI Yaml 模板併提交更改。%{link_to_autodeploy_doc}"
+ - "O branch <strong>%{branch_name}</strong> foi criado. Para configurar o deploy automático, selecione um modelo de Yaml do GitLab CI e commit suas mudanças. %{link_to_autodeploy_doc}"
+ - "<strong>%{branch_name}</strong> ブランチが作成されました。自動デプロイを設定するには、GitLab CI Yaml テンプレートを選択して、変更をコミットしてください。 %{link_to_autodeploy_doc}"
+ - "La branch <strong>%{branch_name}</strong> è stata creata. Per impostare un rilascio automatico scegli un template CI di Gitlab e committa le tue modifiche %{link_to_autodeploy_doc}"
+ - "O ramo <strong>%{branch_name}</strong> foi criado. Para configurar a implantação automática, seleciona um modelo de Yaml do GitLab CI e envia as tuas alterações. %{link_to_autodeploy_doc}"
+ - "Ветка <strong>%{branch_name}</strong> создана. Для настройки автоматического развертывания выберите YAML-шаблон для GitLab CI и зафиксируйте свои изменения. %{link_to_autodeploy_doc}"
+ - "已创建分支 <strong>%{branch_name}</strong> 。如需设置自动部署, 请选择合适的 GitLab CI Yaml 模板并提交更改。%{link_to_autodeploy_doc}"
+ - "Гілка <strong>%{branch_name}</strong> створена. Для настройки автоматичного розгортання виберіть GitLab CI Yaml-шаблон і закомітьте зміни. %{link_to_autodeploy_doc}"
+ - "Клонът <strong>%{branch_name}</strong> беше създаден. За да настроите автоматичното внедряване, изберете Yaml шаблон за GitLab CI и подайте промените си. %{link_to_autodeploy_doc}"
+ - "Branch <strong>%{branch_name}</strong> wurde erstellt. Um die automatische Bereitstellung einzurichten, wähle eine GitLab CI Yaml Vorlage und committe deine Änderungen. %{link_to_autodeploy_doc}"
+ - "<strong>%{branch_name}</strong> 브랜치가 생성되었습니다. 자동 배포를 설정하려면 GitLab CI Yaml 템플릿을 선택하고 변경 사항을 적용하십시오. %{link_to_autodeploy_doc}"
+ - "La branĉo <strong>%{branch_name}</strong> estis kreita. Por agordi aŭtomatan disponigadon, bonvolu elekti Yaml-ŝablonon por GitLab CI kaj enmeti viajn ŝanĝojn. %{link_to_autodeploy_doc}"
+ - "La branche <strong>%{branch_name}</strong> a été créée. Pour mettre en place le déploiement automatisé, sélectionnez un modèle de fichier YAML pour l’intégration continue (CI) de GitLab, et validez les modifications. %{link_to_autodeploy_doc}"
+ - "La rama <strong>%{branch_name}</strong> fue creada. Para configurar el auto despliegue, escoge una plantilla Yaml para GitLab CI y envía tus cambios. %{link_to_autodeploy_doc}"
+"GitLabPages|GitLab Pages are disabled for this project. You can enable them on your project's %{strong_start}Settings > General > Visibility%{strong_end} page.":
+ plural_id:
+ translations:
+ - "GitLab Pagesはこのプロジェクトでは無効になっています。 プロジェクトの%{strong_start} 設定> 全般> 可視性%{strong_end}ページで有効にできます。"
+ - "GitLab Pages отключены для этого проекта. Вы можете включить в поле %{strong_start}Настройки > Общие > Видимость%{strong_end} вашего проекта."
+ - "此项目禁用GitLab Pages。您可以在您的项目的%{strong_start}设置 > 常规 > 可见性%{strong_end} 页面启用。"
+ - "GitLab Pages вимкнено для цього проєкту. Ви можете їх увімкнути перейшовши на сторінку проєкту %{strong_start}Налаштування > Загальні > Видимість%{strong_end}."
+ - "Las páginas de GitLab están deshabilitadas para este proyecto. Puede habilitarlas en los ajustes %{strong_start} de su proyecto > General > Visibilidad%{strong_end}."
+"You can invite a new member to <strong>%{project_name}</strong> or invite another group.":
+ plural_id:
+ translations:
+ - "新しいメンバーを<strong>%{project_name} </strong>に招待するか、別のグループを招待することができます。"
+ - "Podes convidar um novo para <strong>%{project_name}</strong> ou convidar outro grupo."
+ - "邀请新成员或另一个群组加入<strong>%{project_name}</strong>。"
+ - "Puede invitar a un nuevo miembro a <strong>%{project_name}</strong> o invitar a otro grupo."
+ - "<strong>%{project_name}</strong> projesine yeni bir üye davet edebilir veya başka bir grubu davet edebilirsiniz."
+ - "Вы можете пригласить нового участника в <strong>%{project_name}</strong> или пригласить другую группу."
+ - "Ви можете запросити нового учасника до <strong>%{project_name}</strong> або запросити іншу групу."
+"You can invite a new member to <strong>%{project_name}</strong>.":
+ plural_id:
+ translations:
+ - "新しいメンバーを<strong>%{project_name} </strong>に招待できます。"
+ - "Podes convidar um novo membro para <strong>%{project_name}</strong>."
+ - "邀请新成员加入<strong>%{project_name}</strong>。"
+ - "Puedes invitar a un nuevo miembro a <strong>%{project_name}</strong>."
+ - "<strong>%{project_name}</strong> projesine yeni bir üye davet edebilirsiniz."
+ - "Вы можете пригласить нового участника в <strong>%{project_name}</strong>."
+ - "Ви можете запросити нового учасника до <strong>%{project_name}</strong>."
+"You can invite another group to <strong>%{project_name}</strong>.":
+ plural_id:
+ translations:
+ - "他のグループを<strong>%{project_name} </strong>に招待できます。"
+ - "Podes convidar outro grupo para <strong>%{project_name}</strong>."
+ - "您可以邀请另一个群组加入<strong>%{project_name}</strong>。"
+ - "Ви можете запросити нову групу до <strong>%{project_name}</strong>."
+ - "Puedes invitar a otro grupo a <strong>%{project_name}</strong>."
+"Example: <code>192.168.0.0/24</code>. %{read_more_link}.":
+ plural_id:
+ translations:
+"Note that PostgreSQL %{pg_version_upcoming} will become the minimum required version in GitLab %{gl_version_upcoming} (%{gl_version_upcoming_date}). Please consider upgrading your environment to a supported PostgreSQL version soon, see <a href=\\\"%{pg_version_upcoming_url}\\\">the related epic</a> for details.":
+ plural_id:
+ translations:
+"Authorize <strong>%{user}</strong> to use your account?":
+ plural_id:
+ translations:
+"DeployFreeze|Specify times when deployments are not allowed for an environment. The <code>gitlab-ci.yml</code> file must be updated to make deployment jobs aware of the %{freeze_period_link_start}freeze period%{freeze_period_link_end}.":
+ plural_id:
+ translations:
+"<project name>":
+ translations:
+ - "<название проекта>"
+ - "<project name>"
+ - "<proje adı>"
+ - "<naziv projekta>"
+ - "<ім’я проєкту>"
+ - "<프로젝트 이름>"
+"<strong>Deletes</strong> source branch":
+ plural_id:
+ translations:
+ - "<strong>刪除</strong>來源分支"
+ - "<strong>Apagar</strong> branch de origem"
+ - "ソースブランチを<strong>削除</strong>"
+ - "<strong>刪除</strong>來源分支"
+ - "<strong>Apagar</strong> o ramo de origem"
+ - "<strong>Удаляет</strong> исходную ветку"
+ - "<strong>删除</strong>源分支"
+ - "<strong>Видаляє</strong> гілку-джерело"
+ - "<strong>Löscht</strong> den Quellbranch"
+ - "소스 브랜치 <strong>삭제</strong>"
+ - "<strong>Supprime</strong> la branche source"
+ - "<strong>elimina</strong> la rama origen"
+ - "Kaynak dalı <strong>siler</strong>"
+"Badges|You are going to delete this badge. Deleted badges <strong>cannot</strong> be restored.":
+ plural_id:
+ translations:
+ - "Você está prestes a excluir este selo. Selos excluídos <strong>não podem</strong> ser restaurados."
+ - "このバッジを削除しようとしています。削除されたバッジは<strong>復元できません</strong>。"
+ - "Estás prestes a apagar este emblema. Emblemas apagados <strong>não podem</strong> ser restaurados."
+ - "Вы собираетесь удалить этот значок. Удаленные значки <strong>не могут</strong> быть восстановлены."
+ - "您即将删除此徽章。徽章被删除后 <strong>不能</strong> 恢复。"
+ - "Ви збираєтеся видалити цей значок. Вилучені значки <strong>не можуть</strong> бути відновлені."
+ - "Du bist gerade dabei dieses Badge zu entfernen. Entfernte Badges können <strong>nicht</strong> rückgängig gemacht werden."
+ - "이 배지를 삭제하려고합니다. 삭제 된 배지는 <strong>복원 할 수 없습니다</strong>."
+ - "Vous êtes sur le point de supprimer ce badge. Les badges supprimés <strong>ne peuvent pas</strong> être restaurés."
+ - "Va a eliminar esta insignia. Las insignias eliminadas <strong>no se pueden</strong> restaurar."
+ - "Bu rozeti sileceksiniz. Silinen rozetler geri <strong>yüklenemez</strong>."
+"ClusterIntegration| This will permanently delete the following resources: <ul> <li>All installed applications and related resources</li> <li>The <code>gitlab-managed-apps</code> namespace</li> <li>Any project namespaces</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>":
+ plural_id:
+ translations:
+ - "これにより、次のリソースは完全に削除されます <ul> <li>インストールされているすべてのアプリケーションと関連したリソース</li> <li> <code>gitlab-managed-apps</code> 名前空間</li> <li>任意のプロジェクト名前空間</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>"
+ - "此操作将永久删除下列资源: <ul> <li>所有已安装的应用程序和相关资源</li> <li> <code>GitLab管理的应用</code> 命名空间</li> <li>任何项目命名空间</li> <li><code>clusterroles</code></li> <li><code>clusterrolebindings</code></li> </ul>"
+ - "Esto eliminará permanentemente los siguientes recursos: <ul> <li>Todas las aplicaciones instaladas y sus recursos relacionados</li> <li>El espacio de nombres <code>gitlab-managed-apps</code></li> <li>Cualquier espacio de nombres de proyecto</li> <li><code> clusterroles </code></li> <li><code>clusterrolebindings</code></li> </ul>"
+"Configure a <code>.gitlab-webide.yml</code> file in the <code>.gitlab</code> directory to start using the Web Terminal. %{helpStart}Learn more.%{helpEnd}":
+ plural_id:
+ translations:
+ - "Configure um arquivo <code>.gitlab-webide.yml</code> no diretório <code>.gitlab</code> para começar a usar o Terminal Web. %{helpStart}Saiba mais.%{helpEnd}"
+ - "Webターミナルの使用を開始するには、 <code>.gitlab</code> ディレクトリの <code>.gitlab-webide.yml</code> ファイルを設定します。 詳細は%{helpStart}こちら%{helpEnd}です。"
+ - "Сконфигурируйте файл <code>.gitlab-webide.yml</code> в каталоге <code>.gitlab</code> чтобы начать использовать веб-терминал. %{helpStart}Узнайте больше.%{helpEnd}"
+ - "在 <code>.gitlab</code> 目录中配置 <code>.gitlab-webide.yml</code> 文件以开始使用Web终端。 %{helpStart}了解更多。%{helpEnd}"
+ - "Налаштуйте файл <code>.gitlab-webide.yml</code> у директорії <code>.gitlab</code>, щоб почати використовувати Веб-термінал. %{helpStart}Докладніше.%{helpEnd}"
+ - "웹 터미널 사용을 시작하도록 <code>.gitlab</code> 디렉토리에서 <code>.gitlab-webide.yml</code> 파일을 구성하십시오. %{helpStart}자세히 알아보십시오.%{helpEnd}"
+ - "Configure un archivo <code>.gitlab-webide.yml</code> en el directorio <code>.gitlab</code> para comenzar a utilizar el Terminal Web. %{helpStart}Aprende más.%{helpEnd}"
+"Depends on <strong>%d closed</strong> merge request.":
+ plural_id: "Depends on <strong>%d closed</strong> merge requests."
+ translations:
+ - "В зависимости от <strong>%d закрытого</strong> запроса на слияние."
+ - "В зависимости от <strong>%d закрытых</strong> запросов на слияние."
+ - "В зависимости от <strong>%d закрытых</strong> запросов на слияние."
+ - "В зависимости от <strong>%d закрытых</strong> запросов на слияние."
+ - "依赖于<strong>%d个已关闭的</strong>合并请求"
+ - "Залежить від %d <strong>закритого</strong> запиту на злиття."
+ - "Залежить від %d <strong>закритих</strong> запитів на злиття."
+ - "Залежить від %d <strong>закритих</strong> запитів на злиття."
+ - "Залежить від %d <strong>закритих</strong> запитів на злиття."
+ - "<strong>%d kapanan</strong> birleştirme isteğine bağlıdır."
+ - "<strong>%d kapanan</strong> birleştirme isteğine bağlıdır."
+"Go to <strong>Issues</strong> > <strong>Boards</strong> to access your personalized learning issue board.":
+ plural_id:
+ translations:
+ - "转至<strong>议题</strong> > <strong>看板</strong>访问您的个性化学习议题看板。"
+"Labels|<span>Promote label</span> %{labelTitle} <span>to Group Label?</span>":
+ plural_id:
+ translations:
+ - "<span>要讓標籤</span> %{labelTitle} <span>提升到群組標籤嗎?</span>"
+ - "<span>Promover a etiqueta</span> %{labelTitle} <span>para etiqueta do Grupo?</span>"
+ - "%{labelTitle} <span>ラベルをグループラベルに昇格しますか?</span>"
+ - "<span>Повысить метку</span> %{labelTitle} <span>до групповой метки?</span>"
+ - "<span>将标记</span> %{labelTitle} <span>升级为群组标记?</span>"
+ - "<span>Перенести мітку</span> %{labelTitle} <span>на рівень групи?</span>"
+ - "<span>Label</span> %{labelTitle} <span>zu Gruppenlabel hochstufen?</span>"
+ - "<span>라벨</span> %{labelTitle} <span>(을)를 그룹 라벨로 승격하시겠습니까?</span>"
+ - "<span>Promouvoir l’étiquette</span> %{labelTitle} <span>en étiquette de groupe ?</span>"
+ - "<span>¿Promocionar la etiqueta</span> %{labelTitle} <span>a etiqueta de grupo?</span>"
+"Lock this %{issuableDisplayName}? Only <strong>project members</strong> will be able to comment.":
+ plural_id:
+ translations:
+ - "Travar este %{issuableDisplayName}? Apenas <strong>membros do projeto</strong> poderão comentar."
+ - "%{issuableDisplayName} をロックしますか?<strong>プロジェクトメンバー</strong> のみコメントできます。"
+ - "锁定此%{issuableDisplayName}吗?锁定后将只有<strong>项目成员</strong>可以发表评论。"
+ - "Заблокувати цю %{issuableDisplayName}? Лише <strong>учасники проекту</strong> зможуть коментувати."
+ - "%{issuableDisplayName} sperren? Es werden nur noch <strong>Projektmitglieder</strong> kommentieren können."
+ - "Verrouiller ce·t·te %{issuableDisplayName} ? Seuls les <strong>membres du projet</strong> seront en mesure de commenter."
+ - "¿Bloquear este %{issuableDisplayName}? Sólo los <strong>miembros del proyecto</strong> podrán comentar."
+"PrometheusService|<p class=\\\"text-tertiary\\\">No <a href=\\\"%{docsUrl}\\\">common metrics</a> were found</p>":
+ plural_id:
+ translations:
+ - "<p class=\\\"text-tertiary\\\">Nenhuma <a href=\\\"%{docsUrl}\\\">métrica comum</a> foi encontrada</p>"
+ - "<p class=\\\"text-tertiary\\\"><a href=\\\"%{docsUrl}\\\">共通メトリクス</a>は見つかりませんでした</p>"
+ - "<p class=\\\"text-tertiary\\\">Ни одной <a href=\\\"%{docsUrl}\\\">общей метрики</a> не найдено</p>"
+ - "<p class=\\\"text-tertiary\\\">无<a href=\\\"%{docsUrl}\\\">常用指标</a> </p>"
+ - "<p class=\\\"text-tertiary\\\">Ніяких <a href=\\\"%{docsUrl}\\\">загальних метрик</a> не знайдено</p>"
+ - "<p class=\\\"text-tertiary\\\">Es wurden keine <a href=\\\"%{docsUrl}\\\">allgemeinen Metriken</a> gefunden</p>"
+ - "<p class=\\\"text-tertiary\\\"><a href=\\\"%{docsUrl}\\\">공통 메트릭스</a>가 발견되지 않았습니다.</p>"
+ - "<p class=\\\"text-tertiary\\\">Aucune <a href=\\\"%{docsUrl}\\\">métrique commune</a> trouvée</p>"
+ - "<p class=\\\"text-tertiary\\\">No se han encontrado<a href=\\\"%{docsUrl}\\\">métricas comunes</a> </p>"
+"This project does not have billing enabled. To create a cluster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">enable billing <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> and try again.":
+ plural_id:
+ translations:
+ - "Este projeto não possui faturamento ativado. Para criar um cluster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">ative o faturamento <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> e tente novamente."
+ - "このプロジェクトでは課金が有効になっていません。クラスターを作成するには、<a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\"> 課金を有効<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> にして再度お試しください。"
+ - "此项目未启用账单。要创建群集,请 <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">启用账单 <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> 并重试。"
+ - "Для цього проекту вимкнено білінг. Щоб створити кластер, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">увімкніть білінг <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> і спробуйте знову."
+ - "Für dieses Projekt ist keine Abrechnung aktiviert. Um ein Cluster zu erstellen, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">aktiviere die Abrechnung<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> und versuche es erneut."
+ - "Ce projet n’a pas de facturation activée. Afin de créer une grappe de serveurs, veuillez <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">activer la facturation<i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> et réessayer."
+ - "Este proyecto no tiene la facturación habilitada. Para crear un clúster, <a href=%{linkToBilling} target=\\\"_blank\\\" rel=\\\"noopener noreferrer\\\">habilite la facturación <i class=\\\"fa fa-external-link\\\" aria-hidden=\\\"true\\\"></i></a> e inténtelo de nuevo."
+"Unlock this %{issuableDisplayName}? <strong>Everyone</strong> will be able to comment.":
+ plural_id:
+ translations:
+ - "Desbloquear este %{issuableDisplayName}? <strong>Todos</strong> poderão comentar."
+ - "%{issuableDisplayName} のロックを解除しますか? <strong>全員</strong>がコメントできるようになります。"
+ - "解锁此%{issuableDisplayName}吗?解锁后<strong>所有人</strong>都将可以发表评论。"
+ - "Розблокувати %{issuableDisplayName}? <strong>Будь-хто</strong> зможе залишати коментарі."
+ - "Dieses %{issuableDisplayName} entsperren? <strong>Jeder</strong> wird in der Lage sein zu kommentieren."
+ - "%{issuableDisplayName}(을)를 잠금해제 하시겠습니까? <strong>모두가</strong> 코멘트 할 수 있게 됩니다."
+ - "Déverrouiller %{issuableDisplayName} ? <strong>Tout le monde</strong> sera en mesure de commenter."
+ - "Desbloquear este %{issuableDisplayName}? <strong>Todos</strong> podrán comentar."
+"confidentiality|You are going to turn off the confidentiality. This means <strong>everyone</strong> will be able to see and leave a comment on this issue.":
+ plural_id:
+ translations:
+ - "Você está prestes a desligar a confidencialidade. Isso significa que <strong>todos</strong> serão capazes de ver e deixar comentários nesse issue."
+ - "あなたは公開設定に変更しようとしています。これは<strong>すべての人</strong> が閲覧可能になり、課題に対してコメントを残すことができるようになることを意味します。"
+ - "即将关闭私密性。这将使得 <strong>所有用户</strong>都可以查看并且评论当前议题。"
+ - "Ви вимикаєте конфіденційність. Це означає, що <strong>будь-хто</strong> зможе бачити і залишати коментарі для цієї задачі."
+ - "Du willst die Vertraulichkeit deaktivieren. Das bedeutet, dass <strong>alle</strong> das Ticket betrachten und kommentieren können."
+ - "Vous êtes sur le point de désactiver la confidentialité. Cela signifie que <strong>tout le monde</strong> sera en mesure de voir et de laisser un commentaire sur ce ticket."
+ - "Va a desactivar la confidencialidad. Esto significa que <strong>todos</strong> podrán ver y dejar un comentario sobre este tema."
+"confidentiality|You are going to turn on the confidentiality. This means that only team members with <strong>at least Reporter access</strong> are able to see and leave comments on the issue.":
+ plural_id:
+ translations:
+ - "Você está prestes a ligar a confidencialidade. Isso significa que apenas membros da equipe com <strong>ao menos acesso de Relator</strong> serão capazes de ver e deixar comentários nesse issue."
+ - "あなたは公開設定に変更しようとしています。これはチームに限定していた<strong>最小限の報告権限</strong>をなくし、課題に対してコメントを残すことができるようになることを意味します。"
+ - "即将设置私密性。这将使得 <strong>至少有Reporter以上权限</strong>的团队成员才能查看并且评论当前议题。"
+ - "Ви вмикаєте конфіденційність. Це означає що лише учасники команди <strong>рівня репортер або вище</strong> матимуть змогу бачити та залишати коментарі для цієї задачі."
+ - "Du willst die Vertraulichkeit aktivieren. Das bedeutet, dass nur Teammitglieder mit <strong>mindestens Reporter-Zugriff</strong> das Ticket betrachten und kommentieren können."
+ - "Vous êtes sur le point de d’activer la confidentialité. Cela signifie que seuls les membres de l’équipe avec <strong>au moins un accès en tant que rapporteur</strong> seront en mesure de voir et de laisser des commentaires sur le ticket."
+ - "Va a activar la confidencialidad. Esto significa que solo los miembros del equipo con como mínimo,<strong>acceso como Reporter</strong> podrán ver y dejar comentarios sobre la incidencia."
+ - "あなたは非公開設定をオンにしようとしています。これは、最低でも<strong>報告権限</strong>を持ったチームメンバーのみが課題を表示したりコメントを残したりすることができるようになるということです。"
diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb
index c0687cd9b79..e56b88dfce0 100644
--- a/lib/gitlab/i18n/po_linter.rb
+++ b/lib/gitlab/i18n/po_linter.rb
@@ -5,13 +5,14 @@ module Gitlab
class PoLinter
include Gitlab::Utils::StrongMemoize
- attr_reader :po_path, :translation_entries, :metadata_entry, :locale
+ attr_reader :po_path, :translation_entries, :metadata_entry, :locale, :html_todolist
VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze
- def initialize(po_path, locale = I18n.locale.to_s)
+ def initialize(po_path:, html_todolist:, locale: I18n.locale.to_s)
@po_path = po_path
@locale = locale
+ @html_todolist = html_todolist
end
def errors
@@ -19,7 +20,7 @@ module Gitlab
end
def validate_po
- if parse_error = parse_po
+ if (parse_error = parse_po)
return 'PO-syntax errors' => [parse_error]
end
@@ -38,7 +39,11 @@ module Gitlab
end
@translation_entries = entries.map do |entry_data|
- Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_forms)
+ Gitlab::I18n::TranslationEntry.new(
+ entry_data: entry_data,
+ nplurals: metadata_entry.expected_forms,
+ html_allowed: html_todolist.fetch(entry_data[:msgid], false)
+ )
end
nil
@@ -66,6 +71,7 @@ module Gitlab
validate_newlines(errors, entry)
validate_number_of_plurals(errors, entry)
validate_unescaped_chars(errors, entry)
+ validate_html(errors, entry)
validate_translation(errors, entry)
errors
@@ -85,6 +91,23 @@ module Gitlab
end
end
+ def validate_html(errors, entry)
+ common_message = 'contains < or >. Use variables to include HTML in the string, or the &lt; and &gt; codes ' \
+ 'for the symbols. For more info see: https://docs.gitlab.com/ee/development/i18n/externalization.html#html'
+
+ if entry.msgid_contains_potential_html? && !entry.msgid_html_allowed?
+ errors << common_message
+ end
+
+ if entry.plural_id_contains_potential_html? && !entry.plural_id_html_allowed?
+ errors << 'plural id ' + common_message
+ end
+
+ if entry.translations_contain_potential_html? && !entry.translations_html_allowed?
+ errors << 'translation ' + common_message
+ end
+ end
+
def validate_number_of_plurals(errors, entry)
return unless metadata_entry&.expected_forms
return unless entry.translated?
diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb
index 19c10b2e402..25a45332d27 100644
--- a/lib/gitlab/i18n/translation_entry.rb
+++ b/lib/gitlab/i18n/translation_entry.rb
@@ -4,12 +4,14 @@ module Gitlab
module I18n
class TranslationEntry
PERCENT_REGEX = /(?:^|[^%])%(?!{\w*}|[a-z%])/.freeze
+ ANGLE_BRACKET_REGEX = /[<>]/.freeze
- attr_reader :nplurals, :entry_data
+ attr_reader :nplurals, :entry_data, :html_allowed
- def initialize(entry_data, nplurals)
+ def initialize(entry_data:, nplurals:, html_allowed:)
@entry_data = entry_data
@nplurals = nplurals
+ @html_allowed = html_allowed
end
def msgid
@@ -83,8 +85,38 @@ module Gitlab
string =~ PERCENT_REGEX
end
+ def msgid_contains_potential_html?
+ contains_angle_brackets?(msgid)
+ end
+
+ def plural_id_contains_potential_html?
+ contains_angle_brackets?(plural_id)
+ end
+
+ def translations_contain_potential_html?
+ all_translations.any? { |translation| contains_angle_brackets?(translation) }
+ end
+
+ def msgid_html_allowed?
+ html_allowed.present?
+ end
+
+ def plural_id_html_allowed?
+ html_allowed.present? && html_allowed['plural_id'] == plural_id
+ end
+
+ def translations_html_allowed?
+ msgid_html_allowed? && html_allowed['translations'].present? && all_translations.all? do |translation|
+ html_allowed['translations'].include?(translation)
+ end
+ end
+
private
+ def contains_angle_brackets?(string)
+ string =~ ANGLE_BRACKET_REGEX
+ end
+
def translation_entries
@translation_entries ||= entry_data.fetch_values(*translation_keys)
.reject(&:empty?)
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index bdecff0931c..2f8769e261d 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -47,8 +47,8 @@ module Gitlab
def execute(cmd)
output, status = Gitlab::Popen.popen(cmd)
- @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status.zero? # rubocop:disable Gitlab/ModuleWithInstanceVariables
- status.zero?
+ @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status == 0 # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ status == 0
end
def git_bin_path
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 3cb1eb72ceb..081745a49f4 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -28,7 +28,9 @@ module Gitlab
copy_archive
wait_for_archived_file do
- validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size, default_enabled: true)
+ # Disable archive validation by default
+ # See: https://gitlab.com/gitlab-org/gitlab/-/issues/235949
+ validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size)
decompress_archive
end
rescue => e
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index aa961bd8d19..a240c367a42 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -205,6 +205,7 @@ excluded_attributes:
- :state_id
- :duplicated_to_id
- :promoted_to_epic_id
+ - :blocking_issues_count
merge_request:
- :milestone_id
- :sprint_id
@@ -274,6 +275,7 @@ excluded_attributes:
timelogs:
- :issue_id
- :merge_request_id
+ - :note_id
notes:
- :noteable_id
- :review_id
diff --git a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb
index cd947b15154..768c8bb4cbb 100644
--- a/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb
+++ b/lib/gitlab/incident_management/pager_duty/incident_issue_description.rb
@@ -32,7 +32,7 @@ module Gitlab
end
def incident_created_at
- Time.parse(incident_payload['created_at'])
+ Time.zone.parse(incident_payload['created_at'])
rescue
Time.current.utc # PagerDuty provides time in UTC
end
diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb
index 2889dbc68cc..d55906083ff 100644
--- a/lib/gitlab/incoming_email.rb
+++ b/lib/gitlab/incoming_email.rb
@@ -8,11 +8,11 @@ module Gitlab
class << self
def enabled?
- config.enabled && config.address
+ config.enabled && config.address.present?
end
def supports_wildcard?
- config.address && config.address.include?(WILDCARD_PLACEHOLDER)
+ config.address.present? && config.address.include?(WILDCARD_PLACEHOLDER)
end
def supports_issue_creation?
diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb
index 1df899747e0..0beab008f73 100644
--- a/lib/gitlab/instrumentation/redis_base.rb
+++ b/lib/gitlab/instrumentation/redis_base.rb
@@ -96,7 +96,7 @@ module Gitlab
:gitlab_redis_client_requests_duration_seconds,
'Client side Redis request latency, per Redis server, excluding blocking commands',
{},
- [0.005, 0.01, 0.1, 0.5]
+ [0.1, 0.5, 0.75, 1]
)
@request_latency_histogram.observe({ storage: storage_key }, duration)
diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb
index 659fb1472d2..7be54a214dd 100644
--- a/lib/gitlab/issuables_count_for_state.rb
+++ b/lib/gitlab/issuables_count_for_state.rb
@@ -9,9 +9,16 @@ module Gitlab
# The state values that can be safely casted to a Symbol.
STATES = %w[opened closed merged all].freeze
+ attr_reader :project
+
+ def self.declarative_policy_class
+ 'IssuablePolicy'
+ end
+
# finder - The finder class to use for retrieving the issuables.
- def initialize(finder)
+ def initialize(finder, project = nil)
@finder = finder
+ @project = project
@cache = Gitlab::SafeRequestStore[CACHE_KEY] ||= initialize_cache
end
@@ -19,6 +26,11 @@ module Gitlab
self[state || :opened]
end
+ # Define method for each state
+ STATES.each do |state|
+ define_method(state) { self[state] }
+ end
+
# Returns the count for the given state.
#
# state - The name of the state as either a String or a Symbol.
diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb
index 21f837c58bb..d6681347f42 100644
--- a/lib/gitlab/json.rb
+++ b/lib/gitlab/json.rb
@@ -220,5 +220,33 @@ module Gitlab
end
end
end
+
+ class LimitedEncoder
+ LimitExceeded = Class.new(StandardError)
+
+ # Generates JSON for an object or raise an error if the resulting json string is too big
+ #
+ # @param object [Hash, Array, Object] must be hash, array, or an object that responds to .to_h or .to_json
+ # @param limit [Integer] max size of the resulting json string
+ # @return [String]
+ # @raise [LimitExceeded] if the resulting json string is bigger than the specified limit
+ def self.encode(object, limit: 25.megabytes)
+ return ::Gitlab::Json.dump(object) unless Feature.enabled?(:json_limited_encoder)
+
+ buffer = []
+ buffer_size = 0
+
+ ::Yajl::Encoder.encode(object) do |data_chunk|
+ chunk_size = data_chunk.bytesize
+
+ raise LimitExceeded if buffer_size + chunk_size > limit
+
+ buffer << data_chunk
+ buffer_size += chunk_size
+ end
+
+ buffer.join('')
+ end
+ end
end
end
diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb
new file mode 100644
index 00000000000..55afd2b586e
--- /dev/null
+++ b/lib/gitlab/kubernetes/cilium_network_policy.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ class CiliumNetworkPolicy
+ include NetworkPolicyCommon
+ extend ::Gitlab::Utils::Override
+
+ API_VERSION = "cilium.io/v2"
+ KIND = 'CiliumNetworkPolicy'
+
+ def initialize(name:, namespace:, selector:, ingress:, resource_version:, labels: nil, creation_timestamp: nil, egress: nil)
+ @name = name
+ @namespace = namespace
+ @labels = labels
+ @creation_timestamp = creation_timestamp
+ @selector = selector
+ @resource_version = resource_version
+ @ingress = ingress
+ @egress = egress
+ end
+
+ def generate
+ ::Kubeclient::Resource.new.tap do |resource|
+ resource.kind = KIND
+ resource.apiVersion = API_VERSION
+ resource.metadata = metadata
+ resource.spec = spec
+ end
+ end
+
+ def self.from_yaml(manifest)
+ return unless manifest
+
+ policy = YAML.safe_load(manifest, symbolize_names: true)
+ return if !policy[:metadata] || !policy[:spec]
+
+ metadata = policy[:metadata]
+ spec = policy[:spec]
+ self.new(
+ name: metadata[:name],
+ namespace: metadata[:namespace],
+ resource_version: metadata[:resourceVersion],
+ labels: metadata[:labels],
+ selector: spec[:endpointSelector],
+ ingress: spec[:ingress],
+ egress: spec[:egress]
+ )
+ rescue Psych::SyntaxError, Psych::DisallowedClass
+ nil
+ end
+
+ def self.from_resource(resource)
+ return unless resource
+ return if !resource[:metadata] || !resource[:spec]
+
+ metadata = resource[:metadata]
+ spec = resource[:spec].to_h
+ self.new(
+ name: metadata[:name],
+ namespace: metadata[:namespace],
+ resource_version: metadata[:resourceVersion],
+ labels: metadata[:labels]&.to_h,
+ creation_timestamp: metadata[:creationTimestamp],
+ selector: spec[:endpointSelector],
+ ingress: spec[:ingress],
+ egress: spec[:egress]
+ )
+ end
+
+ private
+
+ attr_reader :name, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress
+
+ def selector
+ @selector ||= {}
+ end
+
+ override :spec
+ def spec
+ {
+ endpointSelector: selector,
+ ingress: ingress,
+ egress: egress
+ }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb
index f27ad05599e..49d2969f7f3 100644
--- a/lib/gitlab/kubernetes/helm/base_command.rb
+++ b/lib/gitlab/kubernetes/helm/base_command.rb
@@ -6,21 +6,16 @@ module Gitlab
class BaseCommand
attr_reader :name, :files
- def initialize(rbac:, name:, files:, local_tiller_enabled:)
+ def initialize(rbac:, name:, files:)
@rbac = rbac
@name = name
@files = files
- @local_tiller_enabled = local_tiller_enabled
end
def rbac?
@rbac
end
- def local_tiller_enabled?
- @local_tiller_enabled
- end
-
def pod_resource
pod_service_account_name = rbac? ? service_account_name : nil
diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb
index 24458e1b4b3..a9e93c0c90e 100644
--- a/lib/gitlab/kubernetes/helm/client_command.rb
+++ b/lib/gitlab/kubernetes/helm/client_command.rb
@@ -5,30 +5,11 @@ module Gitlab
module Helm
module ClientCommand
def init_command
- if local_tiller_enabled?
- <<~HEREDOC.chomp
+ <<~SHELL.chomp
export HELM_HOST="localhost:44134"
tiller -listen ${HELM_HOST} -alsologtostderr &
helm init --client-only
- HEREDOC
- else
- # Here we are always upgrading to the latest version of Tiller when
- # installing an app. We ensure the helm version stored in the
- # database is correct by also updating this after transition to
- # :installed,:updated in Clusters::Concerns::ApplicationStatus
- 'helm init --upgrade'
- end
- end
-
- def wait_for_tiller_command
- return if local_tiller_enabled?
-
- helm_check = ['helm', 'version', *optional_tls_flags].shelljoin
- # This is necessary to give Tiller time to restart after upgrade.
- # Ideally we'd be able to use --wait but cannot because of
- # https://github.com/helm/helm/issues/4855
-
- "for i in $(seq 1 30); do #{helm_check} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s)"
+ SHELL
end
def repository_command
@@ -37,12 +18,6 @@ module Gitlab
private
- def tls_flags_if_remote_tiller
- return [] if local_tiller_enabled?
-
- optional_tls_flags
- end
-
def repository_update_command
'helm repo update'
end
diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb
index 3bb41d09994..f8b9601bc98 100644
--- a/lib/gitlab/kubernetes/helm/delete_command.rb
+++ b/lib/gitlab/kubernetes/helm/delete_command.rb
@@ -17,7 +17,6 @@ module Gitlab
def generate_script
super + [
init_command,
- wait_for_tiller_command,
predelete,
delete_command,
postdelete
@@ -29,9 +28,7 @@ module Gitlab
end
def delete_command
- command = ['helm', 'delete', '--purge', name] + tls_flags_if_remote_tiller
-
- command.shelljoin
+ ['helm', 'delete', '--purge', name].shelljoin
end
end
end
diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb
index cf6d993cad4..d166842fce6 100644
--- a/lib/gitlab/kubernetes/helm/install_command.rb
+++ b/lib/gitlab/kubernetes/helm/install_command.rb
@@ -21,7 +21,6 @@ module Gitlab
def generate_script
super + [
init_command,
- wait_for_tiller_command,
repository_command,
repository_update_command,
preinstall,
@@ -39,7 +38,6 @@ module Gitlab
install_flag +
rollback_support_flag +
reset_values_flag +
- tls_flags_if_remote_tiller +
optional_version_flag +
rbac_create_flag +
namespace_flag +
diff --git a/lib/gitlab/kubernetes/helm/patch_command.rb b/lib/gitlab/kubernetes/helm/patch_command.rb
index 1a5fab116bd..a33dbdac134 100644
--- a/lib/gitlab/kubernetes/helm/patch_command.rb
+++ b/lib/gitlab/kubernetes/helm/patch_command.rb
@@ -26,7 +26,6 @@ module Gitlab
def generate_script
super + [
init_command,
- wait_for_tiller_command,
repository_command,
repository_update_command,
upgrade_command
@@ -38,7 +37,6 @@ module Gitlab
def upgrade_command
command = ['helm', 'upgrade', name, chart] +
reuse_values_flag +
- tls_flags_if_remote_tiller +
version_flag +
namespace_flag +
value_flag
diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb
index 2110d586d30..9e3cf58bb1e 100644
--- a/lib/gitlab/kubernetes/kube_client.rb
+++ b/lib/gitlab/kubernetes/kube_client.rb
@@ -21,7 +21,8 @@ module Gitlab
istio: { group: 'apis/networking.istio.io', version: 'v1alpha3' },
knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' },
metrics: { group: 'apis/metrics.k8s.io', version: 'v1beta1' },
- networking: { group: 'apis/networking.k8s.io', version: 'v1' }
+ networking: { group: 'apis/networking.k8s.io', version: 'v1' },
+ cilium_networking: { group: 'apis/cilium.io', version: 'v2' }
}.freeze
SUPPORTED_API_GROUPS.each do |name, params|
@@ -95,6 +96,14 @@ module Gitlab
:delete_network_policy,
to: :networking_client
+ # CiliumNetworkPolicy methods delegate to the apis/cilium.io api
+ # group client
+ delegate :create_cilium_network_policy,
+ :get_cilium_network_policies,
+ :update_cilium_network_policy,
+ :delete_cilium_network_policy,
+ to: :cilium_networking_client
+
attr_reader :api_prefix, :kubeclient_options
DEFAULT_KUBECLIENT_OPTIONS = {
@@ -107,15 +116,15 @@ module Gitlab
def self.graceful_request(cluster_id)
{ status: :connected, response: yield }
rescue *Gitlab::Kubernetes::Errors::CONNECTION
- { status: :unreachable }
+ { status: :unreachable, connection_error: :connection_error }
rescue *Gitlab::Kubernetes::Errors::AUTHENTICATION
- { status: :authentication_failure }
+ { status: :authentication_failure, connection_error: :authentication_error }
rescue Kubeclient::HttpError => e
- { status: kubeclient_error_status(e.message) }
+ { status: kubeclient_error_status(e.message), connection_error: :http_error }
rescue => e
Gitlab::ErrorTracking.track_exception(e, cluster_id: cluster_id)
- { status: :unknown_failure }
+ { status: :unknown_failure, connection_error: :unknown_error }
end
# KubeClient uses the same error class
diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb
index dc13a614551..28810dc4453 100644
--- a/lib/gitlab/kubernetes/network_policy.rb
+++ b/lib/gitlab/kubernetes/network_policy.rb
@@ -3,19 +3,27 @@
module Gitlab
module Kubernetes
class NetworkPolicy
- DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by'
+ include NetworkPolicyCommon
+ extend ::Gitlab::Utils::Override
- def initialize(name:, namespace:, pod_selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil)
+ def initialize(name:, namespace:, selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil)
@name = name
@namespace = namespace
@labels = labels
@creation_timestamp = creation_timestamp
- @pod_selector = pod_selector
+ @selector = selector
@policy_types = policy_types
@ingress = ingress
@egress = egress
end
+ def generate
+ ::Kubeclient::Resource.new.tap do |resource|
+ resource.metadata = metadata
+ resource.spec = spec
+ end
+ end
+
def self.from_yaml(manifest)
return unless manifest
@@ -28,7 +36,7 @@ module Gitlab
name: metadata[:name],
namespace: metadata[:namespace],
labels: metadata[:labels],
- pod_selector: spec[:podSelector],
+ selector: spec[:podSelector],
policy_types: spec[:policyTypes],
ingress: spec[:ingress],
egress: spec[:egress]
@@ -48,81 +56,30 @@ module Gitlab
namespace: metadata[:namespace],
labels: metadata[:labels]&.to_h,
creation_timestamp: metadata[:creationTimestamp],
- pod_selector: spec[:podSelector],
+ selector: spec[:podSelector],
policy_types: spec[:policyTypes],
ingress: spec[:ingress],
egress: spec[:egress]
)
end
- def generate
- ::Kubeclient::Resource.new.tap do |resource|
- resource.metadata = metadata
- resource.spec = spec
- end
- end
-
- def as_json(opts = nil)
- {
- name: name,
- namespace: namespace,
- creation_timestamp: creation_timestamp,
- manifest: manifest,
- is_autodevops: autodevops?,
- is_enabled: enabled?
- }
- end
-
- def autodevops?
- return false unless labels
-
- !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-')
- end
-
- # podSelector selects pods that should be targeted by this
- # policy. We can narrow selection by requiring this policy to
- # match our custom labels. Since DISABLED_BY label will not be
- # on any pod a policy will be effectively disabled.
- def enabled?
- return true unless pod_selector&.key?(:matchLabels)
-
- !pod_selector[:matchLabels]&.key?(DISABLED_BY_LABEL)
- end
-
- def enable
- return if enabled?
-
- pod_selector[:matchLabels].delete(DISABLED_BY_LABEL)
- end
-
- def disable
- @pod_selector ||= {}
- pod_selector[:matchLabels] ||= {}
- pod_selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab')
- end
-
private
- attr_reader :name, :namespace, :labels, :creation_timestamp, :pod_selector, :policy_types, :ingress, :egress
+ attr_reader :name, :namespace, :labels, :creation_timestamp, :policy_types, :ingress, :egress
- def metadata
- meta = { name: name, namespace: namespace }
- meta[:labels] = labels if labels
- meta
+ def selector
+ @selector ||= {}
end
+ override :spec
def spec
{
- podSelector: pod_selector,
+ podSelector: selector,
policyTypes: policy_types,
ingress: ingress,
egress: egress
}
end
-
- def manifest
- YAML.dump({ metadata: metadata, spec: spec }.deep_stringify_keys)
- end
end
end
end
diff --git a/lib/gitlab/kubernetes/network_policy_common.rb b/lib/gitlab/kubernetes/network_policy_common.rb
new file mode 100644
index 00000000000..3b6e46d21ef
--- /dev/null
+++ b/lib/gitlab/kubernetes/network_policy_common.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module NetworkPolicyCommon
+ DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by'
+
+ def as_json(opts = nil)
+ {
+ name: name,
+ namespace: namespace,
+ creation_timestamp: creation_timestamp,
+ manifest: manifest,
+ is_autodevops: autodevops?,
+ is_enabled: enabled?
+ }
+ end
+
+ def autodevops?
+ return false unless labels
+
+ !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-')
+ end
+
+ # selector selects pods that should be targeted by this
+ # policy. It can represent podSelector, nodeSelector or
+ # endpointSelector We can narrow selection by requiring
+ # this policy to match our custom labels. Since DISABLED_BY
+ # label will not be on any pod a policy will be effectively disabled.
+ def enabled?
+ return true unless selector&.key?(:matchLabels)
+
+ !selector[:matchLabels]&.key?(DISABLED_BY_LABEL)
+ end
+
+ def enable
+ return if enabled?
+
+ selector[:matchLabels].delete(DISABLED_BY_LABEL)
+ end
+
+ def disable
+ selector[:matchLabels] ||= {}
+ selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab')
+ end
+
+ private
+
+ def metadata
+ meta = { name: name, namespace: namespace }
+ meta[:labels] = labels if labels
+ meta[:resourceVersion] = resource_version if defined?(resource_version)
+ meta
+ end
+
+ def spec
+ raise NotImplementedError
+ end
+
+ def manifest
+ YAML.dump({ metadata: metadata, spec: spec }.deep_stringify_keys)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/node.rb b/lib/gitlab/kubernetes/node.rb
index bd765ef3852..d516bdde6f6 100644
--- a/lib/gitlab/kubernetes/node.rb
+++ b/lib/gitlab/kubernetes/node.rb
@@ -8,22 +8,29 @@ module Gitlab
end
def all
- nodes.map do |node|
- attributes = node(node)
- attributes.merge(node_metrics(node))
- end
+ {
+ nodes: metadata.presence,
+ node_connection_error: nodes_from_cluster[:connection_error],
+ metrics_connection_error: nodes_metrics_from_cluster[:connection_error]
+ }.compact
end
private
attr_reader :cluster
+ def metadata
+ nodes.map do |node|
+ base_data(node).merge(node_metrics(node))
+ end
+ end
+
def nodes_from_cluster
- graceful_request { cluster.kubeclient.get_nodes }
+ @nodes_from_cluster ||= graceful_request { cluster.kubeclient.get_nodes }
end
def nodes_metrics_from_cluster
- graceful_request { cluster.kubeclient.metrics_client.get_nodes }
+ @nodes_metrics_from_cluster ||= graceful_request { cluster.kubeclient.metrics_client.get_nodes }
end
def nodes
@@ -44,7 +51,7 @@ module Gitlab
::Gitlab::Kubernetes::KubeClient.graceful_request(cluster.id, &block)
end
- def node(node)
+ def base_data(node)
{
'metadata' => {
'name' => node.metadata.name
diff --git a/lib/gitlab/manifest_import/project_creator.rb b/lib/gitlab/manifest_import/project_creator.rb
index 837d65e5f7c..6637cbb9cc8 100644
--- a/lib/gitlab/manifest_import/project_creator.rb
+++ b/lib/gitlab/manifest_import/project_creator.rb
@@ -18,6 +18,7 @@ module Gitlab
params = {
import_url: repository[:url],
+ import_source: repository[:url],
import_type: 'manifest',
namespace_id: group.id,
path: project_path,
diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb
index ac3492dbe33..da3b597a74e 100644
--- a/lib/gitlab/markdown_cache.rb
+++ b/lib/gitlab/markdown_cache.rb
@@ -3,7 +3,7 @@
module Gitlab
module MarkdownCache
# Increment this number every time the renderer changes its output
- CACHE_COMMONMARK_VERSION = 24
+ CACHE_COMMONMARK_VERSION = 25
CACHE_COMMONMARK_VERSION_START = 10
BaseError = Class.new(StandardError)
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index 5fed3d38d7c..7bd55cce363 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -3,7 +3,6 @@
module Gitlab
module Metrics
include Gitlab::Metrics::Prometheus
- include Gitlab::Metrics::Methods
EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze
@@ -81,25 +80,16 @@ module Gitlab
real_time = (real_stop - real_start)
cpu_time = cpu_stop - cpu_start
- real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do
+ trans.observe("gitlab_#{name}_real_duration_seconds".to_sym, real_time) do
docstring "Measure #{name}"
- base_labels Transaction::BASE_LABELS
buckets EXECUTION_MEASUREMENT_BUCKETS
end
- real_duration_seconds.observe(trans.labels, real_time)
-
- cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do
+ trans.observe("gitlab_#{name}_cpu_duration_seconds".to_sym, cpu_time) do
docstring "Measure #{name}"
- base_labels Transaction::BASE_LABELS
buckets EXECUTION_MEASUREMENT_BUCKETS
with_feature "prometheus_metrics_measure_#{name}_cpu_duration"
end
- cpu_duration_seconds.observe(trans.labels, cpu_time)
-
- trans.increment("#{name}_real_time", real_time.in_milliseconds, false)
- trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false)
- trans.increment("#{name}_call_count", 1, false)
retval
end
diff --git a/lib/gitlab/metrics/dashboard/cache.rb b/lib/gitlab/metrics/dashboard/cache.rb
index a9ccf0fea9b..54b5250d209 100644
--- a/lib/gitlab/metrics/dashboard/cache.rb
+++ b/lib/gitlab/metrics/dashboard/cache.rb
@@ -9,34 +9,53 @@ module Gitlab
CACHE_KEYS = 'all_cached_metric_dashboards'
class << self
- # Stores a dashboard in the cache, documenting the key
- # so the cached can be cleared in bulk at another time.
- def fetch(key)
- register_key(key)
+ # This class method (Gitlab::Metrics::Dashboard::Cache.fetch) can be used
+ # when the key does not need to be deleted by `delete_all!`.
+ # For example, out of the box dashboard caches do not need to be deleted.
+ delegate :fetch, to: :"Rails.cache"
- Rails.cache.fetch(key) { yield }
- end
+ alias_method :for, :new
+ end
+
+ def initialize(project)
+ @project = project
+ end
+
+ # Stores a dashboard in the cache, documenting the key
+ # so the cache can be cleared in bulk at another time.
+ def fetch(key)
+ register_key(key)
+
+ Rails.cache.fetch(key) { yield }
+ end
- # Resets all dashboard caches, such that all
- # dashboard content will be loaded from source on
- # subsequent dashboard calls.
- def delete_all!
- all_keys.each { |key| Rails.cache.delete(key) }
+ # Resets all dashboard caches, such that all
+ # dashboard content will be loaded from source on
+ # subsequent dashboard calls.
+ def delete_all!
+ all_keys.each { |key| Rails.cache.delete(key) }
- Rails.cache.delete(CACHE_KEYS)
- end
+ Rails.cache.delete(catalog_key)
+ end
- private
+ private
- def register_key(key)
- new_keys = all_keys.add(key).to_a.join('|')
+ def register_key(key)
+ new_keys = all_keys.add(key).to_a.join('|')
- Rails.cache.write(CACHE_KEYS, new_keys)
- end
+ Rails.cache.write(catalog_key, new_keys)
+ end
+
+ def all_keys
+ keys = Rails.cache.read(catalog_key)&.split('|')
+ Set.new(keys)
+ end
- def all_keys
- Set.new(Rails.cache.read(CACHE_KEYS)&.split('|'))
- end
+ # One key to store them all...
+ # This key is used to store the names of all the keys that contain this
+ # project's dashboards.
+ def catalog_key
+ "#{CACHE_KEYS}_#{@project.id}"
end
end
end
diff --git a/lib/gitlab/metrics/dashboard/defaults.rb b/lib/gitlab/metrics/dashboard/defaults.rb
index 3c39a7c6911..6a5f98a18c8 100644
--- a/lib/gitlab/metrics/dashboard/defaults.rb
+++ b/lib/gitlab/metrics/dashboard/defaults.rb
@@ -7,7 +7,6 @@ module Gitlab
module Dashboard
module Defaults
DEFAULT_PANEL_TYPE = 'area-chart'
- DEFAULT_PANEL_WEIGHT = 0
end
end
end
diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb
index 5e2d78e10a4..2c4793eb75f 100644
--- a/lib/gitlab/metrics/dashboard/finder.rb
+++ b/lib/gitlab/metrics/dashboard/finder.rb
@@ -14,10 +14,7 @@ module Gitlab
::Metrics::Dashboard::SelfMonitoringDashboardService,
# This dashboard is displayed on the K8s cluster settings health page.
- ::Metrics::Dashboard::ClusterDashboardService,
-
- # This dashboard is not yet ready for the world.
- ::Metrics::Dashboard::PodDashboardService
+ ::Metrics::Dashboard::ClusterDashboardService
].freeze
class << self
@@ -72,17 +69,11 @@ module Gitlab
# display_name: String,
# default: Boolean }]
def find_all_paths(project)
- project.repository.metrics_dashboard_paths
- end
-
- # Summary of all known dashboards. Used to populate repo cache.
- # Prefer #find_all_paths.
- def find_all_paths_from_source(project)
- Gitlab::Metrics::Dashboard::Cache.delete_all!
-
- user_facing_dashboard_services(project).flat_map do |service|
+ dashboards = user_facing_dashboard_services(project).flat_map do |service|
service.all_dashboard_paths(project)
end
+
+ Gitlab::Utils.stable_sort_by(dashboards) { |dashboard| dashboard[:display_name].downcase }
end
private
diff --git a/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb b/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb
new file mode 100644
index 00000000000..8b791e110ba
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# Provides methods to list and read dashboard yaml files from a project's repository.
+module Gitlab
+ module Metrics
+ module Dashboard
+ class RepoDashboardFinder
+ DASHBOARD_ROOT = ".gitlab/dashboards"
+ DASHBOARD_EXTENSION = '.yml'
+
+ class << self
+ # Returns list of all user-defined dashboard paths. Used to populate
+ # Repository model cache (Repository#user_defined_metrics_dashboard_paths).
+ # Also deletes all dashboard cache entries.
+ # @return [Array] ex) ['.gitlab/dashboards/dashboard1.yml']
+ def list_dashboards(project)
+ Gitlab::Metrics::Dashboard::Cache.for(project).delete_all!
+
+ file_finder(project).list_files_for(DASHBOARD_ROOT)
+ end
+
+ # Reads the given dashboard from repository, and returns the content as a string.
+ # @return [String]
+ def read_dashboard(project, dashboard_path)
+ file_finder(project).read(dashboard_path)
+ end
+
+ private
+
+ def file_finder(project)
+ Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, DASHBOARD_EXTENSION)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb
index 3444a01bccd..3b49eb1c837 100644
--- a/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb
+++ b/lib/gitlab/metrics/dashboard/stages/custom_metrics_inserter.rb
@@ -9,7 +9,10 @@ module Gitlab
# config. If there are no project-specific metrics,
# this will have no effect.
def transform!
- PrometheusMetricsFinder.new(project: project).execute.each do |project_metric|
+ custom_metrics = PrometheusMetricsFinder.new(project: project, ordered: true).execute
+ custom_metrics = Gitlab::Utils.stable_sort_by(custom_metrics) { |metric| -metric.priority }
+
+ custom_metrics.each do |project_metric|
group = find_or_create_panel_group(dashboard[:panel_groups], project_metric)
panel = find_or_create_panel(group[:panels], project_metric)
find_or_create_metric(panel[:metrics], project_metric)
@@ -83,7 +86,6 @@ module Gitlab
def new_panel_group(metric)
{
group: metric.group_title,
- priority: metric.priority,
panels: []
}
end
diff --git a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb
index c48a7ff25a5..dd85bd0beb1 100644
--- a/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb
+++ b/lib/gitlab/metrics/dashboard/stages/metric_endpoint_inserter.rb
@@ -45,7 +45,9 @@ module Gitlab
raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query
- query
+ # We need to remove any newlines since our UrlBlocker does not allow
+ # multiline URLs.
+ query.to_s.squish
end
end
end
diff --git a/lib/gitlab/metrics/dashboard/stages/sorter.rb b/lib/gitlab/metrics/dashboard/stages/sorter.rb
deleted file mode 100644
index 882211e1441..00000000000
--- a/lib/gitlab/metrics/dashboard/stages/sorter.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Metrics
- module Dashboard
- module Stages
- class Sorter < BaseStage
- def transform!
- missing_panel_groups! unless dashboard[:panel_groups].is_a? Array
-
- sort_groups!
- sort_panels!
- end
-
- private
-
- # Sorts the groups in the dashboard by the :priority key
- def sort_groups!
- dashboard[:panel_groups] = Gitlab::Utils.stable_sort_by(dashboard[:panel_groups]) { |group| -group[:priority].to_i }
- end
-
- # Sorts the panels in the dashboard by the :weight key
- def sort_panels!
- dashboard[:panel_groups].each do |group|
- missing_panels! unless group[:panels].is_a? Array
-
- group[:panels] = Gitlab::Utils.stable_sort_by(group[:panels]) { |panel| -panel[:weight].to_i }
- end
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb b/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb
new file mode 100644
index 00000000000..71da779d16c
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/track_panel_type.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Dashboard
+ module Stages
+ class TrackPanelType < BaseStage
+ def transform!
+ for_panel_groups do |panel_group|
+ for_panels_in(panel_group) do |panel|
+ track_panel_type(panel)
+ end
+ end
+ end
+
+ private
+
+ def track_panel_type(panel)
+ panel_type = panel[:type]
+
+ Gitlab::Tracking.event('MetricsDashboard::Chart', 'chart_rendered', label: panel_type)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb
index 10a2f3c2397..160ecfb85c9 100644
--- a/lib/gitlab/metrics/dashboard/url.rb
+++ b/lib/gitlab/metrics/dashboard/url.rb
@@ -43,6 +43,39 @@ module Gitlab
end
end
+ # Matches dashboard urls for a metric chart embed
+ # for cluster metrics
+ #
+ # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB)
+ def clusters_regex
+ strong_memoize(:clusters_regex) do
+ regex_for_project_metrics(
+ %r{
+ /clusters
+ /(?<cluster_id>\d+)
+ /?
+ }x
+ )
+ end
+ end
+
+ # Matches dashboard urls for a metric chart embed
+ # for a specifc firing GitLab alert
+ #
+ # EX - https://<host>/<namespace>/<project>/prometheus/alerts/<alert_id>/metrics_dashboard
+ def alert_regex
+ strong_memoize(:alert_regex) do
+ regex_for_project_metrics(
+ %r{
+ /prometheus
+ /alerts
+ /(?<alert>\d+)
+ /metrics_dashboard
+ }x
+ )
+ end
+ end
+
# Parses query params out from full url string into hash.
#
# Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group'
@@ -60,22 +93,6 @@ module Gitlab
Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args)
end
- # Matches dashboard urls for a metric chart embed
- # for cluster metrics
- #
- # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB)
- def clusters_regex
- strong_memoize(:clusters_regex) do
- regex_for_project_metrics(
- %r{
- /clusters
- /(?<cluster_id>\d+)
- /?
- }x
- )
- end
- end
-
private
def regex_for_project_metrics(path_suffix_pattern)
@@ -92,16 +109,18 @@ module Gitlab
end
def gitlab_host_pattern
- Regexp.escape(Gitlab.config.gitlab.url)
+ Regexp.escape(gitlab_domain)
end
def project_path_pattern
"\/#{Project.reference_pattern}"
end
+
+ def gitlab_domain
+ Gitlab.config.gitlab.url
+ end
end
end
end
end
end
-
-Gitlab::Metrics::Dashboard::Url.extend_if_ee('::EE::Gitlab::Metrics::Dashboard::Url')
diff --git a/lib/gitlab/metrics/dashboard/validator.rb b/lib/gitlab/metrics/dashboard/validator.rb
new file mode 100644
index 00000000000..8edd9c397f9
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Dashboard
+ module Validator
+ DASHBOARD_SCHEMA_PATH = 'lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json'.freeze
+
+ class << self
+ def validate(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil)
+ errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project)
+ errors.empty?
+ end
+
+ def validate!(content, schema_path = DASHBOARD_SCHEMA_PATH, dashboard_path: nil, project: nil)
+ errors = _validate(content, schema_path, dashboard_path: dashboard_path, project: project)
+ errors.empty? || raise(errors.first)
+ end
+
+ private
+
+ def _validate(content, schema_path, dashboard_path: nil, project: nil)
+ client = Validator::Client.new(content, schema_path, dashboard_path: dashboard_path, project: project)
+ client.execute
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb
new file mode 100644
index 00000000000..c63415abcfc
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/client.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Dashboard
+ module Validator
+ class Client
+ # @param content [Hash] Representing a raw, unprocessed
+ # dashboard object
+ # @param schema_path [String] Representing path to dashboard schema file
+ # @param dashboard_path[String] Representing path to dashboard content file
+ # @param project [Project] Project to validate dashboard against
+ def initialize(content, schema_path, dashboard_path: nil, project: nil)
+ @content = content
+ @schema_path = schema_path
+ @dashboard_path = dashboard_path
+ @project = project
+ end
+
+ def execute
+ errors = validate_against_schema
+ errors += post_schema_validator.validate
+
+ errors.compact
+ end
+
+ private
+
+ attr_reader :content, :schema_path, :project, :dashboard_path
+
+ def custom_formats
+ @custom_formats ||= CustomFormats.new
+ end
+
+ def post_schema_validator
+ PostSchemaValidator.new(
+ project: project,
+ metric_ids: custom_formats.metric_ids_cache,
+ dashboard_path: dashboard_path
+ )
+ end
+
+ def schemer
+ @schemer ||= ::JSONSchemer.schema(Pathname.new(schema_path), formats: custom_formats.format_handlers)
+ end
+
+ def validate_against_schema
+ schemer.validate(content).map do |error|
+ Errors::SchemaValidationError.new(error)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/dashboard/validator/custom_formats.rb b/lib/gitlab/metrics/dashboard/validator/custom_formats.rb
new file mode 100644
index 00000000000..485e80ad1b7
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/custom_formats.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Dashboard
+ module Validator
+ class CustomFormats
+ def format_handlers
+ # Key is custom JSON Schema format name. Value is a proc that takes data and schema and handles
+ # validations.
+ @format_handlers ||= {
+ "add_to_metric_id_cache" => ->(data, schema) { metric_ids_cache << data }
+ }
+ end
+
+ def metric_ids_cache
+ @metric_ids_cache ||= []
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/dashboard/validator/errors.rb b/lib/gitlab/metrics/dashboard/validator/errors.rb
new file mode 100644
index 00000000000..0f6e687d291
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/errors.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Dashboard
+ module Validator
+ module Errors
+ InvalidDashboardError = Class.new(StandardError)
+
+ class SchemaValidationError < InvalidDashboardError
+ def initialize(error = {})
+ super(error_message(error))
+ end
+
+ private
+
+ def error_message(error)
+ if error.is_a?(Hash) && error.present?
+ pretty(error)
+ else
+ "Dashboard failed schema validation"
+ end
+ end
+
+ # based on https://github.com/davishmcclurg/json_schemer/blob/master/lib/json_schemer/errors.rb
+ # with addition ability to translate error messages
+ def pretty(error)
+ data, data_pointer, type, schema = error.values_at('data', 'data_pointer', 'type', 'schema')
+ location = data_pointer.empty? ? 'root' : data_pointer
+
+ case type
+ when 'required'
+ keys = error.fetch('details').fetch('missing_keys').join(', ')
+ _("%{location} is missing required keys: %{keys}") % { location: location, keys: keys }
+ when 'null', 'string', 'boolean', 'integer', 'number', 'array', 'object'
+ _("'%{data}' at %{location} is not of type: %{type}") % { data: data, location: location, type: type }
+ when 'pattern'
+ _("'%{data}' at %{location} does not match pattern: %{pattern}") % { data: data, location: location, pattern: schema.fetch('pattern') }
+ when 'format'
+ _("'%{data}' at %{location} does not match format: %{format}") % { data: data, location: location, format: schema.fetch('format') }
+ when 'const'
+ _("'%{data}' at %{location} is not: %{const}") % { data: data, location: location, const: schema.fetch('const').inspect }
+ when 'enum'
+ _("'%{data}' at %{location} is not one of: %{enum}") % { data: data, location: location, enum: schema.fetch('enum') }
+ else
+ _("'%{data}' at %{location} is invalid: error_type=%{type}") % { data: data, location: location, type: type }
+ end
+ end
+ end
+
+ class DuplicateMetricIds < InvalidDashboardError
+ def initialize
+ super(_("metric_id must be unique across a project"))
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb b/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb
new file mode 100644
index 00000000000..73bfc5a6294
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/post_schema_validator.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Dashboard
+ module Validator
+ class PostSchemaValidator
+ def initialize(metric_ids:, project: nil, dashboard_path: nil)
+ @metric_ids = metric_ids
+ @project = project
+ @dashboard_path = dashboard_path
+ end
+
+ def validate
+ errors = []
+ errors << uniq_metric_ids
+ errors.compact
+ end
+
+ private
+
+ attr_reader :project, :metric_ids, :dashboard_path
+
+ def uniq_metric_ids
+ return Validator::Errors::DuplicateMetricIds.new if metric_ids.uniq!
+
+ uniq_metric_ids_across_project if project.present? || dashboard_path.present?
+ end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def uniq_metric_ids_across_project
+ return ArgumentError.new(_('Both project and dashboard_path are required')) unless
+ dashboard_path.present? && project.present?
+
+ # If PrometheusMetric identifier is not unique across project and dashboard_path,
+ # we need to error because we don't know if the user is trying to create a new metric
+ # or update an existing one.
+ identifier_on_other_dashboard = PrometheusMetric.where(
+ project: project,
+ identifier: metric_ids
+ ).where.not(
+ dashboard_path: dashboard_path
+ ).exists?
+
+ Validator::Errors::DuplicateMetricIds.new if identifier_on_other_dashboard
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/axis.json b/lib/gitlab/metrics/dashboard/validator/schemas/axis.json
new file mode 100644
index 00000000000..54334022426
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/schemas/axis.json
@@ -0,0 +1,14 @@
+{
+ "type": "object",
+ "properties": {
+ "name": { "type": "string" },
+ "format": {
+ "type": "string",
+ "default": "engineering"
+ },
+ "precision": {
+ "type": "number",
+ "default": 2
+ }
+ }
+}
diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json b/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json
new file mode 100644
index 00000000000..313f03be7dc
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/schemas/dashboard.json
@@ -0,0 +1,18 @@
+{
+ "type": "object",
+ "required": ["dashboard", "panel_groups"],
+ "properties": {
+ "dashboard": { "type": "string" },
+ "panel_groups": {
+ "type": "array",
+ "items": { "$ref": "./panel_group.json" }
+ },
+ "templating": {
+ "$ref": "./templating.json"
+ },
+ "links": {
+ "type": "array",
+ "items": { "$ref": "./link.json" }
+ }
+ }
+}
diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/link.json b/lib/gitlab/metrics/dashboard/validator/schemas/link.json
new file mode 100644
index 00000000000..4ea7b5dd324
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/schemas/link.json
@@ -0,0 +1,12 @@
+{
+ "type": "object",
+ "required": ["url"],
+ "properties": {
+ "url": { "type": "string" },
+ "title": { "type": "string" },
+ "type": {
+ "type": "string",
+ "enum": ["grafana"]
+ }
+ }
+}
diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/metric.json b/lib/gitlab/metrics/dashboard/validator/schemas/metric.json
new file mode 100644
index 00000000000..13831b77e3e
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/schemas/metric.json
@@ -0,0 +1,16 @@
+{
+ "type": "object",
+ "required": ["unit"],
+ "oneOf": [{ "required": ["query"] }, { "required": ["query_range"] }],
+ "properties": {
+ "id": {
+ "type": "string",
+ "format": "add_to_metric_id_cache"
+ },
+ "unit": { "type": "string" },
+ "label": { "type": "string" },
+ "query": { "type": ["string", "number"] },
+ "query_range": { "type": ["string", "number"] },
+ "step": { "type": "number" }
+ }
+}
diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json
new file mode 100644
index 00000000000..011eef53e40
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/schemas/panel.json
@@ -0,0 +1,24 @@
+{
+ "type": "object",
+ "required": ["title", "metrics"],
+ "properties": {
+ "type": {
+ "type": "string",
+ "enum": ["area-chart", "anomaly-chart", "bar", "column", "stacked-column", "single-stat", "heatmap"],
+ "default": "area-chart"
+ },
+ "title": { "type": "string" },
+ "y_label": { "type": "string" },
+ "y_axis": { "$ref": "./axis.json" },
+ "max_value": { "type": "number" },
+ "weight": { "type": "number" },
+ "metrics": {
+ "type": "array",
+ "items": { "$ref": "./metric.json" }
+ },
+ "links": {
+ "type": "array",
+ "items": { "$ref": "./link.json" }
+ }
+ }
+}
diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json b/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json
new file mode 100644
index 00000000000..1306fc475db
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/schemas/panel_group.json
@@ -0,0 +1,12 @@
+{
+ "type": "object",
+ "required": ["group", "panels"],
+ "properties": {
+ "group": { "type": "string" },
+ "priority": { "type": "number" },
+ "panels": {
+ "type": "array",
+ "items": { "$ref": "./panel.json" }
+ }
+ }
+}
diff --git a/lib/gitlab/metrics/dashboard/validator/schemas/templating.json b/lib/gitlab/metrics/dashboard/validator/schemas/templating.json
new file mode 100644
index 00000000000..6f8664c89af
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/validator/schemas/templating.json
@@ -0,0 +1,7 @@
+{
+ "type": "object",
+ "required": ["variables"],
+ "properties": {
+ "variables": { "type": "object" }
+ }
+}
diff --git a/lib/gitlab/metrics/elasticsearch_rack_middleware.rb b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb
index 6830eed68d5..870ab148004 100644
--- a/lib/gitlab/metrics/elasticsearch_rack_middleware.rb
+++ b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb
@@ -4,18 +4,10 @@ module Gitlab
module Metrics
# Rack middleware for tracking Elasticsearch metrics from Grape and Web requests.
class ElasticsearchRackMiddleware
- HISTOGRAM_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60].freeze
+ HISTOGRAM_BUCKETS = [0.1, 0.5, 1, 10, 50].freeze
def initialize(app)
@app = app
-
- @requests_total_counter = Gitlab::Metrics.counter(:http_elasticsearch_requests_total,
- 'Amount of calls to Elasticsearch servers during web requests',
- Gitlab::Metrics::Transaction::BASE_LABELS)
- @requests_duration_histogram = Gitlab::Metrics.histogram(:http_elasticsearch_requests_duration_seconds,
- 'Query time for Elasticsearch servers during web requests',
- Gitlab::Metrics::Transaction::BASE_LABELS,
- HISTOGRAM_BUCKETS)
end
def call(env)
@@ -29,12 +21,19 @@ module Gitlab
private
def record_metrics(transaction)
- labels = transaction.labels
query_time = ::Gitlab::Instrumentation::ElasticsearchTransport.query_time
request_count = ::Gitlab::Instrumentation::ElasticsearchTransport.get_request_count
- @requests_total_counter.increment(labels, request_count)
- @requests_duration_histogram.observe(labels, query_time)
+ return unless request_count > 0
+
+ transaction.increment(:http_elasticsearch_requests_total, request_count) do
+ docstring 'Amount of calls to Elasticsearch servers during web requests'
+ end
+
+ transaction.observe(:http_elasticsearch_requests_duration_seconds, query_time) do
+ docstring 'Query time for Elasticsearch servers during web requests'
+ buckets HISTOGRAM_BUCKETS
+ end
end
end
end
diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
index fbeda3b75e0..c6b0a0c5e76 100644
--- a/lib/gitlab/metrics/method_call.rb
+++ b/lib/gitlab/metrics/method_call.rb
@@ -4,16 +4,7 @@ module Gitlab
module Metrics
# Class for tracking timing information about method calls
class MethodCall
- include Gitlab::Metrics::Methods
- BASE_LABELS = { module: nil, method: nil }.freeze
- attr_reader :real_time, :cpu_time, :call_count, :labels
-
- define_histogram :gitlab_method_call_duration_seconds do
- docstring 'Method calls real duration'
- base_labels Transaction::BASE_LABELS.merge(BASE_LABELS)
- buckets [0.01, 0.05, 0.1, 0.5, 1]
- with_feature :prometheus_metrics_method_instrumentation
- end
+ attr_reader :real_time, :cpu_time, :call_count
# name - The full name of the method (including namespace) such as
# `User#sign_in`.
@@ -42,8 +33,14 @@ module Gitlab
@cpu_time += cpu_time
@call_count += 1
- if above_threshold?
- self.class.gitlab_method_call_duration_seconds.observe(@transaction.labels.merge(labels), real_time)
+ if above_threshold? && transaction
+ label_keys = labels.keys
+ transaction.observe(:gitlab_method_call_duration_seconds, real_time, labels) do
+ docstring 'Method calls real duration'
+ label_keys label_keys
+ buckets [0.01, 0.05, 0.1, 0.5, 1]
+ with_feature :prometheus_metrics_method_instrumentation
+ end
end
retval
@@ -54,6 +51,10 @@ module Gitlab
def above_threshold?
real_time.in_milliseconds >= ::Gitlab::Metrics.method_call_threshold
end
+
+ private
+
+ attr_reader :labels, :transaction
end
end
end
diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb
index 83a7b925392..2b5d1c710f6 100644
--- a/lib/gitlab/metrics/methods.rb
+++ b/lib/gitlab/metrics/methods.rb
@@ -69,62 +69,6 @@ module Gitlab
raise ArgumentError, "uknown metric type #{type}"
end
end
-
- # Fetch and/or initialize counter metric
- # @param [Symbol] name
- # @param [Hash] opts
- def fetch_counter(name, opts = {}, &block)
- fetch_metric(:counter, name, opts, &block)
- end
-
- # Fetch and/or initialize gauge metric
- # @param [Symbol] name
- # @param [Hash] opts
- def fetch_gauge(name, opts = {}, &block)
- fetch_metric(:gauge, name, opts, &block)
- end
-
- # Fetch and/or initialize histogram metric
- # @param [Symbol] name
- # @param [Hash] opts
- def fetch_histogram(name, opts = {}, &block)
- fetch_metric(:histogram, name, opts, &block)
- end
-
- # Fetch and/or initialize summary metric
- # @param [Symbol] name
- # @param [Hash] opts
- def fetch_summary(name, opts = {}, &block)
- fetch_metric(:summary, name, opts, &block)
- end
-
- # Define metric accessor method for a Counter
- # @param [Symbol] name
- # @param [Hash] opts
- def define_counter(name, opts = {}, &block)
- define_metric(:counter, name, opts, &block)
- end
-
- # Define metric accessor method for a Gauge
- # @param [Symbol] name
- # @param [Hash] opts
- def define_gauge(name, opts = {}, &block)
- define_metric(:gauge, name, opts, &block)
- end
-
- # Define metric accessor method for a Histogram
- # @param [Symbol] name
- # @param [Hash] opts
- def define_histogram(name, opts = {}, &block)
- define_metric(:histogram, name, opts, &block)
- end
-
- # Define metric accessor method for a Summary
- # @param [Symbol] name
- # @param [Hash] opts
- def define_summary(name, opts = {}, &block)
- define_metric(:summary, name, opts, &block)
- end
end
end
end
diff --git a/lib/gitlab/metrics/methods/metric_options.rb b/lib/gitlab/metrics/methods/metric_options.rb
index 8e6ceb74c09..1e488df3e99 100644
--- a/lib/gitlab/metrics/methods/metric_options.rb
+++ b/lib/gitlab/metrics/methods/metric_options.rb
@@ -4,14 +4,12 @@ module Gitlab
module Metrics
module Methods
class MetricOptions
- SMALL_NETWORK_BUCKETS = [0.005, 0.01, 0.1, 1, 10].freeze
-
def initialize(options = {})
@multiprocess_mode = options[:multiprocess_mode] || :all
- @buckets = options[:buckets] || SMALL_NETWORK_BUCKETS
- @base_labels = options[:base_labels] || {}
+ @buckets = options[:buckets] || ::Prometheus::Client::Histogram::DEFAULT_BUCKETS
@docstring = options[:docstring]
@with_feature = options[:with_feature]
+ @label_keys = options[:label_keys] || []
end
# Documentation describing metric in metrics endpoint '/-/metrics'
@@ -40,12 +38,21 @@ module Gitlab
end
# Base labels are merged with per metric labels
- def base_labels(base_labels = nil)
- @base_labels = base_labels unless base_labels.nil?
+ def base_labels
+ @base_labels ||= @label_keys.product([nil]).to_h
@base_labels
end
+ def label_keys(label_keys = nil)
+ unless label_keys.nil?
+ @label_keys = label_keys
+ @base_labels = nil
+ end
+
+ @label_keys
+ end
+
# Use feature toggle to control whether certain metric is enabled/disabled
def with_feature(name = nil)
@with_feature = name unless name.nil?
@@ -55,6 +62,7 @@ module Gitlab
def evaluate(&block)
instance_eval(&block) if block_given?
+
self
end
end
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index c6a0457ffe5..a6884ea6983 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -10,8 +10,7 @@ module Gitlab
# env - A Hash containing Rack environment details.
def call(env)
- trans = transaction_from_env(env)
- retval = nil
+ trans = WebTransaction.new(env)
begin
retval = trans.run { @app.call(env) }
@@ -24,21 +23,6 @@ module Gitlab
retval
end
-
- def transaction_from_env(env)
- trans = WebTransaction.new(env)
-
- trans.set(:request_uri, filtered_path(env), false)
- trans.set(:request_method, env['REQUEST_METHOD'], false)
-
- trans
- end
-
- private
-
- def filtered_path(env)
- ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI']
- end
end
end
end
diff --git a/lib/gitlab/metrics/redis_rack_middleware.rb b/lib/gitlab/metrics/redis_rack_middleware.rb
deleted file mode 100644
index f0f99c5f45d..00000000000
--- a/lib/gitlab/metrics/redis_rack_middleware.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Metrics
- # Rack middleware for tracking Redis metrics from Grape and Web requests.
- class RedisRackMiddleware
- def initialize(app)
- @app = app
-
- @requests_total_counter = Gitlab::Metrics.counter(:http_redis_requests_total,
- 'Amount of calls to Redis servers during web requests',
- Gitlab::Metrics::Transaction::BASE_LABELS)
- @requests_duration_histogram = Gitlab::Metrics.histogram(:http_redis_requests_duration_seconds,
- 'Query time for Redis servers during web requests',
- Gitlab::Metrics::Transaction::BASE_LABELS,
- Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS)
- end
-
- def call(env)
- transaction = Gitlab::Metrics.current_transaction
-
- @app.call(env)
- ensure
- record_metrics(transaction)
- end
-
- private
-
- def record_metrics(transaction)
- labels = transaction.labels
- query_time = Gitlab::Instrumentation::Redis.query_time
- request_count = Gitlab::Instrumentation::Redis.get_request_count
-
- @requests_total_counter.increment(labels, request_count)
- @requests_duration_histogram.observe(labels, query_time)
- end
- end
- end
-end
diff --git a/lib/gitlab/metrics/samplers/threads_sampler.rb b/lib/gitlab/metrics/samplers/threads_sampler.rb
new file mode 100644
index 00000000000..05acef7ce0c
--- /dev/null
+++ b/lib/gitlab/metrics/samplers/threads_sampler.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Samplers
+ class ThreadsSampler < BaseSampler
+ SAMPLING_INTERVAL_SECONDS = 5
+ KNOWN_PUMA_THREAD_NAMES = ['puma worker check pipe', 'puma server',
+ 'puma threadpool reaper', 'puma threadpool trimmer',
+ 'puma worker check pipe', 'puma stat payload'].freeze
+
+ SIDEKIQ_WORKER_THREAD_NAME = 'sidekiq_worker_thread'
+
+ METRIC_PREFIX = "gitlab_ruby_threads_"
+
+ METRIC_DESCRIPTIONS = {
+ max_expected_threads: "Maximum number of threads expected to be running and performing application work",
+ running_threads: "Number of running Ruby threads by name"
+ }.freeze
+
+ def metrics
+ @metrics ||= METRIC_DESCRIPTIONS.each_with_object({}) do |(name, description), result|
+ result[name] = ::Gitlab::Metrics.gauge(:"#{METRIC_PREFIX}#{name}", description)
+ end
+ end
+
+ def sample
+ metrics[:max_expected_threads].set({}, Gitlab::Runtime.max_threads)
+
+ threads_by_name.each do |name, threads|
+ uses_db, not_using_db = threads.partition { |thread| thread[:uses_db_connection] }
+
+ set_running_threads(name, uses_db_connection: "yes", size: uses_db.size)
+ set_running_threads(name, uses_db_connection: "no", size: not_using_db.size)
+ end
+ end
+
+ private
+
+ def set_running_threads(name, uses_db_connection:, size:)
+ metrics[:running_threads].set({ thread_name: name, uses_db_connection: uses_db_connection }, size)
+ end
+
+ def threads_by_name
+ Thread.list.group_by { |thread| name_for_thread(thread) }
+ end
+
+ def uses_db_connection(thread)
+ thread[:uses_db_connection] ? "yes" : "no"
+ end
+
+ def name_for_thread(thread)
+ thread_name = thread.name.to_s.presence
+
+ if thread_name.presence.nil?
+ 'unnamed'
+ elsif thread_name =~ /puma threadpool \d+/
+ # These are the puma workers processing requests
+ 'puma threadpool'
+ elsif use_thread_name?(thread_name)
+ thread_name
+ else
+ 'unrecognized'
+ end
+ end
+
+ def use_thread_name?(thread_name)
+ thread_name == SIDEKIQ_WORKER_THREAD_NAME ||
+ # Samplers defined in `lib/gitlab/metrics/samplers`
+ thread_name.ends_with?('sampler') ||
+ # Exporters from `lib/gitlab/metrics/exporter`
+ thread_name.ends_with?('exporter') ||
+ KNOWN_PUMA_THREAD_NAMES.include?(thread_name)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index 1c99e1e730c..8c4e5a8d70c 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -12,7 +12,9 @@ module Gitlab
begin
# Old gitlad-shell messages don't provide enqueued_at/created_at attributes
enqueued_at = payload['enqueued_at'] || payload['created_at'] || 0
- trans.set(:sidekiq_queue_duration, Time.current.to_f - enqueued_at)
+ trans.set(:gitlab_transaction_sidekiq_queue_duration_total, Time.current.to_f - enqueued_at) do
+ multiprocess_mode :livesum
+ end
trans.run { yield }
rescue Exception => error # rubocop: disable Lint/RescueException
trans.add_event(:sidekiq_exception)
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index 24107e42aa9..e1f1f37c905 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -5,14 +5,6 @@ module Gitlab
module Subscribers
# Class for tracking the rendering timings of views.
class ActionView < ActiveSupport::Subscriber
- include Gitlab::Metrics::Methods
- define_histogram :gitlab_view_rendering_duration_seconds do
- docstring 'View rendering time'
- base_labels Transaction::BASE_LABELS.merge({ path: nil })
- buckets [0.001, 0.01, 0.1, 1, 10.0]
- with_feature :prometheus_metrics_view_instrumentation
- end
-
attach_to :action_view
SERIES = 'views'
@@ -27,10 +19,14 @@ module Gitlab
def track(event)
tags = tags_for(event)
-
- self.class.gitlab_view_rendering_duration_seconds.observe(current_transaction.labels.merge(tags), event.duration)
-
- current_transaction.increment(:view_duration, event.duration)
+ current_transaction.observe(:gitlab_view_rendering_duration_seconds, event.duration, tags) do
+ docstring 'View rendering time'
+ label_keys %i(view)
+ buckets [0.001, 0.01, 0.1, 1, 10.0]
+ with_feature :prometheus_metrics_view_instrumentation
+ end
+
+ current_transaction.increment(:gitlab_transaction_view_duration_total, event.duration)
end
def relative_path(path)
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index d2736882432..e53ac00e77f 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -5,20 +5,25 @@ module Gitlab
module Subscribers
# Class for tracking the total query duration of a transaction.
class ActiveRecord < ActiveSupport::Subscriber
- include Gitlab::Metrics::Methods
attach_to :active_record
IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze
DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze
def sql(event)
+ # Mark this thread as requiring a database connection. This is used
+ # by the Gitlab::Metrics::Samplers::ThreadsSampler to count threads
+ # using a connection.
+ Thread.current[:uses_db_connection] = true
+
return unless current_transaction
payload = event.payload
-
return if payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql])
- self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0)
+ current_transaction.observe(:gitlab_sql_duration_seconds, event.duration / 1000.0) do
+ buckets [0.05, 0.1, 0.25]
+ end
increment_db_counters(payload)
end
@@ -33,12 +38,6 @@ module Gitlab
private
- define_histogram :gitlab_sql_duration_seconds do
- docstring 'SQL time'
- base_labels Transaction::BASE_LABELS
- buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
- end
-
def select_sql_command?(payload)
payload[:sql].match(/\A((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i)
end
@@ -54,7 +53,7 @@ module Gitlab
end
def increment(counter)
- current_transaction.increment(counter, 1)
+ current_transaction.increment("gitlab_transaction_#{counter}_total".to_sym, 1)
if Gitlab::SafeRequestStore.active?
Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1
diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb
index 2ee7144fe2f..b274d2b1079 100644
--- a/lib/gitlab/metrics/subscribers/rails_cache.rb
+++ b/lib/gitlab/metrics/subscribers/rails_cache.rb
@@ -14,11 +14,10 @@ module Gitlab
return unless current_transaction
return if event.payload[:super_operation] == :fetch
- if event.payload[:hit]
- current_transaction.increment(:cache_read_hit_count, 1, false)
- else
- metric_cache_misses_total.increment(current_transaction.labels)
- current_transaction.increment(:cache_read_miss_count, 1, false)
+ unless event.payload[:hit]
+ current_transaction.increment(:gitlab_cache_misses_total, 1) do
+ docstring 'Cache read miss'
+ end
end
end
@@ -37,25 +36,30 @@ module Gitlab
def cache_fetch_hit(event)
return unless current_transaction
- current_transaction.increment(:cache_read_hit_count, 1)
+ current_transaction.increment(:gitlab_transaction_cache_read_hit_count_total, 1)
end
def cache_generate(event)
return unless current_transaction
- metric_cache_misses_total.increment(current_transaction.labels)
- current_transaction.increment(:cache_read_miss_count, 1)
+ current_transaction.increment(:gitlab_cache_misses_total, 1) do
+ docstring 'Cache read miss'
+ end
+
+ current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1)
end
def observe(key, duration)
return unless current_transaction
- metric_cache_operations_total.increment(current_transaction.labels.merge({ operation: key }))
- metric_cache_operation_duration_seconds.observe({ operation: key }, duration / 1000.0)
- current_transaction.increment(:cache_duration, duration, false)
- current_transaction.increment(:cache_count, 1, false)
- current_transaction.increment("cache_#{key}_duration".to_sym, duration, false)
- current_transaction.increment("cache_#{key}_count".to_sym, 1, false)
+ labels = { operation: key }
+
+ current_transaction.increment(:gitlab_cache_operations_total, 1, labels) do
+ docstring 'Cache operations'
+ label_keys labels.keys
+ end
+
+ metric_cache_operation_duration_seconds.observe(labels, duration / 1000.0)
end
private
@@ -64,14 +68,6 @@ module Gitlab
Transaction.current
end
- def metric_cache_operations_total
- @metric_cache_operations_total ||= ::Gitlab::Metrics.counter(
- :gitlab_cache_operations_total,
- 'Cache operations',
- Transaction::BASE_LABELS
- )
- end
-
def metric_cache_operation_duration_seconds
@metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram(
:gitlab_cache_operation_duration_seconds,
@@ -80,14 +76,6 @@ module Gitlab
[0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0]
)
end
-
- def metric_cache_misses_total
- @metric_cache_misses_total ||= ::Gitlab::Metrics.counter(
- :gitlab_cache_misses_total,
- 'Cache read miss',
- Transaction::BASE_LABELS
- )
- end
end
end
end
diff --git a/lib/gitlab/metrics/templates/Area.metrics-dashboard.yml b/lib/gitlab/metrics/templates/Area.metrics-dashboard.yml
new file mode 100644
index 00000000000..1f7dd25aaee
--- /dev/null
+++ b/lib/gitlab/metrics/templates/Area.metrics-dashboard.yml
@@ -0,0 +1,15 @@
+# Only one dashboard should be defined per file
+# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html
+dashboard: 'Area Panel Example'
+
+# For more information about the required properties of panel_groups
+# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties
+panel_groups:
+ - group: 'Server Statistics'
+ panels:
+ - title: Average amount of time spent by the CPU
+ type: area-chart
+ metrics:
+ - query_range: 'rate(node_cpu_seconds_total[15m])'
+ unit: 'Seconds'
+ label: "Time in Seconds"
diff --git a/lib/gitlab/metrics/templates/Default.metrics-dashboard.yml b/lib/gitlab/metrics/templates/Default.metrics-dashboard.yml
new file mode 100644
index 00000000000..b331e792461
--- /dev/null
+++ b/lib/gitlab/metrics/templates/Default.metrics-dashboard.yml
@@ -0,0 +1,24 @@
+# Only one dashboard should be defined per file
+# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html
+dashboard: 'Single Stat'
+
+# This is where all of the variables that can be manipulated via the UI
+# are initialized
+# Check out: https://docs.gitlab.com/ee/operations/metrics/dashboards/templating_variables.html#templating-variables-for-metrics-dashboards-core
+templating:
+ variables:
+ job: 'prometheus'
+
+# For more information about the required properties of panel_groups
+# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties
+panel_groups:
+ - group: 'Memory'
+ panels:
+ - title: Prometheus
+ type: single-stat
+ metrics:
+ # Queries that make use of variables need to have double curly brackets {}
+ # set to the variables, per the example below
+ - query: 'max(go_memstats_alloc_bytes{job="{{job}}"}) / 1024 /1024'
+ unit: '%'
+ label: "Max"
diff --git a/lib/gitlab/metrics/templates/gauge.metrics-dashboard.yml b/lib/gitlab/metrics/templates/gauge.metrics-dashboard.yml
new file mode 100644
index 00000000000..1c17a3a4d40
--- /dev/null
+++ b/lib/gitlab/metrics/templates/gauge.metrics-dashboard.yml
@@ -0,0 +1,23 @@
+# Only one dashboard should be defined per file
+# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html
+dashboard: 'Gauge Panel Example'
+
+# For more information about the required properties of panel_groups
+# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties
+panel_groups:
+ - group: 'Server Statistics'
+ panels:
+ - title: "Memory usage"
+ # More information about gauge panel types can be found here:
+ # https://docs.gitlab.com/ee/operations/metrics/dashboards/panel_types.html#gauge
+ type: "gauge-chart"
+ min_value: 0
+ max_value: 1024
+ split: 10
+ thresholds:
+ mode: "percentage"
+ values: [60, 90]
+ format: "megabytes"
+ metrics:
+ - query: '(node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)) / 1024 / 1024'
+ unit: 'MB'
diff --git a/lib/gitlab/metrics/templates/index.md b/lib/gitlab/metrics/templates/index.md
new file mode 100644
index 00000000000..59fc85899da
--- /dev/null
+++ b/lib/gitlab/metrics/templates/index.md
@@ -0,0 +1,3 @@
+# Development guide for Metrics Dashboard templates
+
+Please follow [the development guideline](../../../../doc/development/operations/metrics/templates.md)
diff --git a/lib/gitlab/metrics/templates/k8s_area.metrics-dashboard.yml b/lib/gitlab/metrics/templates/k8s_area.metrics-dashboard.yml
new file mode 100644
index 00000000000..aea816658d0
--- /dev/null
+++ b/lib/gitlab/metrics/templates/k8s_area.metrics-dashboard.yml
@@ -0,0 +1,15 @@
+# Only one dashboard should be defined per file
+# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html
+dashboard: 'Area Panel Example'
+
+# For more information about the required properties of panel_groups
+# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties
+panel_groups:
+ - group: 'Server Statistics'
+ panels:
+ - title: "Core Usage (Pod Average)"
+ type: area-chart
+ metrics:
+ - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod)) OR avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod_name))'
+ unit: 'cores'
+ label: "Pod Average (in seconds)"
diff --git a/lib/gitlab/metrics/templates/k8s_gauge.metrics-dashboard.yml b/lib/gitlab/metrics/templates/k8s_gauge.metrics-dashboard.yml
new file mode 100644
index 00000000000..7f97719765b
--- /dev/null
+++ b/lib/gitlab/metrics/templates/k8s_gauge.metrics-dashboard.yml
@@ -0,0 +1,23 @@
+# Only one dashboard should be defined per file
+# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html
+dashboard: 'Gauge K8s Panel Example'
+
+# For more information about the required properties of panel_groups
+# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties
+panel_groups:
+ - group: 'Server Statistics'
+ panels:
+ - title: "Memory usage"
+ # More information about gauge panel types can be found here:
+ # https://docs.gitlab.com/ee/operations/metrics/dashboards/panel_types.html#gauge
+ type: "gauge-chart"
+ min_value: 0
+ max_value: 1024
+ split: 10
+ thresholds:
+ mode: "percentage"
+ values: [60, 90]
+ format: "megabytes"
+ metrics:
+ - query: 'avg(sum(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024 OR avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024'
+ unit: 'MB'
diff --git a/lib/gitlab/metrics/templates/k8s_single-stat.metrics-dashboard.yml b/lib/gitlab/metrics/templates/k8s_single-stat.metrics-dashboard.yml
new file mode 100644
index 00000000000..829e12357ff
--- /dev/null
+++ b/lib/gitlab/metrics/templates/k8s_single-stat.metrics-dashboard.yml
@@ -0,0 +1,17 @@
+# Only one dashboard should be defined per file
+# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html
+dashboard: 'Single Stat Panel Example'
+
+# For more information about the required properties of panel_groups
+# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties
+panel_groups:
+ - group: 'Server Statistics'
+ panels:
+ - title: "Memory usage"
+ # More information about heatmap panel types can be found here:
+ # https://docs.gitlab.com/ee/operations/metrics/dashboards/panel_types.html#single-stat
+ type: "single-stat"
+ metrics:
+ - query: 'avg(sum(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024 OR avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024'
+ unit: 'MB'
+ label: "Used memory"
diff --git a/lib/gitlab/metrics/templates/single-stat.metrics-dashboard.yml b/lib/gitlab/metrics/templates/single-stat.metrics-dashboard.yml
new file mode 100644
index 00000000000..18c27fffc7c
--- /dev/null
+++ b/lib/gitlab/metrics/templates/single-stat.metrics-dashboard.yml
@@ -0,0 +1,17 @@
+# Only one dashboard should be defined per file
+# More info: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html
+dashboard: 'Heatmap Panel Example'
+
+# For more information about the required properties of panel_groups
+# please visit: https://docs.gitlab.com/ee/operations/metrics/dashboards/yaml.html#panel-group-panel_groups-properties
+panel_groups:
+ - group: 'Server Statistics'
+ panels:
+ - title: "Memory usage"
+ # More information about heatmap panel types can be found here:
+ # https://docs.gitlab.com/ee/operations/metrics/dashboards/panel_types.html#single-stat
+ type: "single-stat"
+ metrics:
+ - query: '(node_memory_MemTotal_bytes - (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes)) / 1024 / 1024'
+ unit: 'MB'
+ label: "Used memory"
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index da06be9c79c..95bc90f9dad 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -6,20 +6,35 @@ module Gitlab
class Transaction
include Gitlab::Metrics::Methods
- # base labels shared among all transactions
- BASE_LABELS = { controller: nil, action: nil, feature_category: nil }.freeze
+ # base label keys shared among all transactions
+ BASE_LABEL_KEYS = %i(controller action feature_category).freeze
# labels that potentially contain sensitive information and will be filtered
- FILTERED_LABELS = [:branch, :path].freeze
+ FILTERED_LABEL_KEYS = %i(branch path).freeze
THREAD_KEY = :_gitlab_metrics_transaction
+ SMALL_BUCKETS = [0.1, 0.25, 0.5, 1.0, 2.5, 5.0].freeze
+
# The series to store events (e.g. Git pushes) in.
EVENT_SERIES = 'events'
attr_reader :method
- def self.current
- Thread.current[THREAD_KEY]
+ class << self
+ def current
+ Thread.current[THREAD_KEY]
+ end
+
+ def prometheus_metric(name, type, &block)
+ fetch_metric(type, name) do
+ # set default metric options
+ docstring "#{name.to_s.humanize} #{type}"
+
+ evaluate(&block)
+ # always filter sensitive labels and merge with base ones
+ label_keys BASE_LABEL_KEYS | (label_keys - FILTERED_LABEL_KEYS)
+ end
+ end
end
def initialize
@@ -27,9 +42,6 @@ module Gitlab
@started_at = nil
@finished_at = nil
-
- @memory_before = 0
- @memory_after = 0
end
def duration
@@ -40,25 +52,22 @@ module Gitlab
System.thread_cpu_duration(@thread_cputime_start)
end
- def allocated_memory
- @memory_after - @memory_before
- end
-
def run
Thread.current[THREAD_KEY] = self
- @memory_before = System.memory_usage_rss
@started_at = System.monotonic_time
@thread_cputime_start = System.thread_cpu_time
yield
ensure
- @memory_after = System.memory_usage_rss
@finished_at = System.monotonic_time
- self.class.gitlab_transaction_cputime_seconds.observe(labels, thread_cpu_duration)
- self.class.gitlab_transaction_duration_seconds.observe(labels, duration)
- self.class.gitlab_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0)
+ observe(:gitlab_transaction_cputime_seconds, thread_cpu_duration) do
+ buckets SMALL_BUCKETS
+ end
+ observe(:gitlab_transaction_duration_seconds, duration) do
+ buckets SMALL_BUCKETS
+ end
Thread.current[THREAD_KEY] = nil
end
@@ -71,8 +80,12 @@ module Gitlab
# event_name - The name of the event (e.g. "git_push").
# tags - A set of tags to attach to the event.
def add_event(event_name, tags = {})
- filtered_tags = filter_tags(tags)
- self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: filtered_tags).increment(filtered_tags.merge(labels))
+ event_name = "gitlab_transaction_event_#{event_name}_total".to_sym
+ metric = self.class.prometheus_metric(event_name, :counter) do
+ label_keys tags.keys
+ end
+
+ metric.increment(filter_labels(tags))
end
# Returns a MethodCall object for the given name.
@@ -84,52 +97,70 @@ module Gitlab
method
end
- def increment(name, value, use_prometheus = true)
- self.class.transaction_metric(name, :counter).increment(labels, value) if use_prometheus
- end
+ # Increment counter metric
+ #
+ # It will initialize the metric if metric is not found
+ #
+ # block - if provided can be used to initialize metric with custom options (docstring, labels, with_feature)
+ #
+ # Example:
+ # ```
+ # transaction.increment(:mestric_name, 1, { docstring: 'Custom title', base_labels: {sane: 'yes'} } ) do
+ #
+ # transaction.increment(:mestric_name, 1) do
+ # docstring 'Custom title'
+ # label_keys %i(sane)
+ # end
+ # ```
+ def increment(name, value = 1, labels = {}, &block)
+ counter = self.class.prometheus_metric(name, :counter, &block)
- def set(name, value, use_prometheus = true)
- self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus
+ counter.increment(filter_labels(labels), value)
end
- def labels
- BASE_LABELS
- end
+ # Set gauge metric
+ #
+ # It will initialize the metric if metric is not found
+ #
+ # block - if provided, it can be used to initialize metric with custom options (docstring, labels, with_feature, multiprocess_mode)
+ # - multiprocess_mode is :all by default
+ #
+ # Example:
+ # ```
+ # transaction.set(:mestric_name, 1) do
+ # multiprocess_mode :livesum
+ # end
+ # ```
+ def set(name, value, labels = {}, &block)
+ gauge = self.class.prometheus_metric(name, :gauge, &block)
- define_histogram :gitlab_transaction_cputime_seconds do
- docstring 'Transaction thread cputime'
- base_labels BASE_LABELS
- buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
+ gauge.set(filter_labels(labels), value)
end
- define_histogram :gitlab_transaction_duration_seconds do
- docstring 'Transaction duration'
- base_labels BASE_LABELS
- buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
- end
+ # Observe histogram metric
+ #
+ # It will initialize the metric if metric is not found
+ #
+ # block - if provided, it can be used to initialize metric with custom options (docstring, labels, with_feature, buckets)
+ #
+ # Example:
+ # ```
+ # transaction.observe(:mestric_name, 1) do
+ # buckets [100, 1000, 10000, 100000, 1000000, 10000000]
+ # end
+ # ```
+ def observe(name, value, labels = {}, &block)
+ histogram = self.class.prometheus_metric(name, :histogram, &block)
- define_histogram :gitlab_transaction_allocated_memory_bytes do
- docstring 'Transaction allocated memory bytes'
- base_labels BASE_LABELS
- buckets [100, 1000, 10000, 100000, 1000000, 10000000]
+ histogram.observe(filter_labels(labels), value)
end
- def self.transaction_metric(name, type, prefix: nil, tags: {})
- metric_name = "gitlab_transaction_#{prefix}#{name}_total".to_sym
- fetch_metric(type, metric_name) do
- docstring "Transaction #{prefix}#{name} #{type}"
- base_labels tags.merge(BASE_LABELS)
-
- if type == :gauge
- multiprocess_mode :livesum
- end
- end
+ def labels
+ BASE_LABEL_KEYS.product([nil]).to_h
end
- private
-
- def filter_tags(tags)
- tags.without(*FILTERED_LABELS)
+ def filter_labels(labels)
+ labels.empty? ? self.labels : labels.without(*FILTERED_LABEL_KEYS).merge(self.labels)
end
end
end
diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb
index 630788f1a8a..ce1065c0cb3 100644
--- a/lib/gitlab/middleware/rails_queue_duration.rb
+++ b/lib/gitlab/middleware/rails_queue_duration.rb
@@ -19,25 +19,19 @@ module Gitlab
if trans && proxy_start
# Time in milliseconds since gitlab-workhorse started the request
duration = Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000
- trans.set(:rails_queue_duration, duration)
+ trans.set(:gitlab_transaction_rails_queue_duration_total, duration) do
+ multiprocess_mode :livesum
+ end
duration_s = Gitlab::Utils.ms_to_round_sec(duration)
- metric_rails_queue_duration_seconds.observe(trans.labels, duration_s)
+ trans.observe(:gitlab_rails_queue_duration_seconds, duration_s) do
+ docstring 'Measures latency between GitLab Workhorse forwarding a request to Rails'
+ end
env[GITLAB_RAILS_QUEUE_DURATION_KEY] = duration_s
end
@app.call(env)
end
-
- private
-
- def metric_rails_queue_duration_seconds
- @metric_rails_queue_duration_seconds ||= Gitlab::Metrics.histogram(
- :gitlab_rails_queue_duration_seconds,
- 'Measures latency between GitLab Workhorse forwarding a request to Rails',
- Gitlab::Metrics::Transaction::BASE_LABELS
- )
- end
end
end
end
diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb
index 1c49379e8d2..6573506e926 100644
--- a/lib/gitlab/middleware/read_only/controller.rb
+++ b/lib/gitlab/middleware/read_only/controller.rb
@@ -136,7 +136,7 @@ module Gitlab
end
def graphql_query?
- request.post? && request.path.start_with?(GRAPHQL_URL)
+ request.post? && request.path.start_with?(File.join(relative_url, GRAPHQL_URL))
end
end
end
diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb
index 33e0c6aa9b7..002171854ad 100644
--- a/lib/gitlab/multi_collection_paginator.rb
+++ b/lib/gitlab/multi_collection_paginator.rb
@@ -34,7 +34,7 @@ module Gitlab
@second_collection_pages ||= Hash.new do |hash, page|
second_collection_page = page - first_collection_page_count
- offset = if second_collection_page < 1 || first_collection_page_count.zero?
+ offset = if second_collection_page < 1 || first_collection_page_count == 0
0
else
per_page - first_collection_last_page_size
diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb
index c8cb8b6e020..33e709360ad 100644
--- a/lib/gitlab/pages.rb
+++ b/lib/gitlab/pages.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Gitlab
- class Pages
+ module Pages
VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze
INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'.freeze
MAX_SIZE = 1.terabyte
diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb
new file mode 100644
index 00000000000..e3dbeee7b13
--- /dev/null
+++ b/lib/gitlab/pages/settings.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Pages
+ class Settings < ::SimpleDelegator
+ DiskAccessDenied = Class.new(StandardError)
+
+ def path
+ if ::Gitlab::Runtime.web_server? && ENV['GITLAB_PAGES_DENY_DISK_ACCESS'] == '1'
+ begin
+ raise DiskAccessDenied
+ rescue DiskAccessDenied => ex
+ ::Gitlab::ErrorTracking.track_exception(ex)
+ end
+ end
+
+ super
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb
new file mode 100644
index 00000000000..651e3d5a807
--- /dev/null
+++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Pagination
+ class GitalyKeysetPager
+ attr_reader :request_context, :project
+ delegate :params, to: :request_context
+
+ def initialize(request_context, project)
+ @request_context = request_context
+ @project = project
+ end
+
+ # It is expected that the given finder will respond to `execute` method with `gitaly_pagination: true` option
+ # and supports pagination via gitaly.
+ def paginate(finder)
+ return paginate_via_gitaly(finder) if keyset_pagination_enabled?
+
+ branches = ::Kaminari.paginate_array(finder.execute)
+ Gitlab::Pagination::OffsetPagination
+ .new(request_context)
+ .paginate(branches)
+ end
+
+ private
+
+ def keyset_pagination_enabled?
+ Feature.enabled?(:branch_list_keyset_pagination, project) && params[:pagination] == 'keyset'
+ end
+
+ def paginate_via_gitaly(finder)
+ finder.execute(gitaly_pagination: true).tap do |records|
+ apply_headers(records)
+ end
+ end
+
+ def apply_headers(records)
+ if records.count == params[:per_page]
+ Gitlab::Pagination::Keyset::HeaderBuilder
+ .new(request_context)
+ .add_next_page_header(
+ query_params_for(records.last)
+ )
+ end
+ end
+
+ def query_params_for(record)
+ # NOTE: page_token is name for now, but it could be dynamic if we have other gitaly finders
+ # that is based on something other than name
+ { page_token: record.name }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/pagination/keyset/header_builder.rb b/lib/gitlab/pagination/keyset/header_builder.rb
new file mode 100644
index 00000000000..69c468207f6
--- /dev/null
+++ b/lib/gitlab/pagination/keyset/header_builder.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Pagination
+ module Keyset
+ class HeaderBuilder
+ attr_reader :request_context
+ delegate :params, :header, :request, to: :request_context
+
+ def initialize(request_context)
+ @request_context = request_context
+ end
+
+ def add_next_page_header(query_params)
+ link = next_page_link(page_href(query_params))
+ header('Links', link)
+ header('Link', link)
+ end
+
+ private
+
+ def next_page_link(href)
+ %(<#{href}>; rel="next")
+ end
+
+ def page_href(query_params)
+ base_request_uri.tap do |uri|
+ uri.query = updated_params(query_params).to_query
+ end.to_s
+ end
+
+ def base_request_uri
+ @base_request_uri ||= URI.parse(request.url).tap do |uri|
+ uri.host = Gitlab.config.gitlab.host
+ uri.port = Gitlab.config.gitlab.port
+ end
+ end
+
+ def updated_params(query_params)
+ params.merge(query_params)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/pagination/keyset/request_context.rb b/lib/gitlab/pagination/keyset/request_context.rb
index 070fa844347..ba17fb03681 100644
--- a/lib/gitlab/pagination/keyset/request_context.rb
+++ b/lib/gitlab/pagination/keyset/request_context.rb
@@ -24,9 +24,11 @@ module Gitlab
end
def apply_headers(next_page)
- link = pagination_links(next_page)
- request.header('Links', link)
- request.header('Link', link)
+ Gitlab::Pagination::Keyset::HeaderBuilder
+ .new(request)
+ .add_next_page_header(
+ query_params_for(next_page)
+ )
end
private
@@ -63,25 +65,8 @@ module Gitlab
end
end
- def page_href(page)
- base_request_uri.tap do |uri|
- uri.query = query_params_for(page).to_query
- end.to_s
- end
-
- def pagination_links(next_page)
- %(<#{page_href(next_page)}>; rel="next")
- end
-
- def base_request_uri
- @base_request_uri ||= URI.parse(request.request.url).tap do |uri|
- uri.host = Gitlab.config.gitlab.host
- uri.port = Gitlab.config.gitlab.port
- end
- end
-
def query_params_for(page)
- request.params.merge(lower_bounds_params(page))
+ lower_bounds_params(page)
end
end
end
diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb
index e286c3d467e..2c95a9e8d91 100644
--- a/lib/gitlab/polling_interval.rb
+++ b/lib/gitlab/polling_interval.rb
@@ -20,7 +20,7 @@ module Gitlab
end
def self.polling_enabled?
- !Gitlab::CurrentSettings.polling_interval_multiplier.zero?
+ Gitlab::CurrentSettings.polling_interval_multiplier != 0
end
end
end
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index e6b25e71eb3..f8141278e48 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -81,7 +81,7 @@ module Gitlab
counts = %i(limited_milestones_count limited_notes_count
limited_merge_requests_count limited_issues_count
limited_blobs_count wiki_blobs_count)
- counts.all? { |count_method| public_send(count_method).zero? } # rubocop:disable GitlabSecurity/PublicSend
+ counts.all? { |count_method| public_send(count_method) == 0 } # rubocop:disable GitlabSecurity/PublicSend
end
private
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index 69499b5494e..56e1154a672 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -77,12 +77,12 @@ module Gitlab
# metric labels to their respective values.
#
# @return [Hash] mapping labels to their aggregate numeric values, or the empty hash if no results were found
- def aggregate(aggregate_query, time: Time.now)
+ def aggregate(aggregate_query, time: Time.now, transform_value: :to_f)
response = query(aggregate_query, time: time)
response.to_h do |result|
key = block_given? ? yield(result['metric']) : result['metric']
_timestamp, value = result['value']
- [key, value.to_i]
+ [key, value.public_send(transform_value)] # rubocop:disable GitlabSecurity/PublicSend
end
end
diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb
index 609087d8137..8a432edbd78 100644
--- a/lib/gitlab/reactive_cache_set_cache.rb
+++ b/lib/gitlab/reactive_cache_set_cache.rb
@@ -20,7 +20,7 @@ module Gitlab
keys << cache_key(key)
redis.pipelined do
- keys.each_slice(1000) { |subset| redis.del(*subset) }
+ keys.each_slice(1000) { |subset| redis.unlink(*subset) }
end
end
end
diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb
new file mode 100644
index 00000000000..ff5754675e2
--- /dev/null
+++ b/lib/gitlab/redis/hll.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Redis
+ class HLL
+ KEY_REGEX = %r{\A(\w|-|:)*\{\w*\}(\w|-|:)*\z}.freeze
+ KeyFormatError = Class.new(StandardError)
+
+ def self.count(params)
+ self.new.count(params)
+ end
+
+ def self.add(params)
+ self.new.add(params)
+ end
+
+ def count(keys:)
+ Gitlab::Redis::SharedState.with do |redis|
+ redis.pfcount(*keys)
+ end
+ end
+
+ # Check a basic format for the Redis key in order to ensure the keys are in the same hash slot
+ #
+ # Examples of keys
+ # project:{1}:set_a
+ # project:{1}:set_b
+ # project:{2}:set_c
+ # 2020-216-{project_action}
+ # i_{analytics}_dev_ops_score-2020-32
+ def add(key:, value:, expiry:)
+ unless KEY_REGEX.match?(key)
+ raise KeyFormatError.new("Invalid key format. #{key} key should have changeable parts in curly braces. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands")
+ end
+
+ Gitlab::Redis::SharedState.with do |redis|
+ redis.multi do |multi|
+ multi.pfadd(key, value)
+ multi.expire(key, expiry)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 784f8b48f3c..1e1e0d856b7 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -3,7 +3,7 @@
module Gitlab
module Regex
module Packages
- CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt].freeze
+ CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze
CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze
def conan_file_name_regex
@@ -160,6 +160,15 @@ module Gitlab
"can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', '*' and spaces"
end
+ # https://gitlab.com/gitlab-org/cluster-integration/gitlab-agent/-/blob/master/doc/identity_and_auth.md#agent-identity-and-name
+ def cluster_agent_name_regex
+ /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/
+ end
+
+ def cluster_agent_name_regex_message
+ %q{can contain only lowercase letters, digits, and '-', but cannot start or end with '-'}
+ end
+
def kubernetes_namespace_regex
/\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/
end
diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb
index 688a4a39dba..f6a5c6ed754 100644
--- a/lib/gitlab/repository_cache_adapter.rb
+++ b/lib/gitlab/repository_cache_adapter.rb
@@ -58,11 +58,19 @@ module Gitlab
# wrong answer. We handle that by querying the full list - which fills
# the cache - and using it directly to answer the question.
define_method("#{name}_include?") do |value|
- if strong_memoized?(name) || !redis_set_cache.exist?(name)
- return __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend
- end
+ ivar = "@#{name}_include"
+ memoized = instance_variable_get(ivar) || {}
+
+ next memoized[value] if memoized.key?(value)
+
+ memoized[value] =
+ if strong_memoized?(name) || !redis_set_cache.exist?(name)
+ __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend
+ else
+ redis_set_cache.include?(name, value)
+ end
- redis_set_cache.include?(name, value)
+ instance_variable_set(ivar, memoized)[value]
end
end
@@ -241,7 +249,7 @@ module Gitlab
end
def expire_redis_hash_method_caches(methods)
- methods.each { |name| redis_hash_cache.delete(name) }
+ redis_hash_cache.delete(*methods)
end
# All cached repository methods depend on the existence of a Git repository,
diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb
index d2a7b450000..d479d3115a6 100644
--- a/lib/gitlab/repository_hash_cache.rb
+++ b/lib/gitlab/repository_hash_cache.rb
@@ -31,10 +31,18 @@ module Gitlab
"#{type}:#{namespace}:hash"
end
- # @param key [String]
- # @return [Integer] 0 or 1 depending on success
- def delete(key)
- with { |redis| redis.del(cache_key(key)) }
+ # @param keys [String] one or multiple keys to delete
+ # @return [Integer] the number of keys successfully deleted
+ def delete(*keys)
+ return 0 if keys.empty?
+
+ with do |redis|
+ keys = keys.map { |key| cache_key(key) }
+
+ Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
+ redis.unlink(*keys)
+ end
+ end
end
# Check if the provided hash key exists in the hash.
diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb
index 1f6e0519b4c..5d5d407c172 100644
--- a/lib/gitlab/search/parsed_query.rb
+++ b/lib/gitlab/search/parsed_query.rb
@@ -3,6 +3,8 @@
module Gitlab
module Search
class ParsedQuery
+ include Gitlab::Utils::StrongMemoize
+
attr_reader :term, :filters
def initialize(term, filters)
@@ -11,13 +13,44 @@ module Gitlab
end
def filter_results(results)
- filters = @filters.reject { |filter| filter[:matcher].nil? }
- return unless filters
+ with_matcher = ->(filter) { filter[:matcher].present? }
+
+ excluding = excluding_filters.select(&with_matcher)
+ including = including_filters.select(&with_matcher)
+
+ return unless excluding.any? || including.any?
+
+ results.select! do |result|
+ including.all? { |filter| filter[:matcher].call(filter, result) }
+ end
+
+ results.reject! do |result|
+ excluding.any? { |filter| filter[:matcher].call(filter, result) }
+ end
+
+ results
+ end
+
+ private
+
+ def including_filters
+ processed_filters(:including)
+ end
+
+ def excluding_filters
+ processed_filters(:excluding)
+ end
+
+ def processed_filters(type)
+ excluding, including = strong_memoize(:processed_filters) do
+ filters.partition { |filter| filter[:negated] }
+ end
- results.select do |result|
- filters.all? do |filter|
- filter[:matcher].call(filter, result)
- end
+ case type
+ when :including then including
+ when :excluding then excluding
+ else
+ raise ArgumentError.new(type)
end
end
end
diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb
index ba0e16607a6..27ea0b7367f 100644
--- a/lib/gitlab/search/query.rb
+++ b/lib/gitlab/search/query.rb
@@ -20,7 +20,10 @@ module Gitlab
private
def filter(name, **attributes)
- filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes)
+ filter = {
+ parser: @filter_options[:default_parser],
+ name: name
+ }.merge(attributes)
@filters << filter
end
@@ -33,12 +36,13 @@ module Gitlab
fragments = []
filters = @filters.each_with_object([]) do |filter, parsed_filters|
- match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ }
+ match = @raw_query.split.find { |part| part =~ /\A-?#{filter[:name]}:/ }
next unless match
input = match.split(':')[1..-1].join
next if input.empty?
+ filter[:negated] = match.start_with?("-")
filter[:value] = parse_filter(filter, input)
filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?')
fragments << match
diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb
index d652719721e..e26d45e1b33 100644
--- a/lib/gitlab/seeder.rb
+++ b/lib/gitlab/seeder.rb
@@ -66,7 +66,7 @@ module Gitlab
estimated_minutes = (size.to_f / ESTIMATED_INSERT_PER_MINUTE).round
humanized_minutes = 'minute'.pluralize(estimated_minutes)
- if estimated_minutes.zero?
+ if estimated_minutes == 0
"Rough estimated time: less than a minute ⏰"
else
"Rough estimated time: #{estimated_minutes} #{humanized_minutes} ⏰"
diff --git a/lib/gitlab/service_desk_email.rb b/lib/gitlab/service_desk_email.rb
index f8dba82cb40..52da10eff3e 100644
--- a/lib/gitlab/service_desk_email.rb
+++ b/lib/gitlab/service_desk_email.rb
@@ -17,6 +17,12 @@ module Gitlab
def config
Gitlab.config.service_desk_email
end
+
+ def address_for_key(key)
+ return if config.address.blank?
+
+ config.address.sub(Gitlab::IncomingEmail::WILDCARD_PLACEHOLDER, key)
+ end
end
end
end
diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb
index 6ba9ee26634..591265d014e 100644
--- a/lib/gitlab/set_cache.rb
+++ b/lib/gitlab/set_cache.rb
@@ -22,7 +22,7 @@ module Gitlab
keys = keys.map { |key| cache_key(key) }
Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
- unlink_or_delete(redis, keys)
+ redis.unlink(*keys)
end
end
end
@@ -60,17 +60,5 @@ module Gitlab
def with(&blk)
Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord
end
-
- def unlink_or_delete(redis, keys)
- if Feature.enabled?(:repository_set_cache_unlink, default_enabled: true)
- redis.unlink(*keys)
- else
- redis.del(*keys)
- end
- rescue ::Redis::CommandError => e
- Gitlab::ErrorTracking.log_exception(e)
-
- redis.del(*keys)
- end
end
end
diff --git a/lib/gitlab/sidekiq_cluster.rb b/lib/gitlab/sidekiq_cluster.rb
index e74ae8d0f03..d05c717d2fa 100644
--- a/lib/gitlab/sidekiq_cluster.rb
+++ b/lib/gitlab/sidekiq_cluster.rb
@@ -126,7 +126,7 @@ module Gitlab
def self.concurrency(queues, min_concurrency, max_concurrency)
concurrency_from_queues = queues.length + 1
- max = max_concurrency.positive? ? max_concurrency : concurrency_from_queues
+ max = max_concurrency > 0 ? max_concurrency : concurrency_from_queues
min = [min_concurrency, max].min
concurrency_from_queues.clamp(min, max)
diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb
index 9d0d67a488f..b8a4eedd620 100644
--- a/lib/gitlab/sidekiq_daemon/memory_killer.rb
+++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb
@@ -239,7 +239,7 @@ module Gitlab
memory_growth_kb = get_job_options(job, 'memory_killer_memory_growth_kb', 0).to_i
max_memory_growth_kb = get_job_options(job, 'memory_killer_max_memory_growth_kb', DEFAULT_MAX_MEMORY_GROWTH_KB).to_i
- return 0 if memory_growth_kb.zero?
+ return 0 if memory_growth_kb == 0
time_elapsed = [Gitlab::Metrics::System.monotonic_time - job[:started_at], 0].max
[memory_growth_kb * time_elapsed, max_memory_growth_kb].min
diff --git a/lib/gitlab/sidekiq_logger.rb b/lib/gitlab/sidekiq_logger.rb
deleted file mode 100644
index ce82a6f04bb..00000000000
--- a/lib/gitlab/sidekiq_logger.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- class SidekiqLogger < Gitlab::Logger
- def self.file_name_noext
- 'sidekiq'
- end
- end
-end
diff --git a/lib/gitlab/sidekiq_logging/exception_handler.rb b/lib/gitlab/sidekiq_logging/exception_handler.rb
index a6d6819bf8e..8ae6addc2c6 100644
--- a/lib/gitlab/sidekiq_logging/exception_handler.rb
+++ b/lib/gitlab/sidekiq_logging/exception_handler.rb
@@ -18,7 +18,7 @@ module Gitlab
data.merge!(job_data) if job_data.present?
end
- data[:error_backtrace] = Gitlab::BacktraceCleaner.clean_backtrace(job_exception.backtrace) if job_exception.backtrace.present?
+ data[:error_backtrace] = Rails.backtrace_cleaner.clean(job_exception.backtrace) if job_exception.backtrace.present?
Sidekiq.logger.warn(data)
end
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index 4eef3fbd12e..5f912818605 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -19,6 +19,7 @@ module Gitlab
chain.add ::Labkit::Middleware::Sidekiq::Server
chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger
chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server
+ chain.add ::Gitlab::SidekiqVersioning::Middleware
chain.add ::Gitlab::SidekiqStatus::ServerMiddleware
chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server
chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server
diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb
index 49c4fdc3033..0b38c98f710 100644
--- a/lib/gitlab/sidekiq_middleware/memory_killer.rb
+++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb
@@ -55,7 +55,7 @@ module Gitlab
def get_rss
output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s)
- return 0 unless status.zero?
+ return 0 unless status == 0
output.to_i
end
diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb
index 6a942a6ce06..0635c07ae4b 100644
--- a/lib/gitlab/sidekiq_middleware/server_metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb
@@ -14,6 +14,10 @@ module Gitlab
end
def call(worker, job, queue)
+ # This gives all the sidekiq worker threads a name, so we can recognize them
+ # in metrics and can use them in the `ThreadsSampler` for setting a label
+ Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME
+
labels = create_labels(worker.class, queue)
queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job)
diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb
index 0dafccb3d34..2293e2adee1 100644
--- a/lib/gitlab/sidekiq_status.rb
+++ b/lib/gitlab/sidekiq_status.rb
@@ -50,7 +50,7 @@ module Gitlab
#
# Returns true or false.
def self.all_completed?(job_ids)
- self.num_running(job_ids).zero?
+ self.num_running(job_ids) == 0
end
# Returns true if the given job is running or enqueued.
diff --git a/lib/gitlab/sidekiq_versioning/middleware.rb b/lib/gitlab/sidekiq_versioning/middleware.rb
new file mode 100644
index 00000000000..2ffee617376
--- /dev/null
+++ b/lib/gitlab/sidekiq_versioning/middleware.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SidekiqVersioning
+ class Middleware
+ def call(worker, job, queue)
+ worker.job_version = job['version'] if worker.is_a?(ApplicationWorker)
+
+ yield
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sidekiq_versioning/worker.rb b/lib/gitlab/sidekiq_versioning/worker.rb
new file mode 100644
index 00000000000..fe9bae6b8a1
--- /dev/null
+++ b/lib/gitlab/sidekiq_versioning/worker.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SidekiqVersioning
+ module Worker
+ extend ActiveSupport::Concern
+
+ included do
+ version 0
+
+ attr_writer :job_version
+ end
+
+ class_methods do
+ def version(new_version = nil)
+ if new_version
+ sidekiq_options version: new_version.to_i
+ else
+ get_sidekiq_options['version']
+ end
+ end
+ end
+
+ # Version is not set if `new.perform` is called directly,
+ # and in that case we fallback to latest version
+ def job_version
+ @job_version ||= self.class.version
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb
index b60f0b78fef..b8affb42372 100644
--- a/lib/gitlab/slash_commands/presenters/base.rb
+++ b/lib/gitlab/slash_commands/presenters/base.rb
@@ -69,7 +69,6 @@ module Gitlab
def resource_url
url_for(
[
- resource.project.namespace.becomes(Namespace),
resource.project,
resource
]
diff --git a/lib/gitlab/slash_commands/presenters/issue_search.rb b/lib/gitlab/slash_commands/presenters/issue_search.rb
index fffa082baac..f5b1670b2e9 100644
--- a/lib/gitlab/slash_commands/presenters/issue_search.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_search.rb
@@ -22,7 +22,7 @@ module Gitlab
def attachments
resource.map do |issue|
- url = "[#{issue.to_reference}](#{url_for([namespace, project, issue])})"
+ url = "[#{issue.to_reference}](#{url_for([project, issue])})"
{
color: color(issue),
@@ -39,10 +39,6 @@ module Gitlab
def project
@project ||= resource.first.project
end
-
- def namespace
- @namespace ||= project.namespace.becomes(Namespace)
- end
end
end
end
diff --git a/lib/gitlab/slash_commands/presenters/issue_show.rb b/lib/gitlab/slash_commands/presenters/issue_show.rb
index 448381b64ed..e9df015f249 100644
--- a/lib/gitlab/slash_commands/presenters/issue_show.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_show.rb
@@ -23,14 +23,14 @@ module Gitlab
def text
message = ["**#{status_text(resource)}**"]
- if resource.upvotes.zero? && resource.downvotes.zero? && resource.user_notes_count.zero?
+ if resource.upvotes == 0 && resource.downvotes == 0 && resource.user_notes_count == 0
return message.join
end
message << " · "
- message << ":+1: #{resource.upvotes} " unless resource.upvotes.zero?
- message << ":-1: #{resource.downvotes} " unless resource.downvotes.zero?
- message << ":speech_balloon: #{resource.user_notes_count}" unless resource.user_notes_count.zero?
+ message << ":+1: #{resource.upvotes} " unless resource.upvotes == 0
+ message << ":-1: #{resource.downvotes} " unless resource.downvotes == 0
+ message << ":speech_balloon: #{resource.user_notes_count}" unless resource.user_notes_count == 0
message.join
end
diff --git a/lib/gitlab/static_site_editor/config.rb b/lib/gitlab/static_site_editor/config.rb
index 08ed6599a6e..d335a434335 100644
--- a/lib/gitlab/static_site_editor/config.rb
+++ b/lib/gitlab/static_site_editor/config.rb
@@ -3,7 +3,7 @@
module Gitlab
module StaticSiteEditor
class Config
- SUPPORTED_EXTENSIONS = %w[.md .md.erb].freeze
+ SUPPORTED_EXTENSIONS = %w[.md].freeze
def initialize(repository, ref, file_path, return_url)
@repository = repository
@@ -42,6 +42,8 @@ module Gitlab
end
def extension_supported?
+ return true if file_path.end_with?('.md.erb') && Feature.enabled?(:sse_erb_support, project)
+
SUPPORTED_EXTENSIONS.any? { |ext| file_path.end_with?(ext) }
end
diff --git a/lib/gitlab/suggestions/suggestion_set.rb b/lib/gitlab/suggestions/suggestion_set.rb
index abb05ba56a7..f9a635734a3 100644
--- a/lib/gitlab/suggestions/suggestion_set.rb
+++ b/lib/gitlab/suggestions/suggestion_set.rb
@@ -83,7 +83,7 @@ module Gitlab
end
unless suggestion.appliable?(cached: false)
- return _('A suggestion is not applicable.')
+ return suggestion.inapplicable_reason(cached: false)
end
unless latest_source_head?(suggestion)
diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb
index 6ccb442b1e0..73187d8dea8 100644
--- a/lib/gitlab/task_helpers.rb
+++ b/lib/gitlab/task_helpers.rb
@@ -95,7 +95,7 @@ module Gitlab
def run_command!(command)
output, status = Gitlab::Popen.popen(command)
- raise Gitlab::TaskFailedError.new(output) unless status.zero?
+ raise Gitlab::TaskFailedError.new(output) unless status == 0
output
end
diff --git a/lib/gitlab/template/metrics_dashboard_template.rb b/lib/gitlab/template/metrics_dashboard_template.rb
new file mode 100644
index 00000000000..88fc3007b63
--- /dev/null
+++ b/lib/gitlab/template/metrics_dashboard_template.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Template
+ class MetricsDashboardTemplate < BaseTemplate
+ def content
+ explanation = "# This file is a template, and might need editing before it works on your project."
+ [explanation, super].join("\n")
+ end
+
+ class << self
+ def extension
+ '.metrics-dashboard.yml'
+ end
+
+ def categories
+ {
+ "General" => ''
+ }
+ end
+
+ def base_dir
+ Rails.root.join('lib/gitlab/metrics/templates')
+ end
+
+ def finder(project = nil)
+ Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb
index c237f4a7404..6a3e2062070 100644
--- a/lib/gitlab/untrusted_regexp.rb
+++ b/lib/gitlab/untrusted_regexp.rb
@@ -31,7 +31,7 @@ module Gitlab
def scan(text)
matches = scan_regexp.scan(text).to_a
- matches.map!(&:first) if regexp.number_of_capturing_groups.zero?
+ matches.map!(&:first) if regexp.number_of_capturing_groups == 0
matches
end
@@ -68,7 +68,7 @@ module Gitlab
# groups, so work around it
def scan_regexp
@scan_regexp ||=
- if regexp.number_of_capturing_groups.zero?
+ if regexp.number_of_capturing_groups == 0
RE2::Regexp.new('(' + regexp.source + ')')
else
regexp
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 9d7e6536608..70efe86143e 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -10,11 +10,8 @@
# alt_usage_data { Gitlab::VERSION }
# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
-
module Gitlab
class UsageData
- BATCH_SIZE = 100
-
class << self
include Gitlab::Utils::UsageData
include Gitlab::Utils::StrongMemoize
@@ -40,6 +37,7 @@ module Gitlab
.merge(usage_activity_by_stage)
.merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period))
.merge(analytics_unique_visits_data)
+ .merge(compliance_unique_visits_data)
end
end
@@ -60,13 +58,12 @@ module Gitlab
end
def recorded_at
- Time.now
+ Time.current
end
# rubocop: disable Metrics/AbcSize
# rubocop: disable CodeReuse/ActiveRecord
def system_usage_data
- alert_bot_incident_count = count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
{
@@ -84,9 +81,11 @@ module Gitlab
auto_devops_enabled: count(::ProjectAutoDevops.enabled),
auto_devops_disabled: count(::ProjectAutoDevops.disabled),
deploy_keys: count(DeployKey),
+ # rubocop: disable UsageData/LargeTable:
deployments: deployment_count(Deployment),
successful_deployments: deployment_count(Deployment.success),
failed_deployments: deployment_count(Deployment.failed),
+ # rubocop: enable UsageData/LargeTable:
environments: count(::Environment),
clusters: count(::Clusters::Cluster),
clusters_enabled: count(::Clusters::Cluster.enabled),
@@ -122,8 +121,8 @@ module Gitlab
issues_created_from_alerts: total_alert_issues,
issues_created_gitlab_alerts: issues_created_manually_from_alerts,
issues_created_manually_from_alerts: issues_created_manually_from_alerts,
- incident_issues: alert_bot_incident_count,
- alert_bot_incident_issues: alert_bot_incident_count,
+ incident_issues: count(::Issue.incident, start: issue_minimum_id, finish: issue_maximum_id),
+ alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id),
incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id),
keys: count(Key),
label_lists: count(List.label),
@@ -141,6 +140,7 @@ module Gitlab
projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id),
projects_with_terraform_states: distinct_count(::Terraform::State, :project_id),
protected_branches: count(ProtectedBranch),
+ protected_branches_except_default: count(ProtectedBranch.where.not(name: ['main', 'master', Gitlab::CurrentSettings.default_branch_name])),
releases: count(Release),
remote_mirrors: count(RemoteMirror),
personal_snippets: count(PersonalSnippet),
@@ -159,7 +159,8 @@ module Gitlab
usage_counters,
user_preferences_usage,
ingress_modsecurity_usage,
- container_expiration_policies_usage
+ container_expiration_policies_usage,
+ service_desk_counts
).tap do |data|
data[:snippets] = data[:personal_snippets] + data[:project_snippets]
end
@@ -170,9 +171,11 @@ module Gitlab
def system_usage_data_monthly
{
counts_monthly: {
+ # rubocop: disable UsageData/LargeTable:
deployments: deployment_count(Deployment.where(last_28_days_time_period)),
successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)),
failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)),
+ # rubocop: enable UsageData/LargeTable:
personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)),
project_snippets: count(ProjectSnippet.where(last_28_days_time_period))
}.tap do |data|
@@ -254,22 +257,17 @@ module Gitlab
enabled: alt_usage_data(fallback: nil) { Gitlab.config.pages.enabled },
version: alt_usage_data { Gitlab::Pages::VERSION }
},
+ container_registry_server: {
+ vendor: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_vendor },
+ version: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_version }
+ },
database: {
adapter: alt_usage_data { Gitlab::Database.adapter_name },
version: alt_usage_data { Gitlab::Database.version }
- },
- app_server: { type: app_server_type }
+ }
}
end
- def app_server_type
- Gitlab::Runtime.identify.to_s
- rescue Gitlab::Runtime::IdentificationError => e
- Gitlab::AppLogger.error(e.message)
- Gitlab::ErrorTracking.track_exception(e)
- 'unknown_app_server_type'
- end
-
def object_store_config(component)
config = alt_usage_data(fallback: nil) do
Settings[component]['object_store']
@@ -308,6 +306,7 @@ module Gitlab
Gitlab::UsageData::Topology.new.topology_usage_data
end
+ # rubocop: disable UsageData/DistinctCountByLargeForeignKey
def ingress_modsecurity_usage
##
# This method measures usage of the Modsecurity Web Application Firewall across the entire
@@ -328,6 +327,7 @@ module Gitlab
ingress_modsecurity_not_installed: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_not_installed), column)
}
end
+ # rubocop: enable UsageData/DistinctCountByLargeForeignKey
# rubocop: disable CodeReuse/ActiveRecord
def container_expiration_policies_usage
@@ -336,40 +336,46 @@ module Gitlab
finish = ::Project.maximum(:id)
results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish)
+ # rubocop: disable UsageData/LargeTable
base = ::ContainerExpirationPolicy.active
+ # rubocop: enable UsageData/LargeTable
results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish)
+ # rubocop: disable UsageData/LargeTable
%i[keep_n cadence older_than].each do |option|
::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend
results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish)
end
end
+ # rubocop: enable UsageData/LargeTable
results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish)
results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish)
results
end
- # rubocop: enable CodeReuse/ActiveRecord
- # rubocop: disable CodeReuse/ActiveRecord
def services_usage
- Service.available_services_names.without('jira').each_with_object({}) do |service_name, response|
- response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, type: "#{service_name}_service".camelize))
- end.merge(jira_usage).merge(jira_import_usage)
+ # rubocop: disable UsageData/LargeTable:
+ Service.available_services_names.each_with_object({}) do |service_name, response|
+ response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, instance: false, type: "#{service_name}_service".camelize))
+ response["templates_#{service_name}_active".to_sym] = count(Service.active.where(template: true, type: "#{service_name}_service".camelize))
+ response["instances_#{service_name}_active".to_sym] = count(Service.active.where(instance: true, type: "#{service_name}_service".camelize))
+ response["projects_inheriting_instance_#{service_name}_active".to_sym] = count(Service.active.where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize))
+ end.merge(jira_usage, jira_import_usage)
+ # rubocop: enable UsageData/LargeTable:
end
def jira_usage
# Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999
# so we can just check for subdomains of atlassian.net
-
results = {
projects_jira_server_active: 0,
- projects_jira_cloud_active: 0,
- projects_jira_active: 0
+ projects_jira_cloud_active: 0
}
- JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: BATCH_SIZE) do |services|
+ # rubocop: disable UsageData/LargeTable:
+ JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services|
counts = services.group_by do |service|
# TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404
service_url = service.data_fields&.url || (service.properties && service.properties['url'])
@@ -378,23 +384,16 @@ module Gitlab
results[:projects_jira_server_active] += counts[:server].size if counts[:server]
results[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud]
- results[:projects_jira_active] += services.size
end
-
+ # rubocop: enable UsageData/LargeTable:
results
rescue ActiveRecord::StatementInvalid
- { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK, projects_jira_active: FALLBACK }
- end
-
- def successful_deployments_with_cluster(scope)
- scope
- .joins(cluster: :deployments)
- .merge(Clusters::Cluster.enabled)
- .merge(Deployment.success)
+ { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK }
end
# rubocop: enable CodeReuse/ActiveRecord
def jira_import_usage
+ # rubocop: disable UsageData/LargeTable
finished_jira_imports = JiraImportState.finished
{
@@ -402,21 +401,28 @@ module Gitlab
jira_imports_projects_count: distinct_count(finished_jira_imports, :project_id),
jira_imports_total_imported_issues_count: alt_usage_data { JiraImportState.finished_imports_count }
}
+ # rubocop: enable UsageData/LargeTable
end
+ # rubocop: disable CodeReuse/ActiveRecord
+ # rubocop: disable UsageData/LargeTable
+ def successful_deployments_with_cluster(scope)
+ scope
+ .joins(cluster: :deployments)
+ .merge(Clusters::Cluster.enabled)
+ .merge(Deployment.success)
+ end
+ # rubocop: enable UsageData/LargeTable
+ # rubocop: enable CodeReuse/ActiveRecord
+
def user_preferences_usage
{} # augmented in EE
end
# rubocop: disable CodeReuse/ActiveRecord
def merge_requests_users(time_period)
- query =
- Event
- .where(target_type: Event::TARGET_TYPES[:merge_request].to_s)
- .where(time_period)
-
distinct_count(
- query,
+ Event.where(target_type: Event::TARGET_TYPES[:merge_request].to_s).where(time_period),
:author_id,
start: user_minimum_id,
finish: user_maximum_id
@@ -454,6 +460,7 @@ module Gitlab
end
# rubocop: disable CodeReuse/ActiveRecord
+ # rubocop: disable UsageData/LargeTable
def usage_activity_by_stage_configure(time_period)
{
clusters_applications_cert_managers: cluster_applications_user_distinct_count(::Clusters::Applications::CertManager, time_period),
@@ -474,6 +481,7 @@ module Gitlab
project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period)
}
end
+ # rubocop: enable UsageData/LargeTable
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
@@ -532,7 +540,9 @@ module Gitlab
issues: distinct_count(::Issue.where(time_period), :author_id),
notes: distinct_count(::Note.where(time_period), :author_id),
projects: distinct_count(::Project.where(time_period), :creator_id),
- todos: distinct_count(::Todo.where(time_period), :author_id)
+ todos: distinct_count(::Todo.where(time_period), :author_id),
+ service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period),
+ service_desk_issues: count(::Issue.service_desk.where(time_period))
}
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -574,21 +584,30 @@ module Gitlab
end
def analytics_unique_visits_data
- results = ::Gitlab::Analytics::UniqueVisits::TARGET_IDS.each_with_object({}) do |target_id, hash|
- hash[target_id] = redis_usage_data { unique_visit_service.weekly_unique_visits_for_target(target_id) }
+ results = ::Gitlab::Analytics::UniqueVisits.analytics_ids.each_with_object({}) do |target_id, hash|
+ hash[target_id] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target_id) }
end
- results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.weekly_unique_visits_for_any_target }
+ results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics) }
+ results['analytics_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :analytics, start_date: 4.weeks.ago.to_date, end_date: Date.current) }
{ analytics_unique_visits: results }
end
- def action_monthly_active_users(time_period)
- return {} unless Feature.enabled?(Gitlab::UsageDataCounters::TrackUniqueActions::FEATURE_FLAG)
+ def compliance_unique_visits_data
+ results = ::Gitlab::Analytics::UniqueVisits.compliance_ids.each_with_object({}) do |target_id, hash|
+ hash[target_id] = redis_usage_data { unique_visit_service.unique_visits_for(targets: target_id) }
+ end
+ results['compliance_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance) }
+ results['compliance_unique_visits_for_any_target_monthly'] = redis_usage_data { unique_visit_service.unique_visits_for(targets: :compliance, start_date: 4.weeks.ago.to_date, end_date: Date.current) }
+ { compliance_unique_visits: results }
+ end
+
+ def action_monthly_active_users(time_period)
counter = Gitlab::UsageDataCounters::TrackUniqueActions
project_count = redis_usage_data do
- counter.count_unique_events(
+ counter.count_unique(
event_action: Gitlab::UsageDataCounters::TrackUniqueActions::PUSH_ACTION,
date_from: time_period[:created_at].first,
date_to: time_period[:created_at].last
@@ -596,7 +615,7 @@ module Gitlab
end
design_count = redis_usage_data do
- counter.count_unique_events(
+ counter.count_unique(
event_action: Gitlab::UsageDataCounters::TrackUniqueActions::DESIGN_ACTION,
date_from: time_period[:created_at].first,
date_to: time_period[:created_at].last
@@ -604,7 +623,7 @@ module Gitlab
end
wiki_count = redis_usage_data do
- counter.count_unique_events(
+ counter.count_unique(
event_action: Gitlab::UsageDataCounters::TrackUniqueActions::WIKI_ACTION,
date_from: time_period[:created_at].first,
date_to: time_period[:created_at].last
@@ -620,6 +639,31 @@ module Gitlab
private
+ def distinct_count_service_desk_enabled_projects(time_period)
+ project_creator_id_start = user_minimum_id
+ project_creator_id_finish = user_maximum_id
+
+ distinct_count(::Project.service_desk_enabled.where(time_period), :creator_id, start: project_creator_id_start, finish: project_creator_id_finish) # rubocop: disable CodeReuse/ActiveRecord
+ end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def service_desk_counts
+ # rubocop: disable UsageData/LargeTable:
+ projects_with_service_desk = ::Project.where(service_desk_enabled: true)
+ # rubocop: enable UsageData/LargeTable:
+ {
+ service_desk_enabled_projects: count(projects_with_service_desk),
+ service_desk_issues: count(
+ ::Issue.where(
+ project: projects_with_service_desk,
+ author: ::User.support_bot,
+ confidential: true
+ )
+ )
+ }
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
def unique_visit_service
strong_memoize(:unique_visit_service) do
::Gitlab::Analytics::UniqueVisits.new
@@ -685,9 +729,11 @@ module Gitlab
end
# rubocop: disable CodeReuse/ActiveRecord
+ # rubocop: disable UsageData/DistinctCountByLargeForeignKey
def cluster_applications_user_distinct_count(applications, time_period)
distinct_count(applications.where(time_period).available.joins(:cluster), 'clusters.user_id')
end
+ # rubocop: enable UsageData/DistinctCountByLargeForeignKey
def clusters_user_distinct_count(clusters, time_period)
distinct_count(clusters.where(time_period), :user_id)
diff --git a/lib/gitlab/usage_data/topology.rb b/lib/gitlab/usage_data/topology.rb
index 4bca2cb07e4..edc4dc75750 100644
--- a/lib/gitlab/usage_data/topology.rb
+++ b/lib/gitlab/usage_data/topology.rb
@@ -17,6 +17,9 @@ module Gitlab
'registry' => 'registry'
}.freeze
+ # If these errors occur, all subsequent queries are likely to fail for the same error
+ TIMEOUT_ERRORS = [Errno::ETIMEDOUT, Net::OpenTimeout, Net::ReadTimeout].freeze
+
CollectionFailure = Struct.new(:query, :error) do
def to_h
{ query => error }
@@ -51,7 +54,7 @@ module Gitlab
def topology_app_requests_per_hour(client)
result = query_safely('gitlab_usage_ping:ops:rate5m', 'app_requests', fallback: nil) do |query|
- client.query(one_week_average(query)).first
+ client.query(aggregate_one_week(query)).first
end
return unless result
@@ -63,7 +66,9 @@ module Gitlab
def topology_node_data(client)
# node-level data
by_instance_mem = topology_node_memory(client)
+ by_instance_mem_utilization = topology_node_memory_utilization(client)
by_instance_cpus = topology_node_cpus(client)
+ by_instance_cpu_utilization = topology_node_cpu_utilization(client)
by_instance_uname_info = topology_node_uname_info(client)
# service-level data
by_instance_by_job_by_type_memory = topology_all_service_memory(client)
@@ -73,7 +78,9 @@ module Gitlab
@instances.map do |instance|
{
node_memory_total_bytes: by_instance_mem[instance],
+ node_memory_utilization: by_instance_mem_utilization[instance],
node_cpus: by_instance_cpus[instance],
+ node_cpu_utilization: by_instance_cpu_utilization[instance],
node_uname_info: by_instance_uname_info[instance],
node_services:
topology_node_services(
@@ -84,14 +91,26 @@ module Gitlab
end
def topology_node_memory(client)
- query_safely('gitlab_usage_ping:node_memory_total_bytes:avg', 'node_memory', fallback: {}) do |query|
- aggregate_by_instance(client, one_week_average(query))
+ query_safely('gitlab_usage_ping:node_memory_total_bytes:max', 'node_memory', fallback: {}) do |query|
+ aggregate_by_instance(client, aggregate_one_week(query, aggregation: :max))
+ end
+ end
+
+ def topology_node_memory_utilization(client)
+ query_safely('gitlab_usage_ping:node_memory_utilization:avg', 'node_memory_utilization', fallback: {}) do |query|
+ aggregate_by_instance(client, aggregate_one_week(query), transform_value: :to_f)
end
end
def topology_node_cpus(client)
query_safely('gitlab_usage_ping:node_cpus:count', 'node_cpus', fallback: {}) do |query|
- aggregate_by_instance(client, one_week_average(query))
+ aggregate_by_instance(client, aggregate_one_week(query, aggregation: :max))
+ end
+ end
+
+ def topology_node_cpu_utilization(client)
+ query_safely('gitlab_usage_ping:node_cpu_utilization:avg', 'node_cpu_utilization', fallback: {}) do |query|
+ aggregate_by_instance(client, aggregate_one_week(query), transform_value: :to_f)
end
end
@@ -114,25 +133,25 @@ module Gitlab
def topology_service_memory_rss(client)
query_safely(
'gitlab_usage_ping:node_service_process_resident_memory_bytes:avg', 'service_rss', fallback: {}
- ) { |query| aggregate_by_labels(client, one_week_average(query)) }
+ ) { |query| aggregate_by_labels(client, aggregate_one_week(query)) }
end
def topology_service_memory_uss(client)
query_safely(
'gitlab_usage_ping:node_service_process_unique_memory_bytes:avg', 'service_uss', fallback: {}
- ) { |query| aggregate_by_labels(client, one_week_average(query)) }
+ ) { |query| aggregate_by_labels(client, aggregate_one_week(query)) }
end
def topology_service_memory_pss(client)
query_safely(
'gitlab_usage_ping:node_service_process_proportional_memory_bytes:avg', 'service_pss', fallback: {}
- ) { |query| aggregate_by_labels(client, one_week_average(query)) }
+ ) { |query| aggregate_by_labels(client, aggregate_one_week(query)) }
end
def topology_all_service_process_count(client)
query_safely(
'gitlab_usage_ping:node_service_process:count', 'service_process_count', fallback: {}
- ) { |query| aggregate_by_labels(client, one_week_average(query)) }
+ ) { |query| aggregate_by_labels(client, aggregate_one_week(query)) }
end
def topology_all_service_server_types(client)
@@ -142,6 +161,11 @@ module Gitlab
end
def query_safely(query, query_name, fallback:)
+ if timeout_error_exists?
+ @failures << CollectionFailure.new(query_name, 'timeout_cancellation')
+ return fallback
+ end
+
result = yield query
return result if result.present?
@@ -153,6 +177,14 @@ module Gitlab
fallback
end
+ def timeout_error_exists?
+ timeout_error_names = TIMEOUT_ERRORS.map(&:to_s).to_set
+
+ @failures.any? do |failure|
+ timeout_error_names.include?(failure.error)
+ end
+ end
+
def topology_node_services(instance, all_process_counts, all_process_memory, all_server_types)
# returns all node service data grouped by service name as the key
instance_service_data =
@@ -160,14 +192,17 @@ module Gitlab
.deep_merge(topology_instance_service_memory(instance, all_process_memory))
.deep_merge(topology_instance_service_server_types(instance, all_server_types))
- # map to list of hashes where service names become values instead, and remove
+ # map to list of hashes where service names become values instead, and skip
# unknown services, since they might not be ours
instance_service_data.each_with_object([]) do |entry, list|
service, service_metrics = entry
- gitlab_service = JOB_TO_SERVICE_NAME[service.to_s]
- next unless gitlab_service
+ service_name = service.to_s.strip
- list << { name: gitlab_service }.merge(service_metrics)
+ if gitlab_service = JOB_TO_SERVICE_NAME[service_name]
+ list << { name: gitlab_service }.merge(service_metrics)
+ else
+ @failures << CollectionFailure.new('service_unknown', service_name)
+ end
end
end
@@ -210,7 +245,7 @@ module Gitlab
def normalize_localhost_address(instance)
ip_addr = IPAddr.new(instance)
- is_local_ip = ip_addr.loopback? || ip_addr.to_i.zero?
+ is_local_ip = ip_addr.loopback? || ip_addr.to_i == 0
is_local_ip ? 'localhost' : instance
rescue IPAddr::InvalidAddressError
@@ -228,17 +263,17 @@ module Gitlab
end
end
- def one_week_average(query)
- "avg_over_time (#{query}[1w])"
+ def aggregate_one_week(query, aggregation: :avg)
+ "#{aggregation}_over_time (#{query}[1w])"
end
- def aggregate_by_instance(client, query)
- client.aggregate(query) { |metric| normalize_and_track_instance(metric['instance']) }
+ def aggregate_by_instance(client, query, transform_value: :to_i)
+ client.aggregate(query, transform_value: transform_value) { |metric| normalize_and_track_instance(metric['instance']) }
end
# Will retain a composite key that values are mapped to
- def aggregate_by_labels(client, query)
- client.aggregate(query) do |metric|
+ def aggregate_by_labels(client, query, transform_value: :to_i)
+ client.aggregate(query, transform_value: transform_value) do |metric|
metric['instance'] = normalize_and_track_instance(metric['instance'])
metric
end
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
new file mode 100644
index 00000000000..c9c39225068
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module UsageDataCounters
+ module HLLRedisCounter
+ DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks
+ DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days
+ DEFAULT_REDIS_SLOT = ''.freeze
+
+ UnknownEvent = Class.new(StandardError)
+ UnknownAggregation = Class.new(StandardError)
+
+ KNOWN_EVENTS_PATH = 'lib/gitlab/usage_data_counters/known_events.yml'.freeze
+ ALLOWED_AGGREGATIONS = %i(daily weekly).freeze
+
+ # Track event on entity_id
+ # Increment a Redis HLL counter for unique event_name and entity_id
+ #
+ # All events should be added to know_events file lib/gitlab/usage_data_counters/known_events.yml
+ #
+ # Event example:
+ #
+ # - name: g_compliance_dashboard # Unique event name
+ # redis_slot: compliance # Optional slot name, if not defined it will use name as a slot, used for totals
+ # category: compliance # Group events in categories
+ # expiry: 29 # Optional expiration time in days, default value 29 days for daily and 6.weeks for weekly
+ # aggregation: daily # Aggregation level, keys are stored daily or weekly
+ #
+ # Usage:
+ #
+ # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event(user_id, 'g_compliance_dashboard')
+ # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current)
+ class << self
+ def track_event(entity_id, event_name, time = Time.zone.now)
+ event = event_for(event_name)
+
+ raise UnknownEvent.new("Unknown event #{event_name}") unless event.present?
+
+ Gitlab::Redis::HLL.add(key: redis_key(event, time), value: entity_id, expiry: expiry(event))
+ end
+
+ def unique_events(event_names:, start_date:, end_date:)
+ events = events_for(Array(event_names))
+
+ raise 'Events should be in same slot' unless events_in_same_slot?(events)
+ raise 'Events should be in same category' unless events_in_same_category?(events)
+ raise 'Events should have same aggregation level' unless events_same_aggregation?(events)
+
+ aggregation = events.first[:aggregation]
+
+ keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date)
+
+ Gitlab::Redis::HLL.count(keys: keys)
+ end
+
+ def events_for_category(category)
+ known_events.select { |event| event[:category] == category }.map { |event| event[:name] }
+ end
+
+ private
+
+ def keys_for_aggregation(aggregation, events:, start_date:, end_date:)
+ if aggregation.to_sym == :daily
+ daily_redis_keys(events: events, start_date: start_date, end_date: end_date)
+ else
+ weekly_redis_keys(events: events, start_date: start_date, end_date: end_date)
+ end
+ end
+
+ def known_events
+ @known_events ||= YAML.load_file(Rails.root.join(KNOWN_EVENTS_PATH)).map(&:with_indifferent_access)
+ end
+
+ def known_events_names
+ known_events.map { |event| event[:name] }
+ end
+
+ def events_in_same_slot?(events)
+ slot = events.first[:redis_slot]
+ events.all? { |event| event[:redis_slot] == slot }
+ end
+
+ def events_in_same_category?(events)
+ category = events.first[:category]
+ events.all? { |event| event[:category] == category }
+ end
+
+ def events_same_aggregation?(events)
+ aggregation = events.first[:aggregation]
+ events.all? { |event| event[:aggregation] == aggregation }
+ end
+
+ def expiry(event)
+ return event[:expiry] if event[:expiry].present?
+
+ event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH
+ end
+
+ def event_for(event_name)
+ known_events.find { |event| event[:name] == event_name }
+ end
+
+ def events_for(event_names)
+ known_events.select { |event| event_names.include?(event[:name]) }
+ end
+
+ def redis_slot(event)
+ event[:redis_slot] || DEFAULT_REDIS_SLOT
+ end
+
+ # Compose the key in order to store events daily or weekly
+ def redis_key(event, time)
+ raise UnknownEvent.new("Unknown event #{event[:name]}") unless known_events_names.include?(event[:name].to_s)
+ raise UnknownAggregation.new("Use :daily or :weekly aggregation") unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym)
+
+ slot = redis_slot(event)
+ key = if slot.present?
+ event[:name].to_s.gsub(slot, "{#{slot}}")
+ else
+ "{#{event[:name]}}"
+ end
+
+ if event[:aggregation].to_sym == :daily
+ year_day = time.strftime('%G-%j')
+ "#{year_day}-#{key}"
+ else
+ year_week = time.strftime('%G-%V')
+ "#{key}-#{year_week}"
+ end
+ end
+
+ def daily_redis_keys(events:, start_date:, end_date:)
+ (start_date.to_date..end_date.to_date).map do |date|
+ events.map { |event| redis_key(event, date) }
+ end.flatten
+ end
+
+ def weekly_redis_keys(events:, start_date:, end_date:)
+ weeks = end_date.to_date.cweek - start_date.to_date.cweek
+ weeks = 1 if weeks == 0
+
+ (0..(weeks - 1)).map do |week_increment|
+ events.map { |event| redis_key(event, start_date + week_increment * 7.days) }
+ end.flatten
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml
new file mode 100644
index 00000000000..b7e516fa8b1
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/known_events.yml
@@ -0,0 +1,88 @@
+---
+# Compliance category
+- name: g_compliance_dashboard
+ redis_slot: compliance
+ category: compliance
+ expiry: 84 # expiration time in days, equivalent to 12 weeks
+ aggregation: weekly
+- name: g_compliance_audit_events
+ category: compliance
+ redis_slot: compliance
+ expiry: 84
+ aggregation: weekly
+- name: i_compliance_audit_events
+ category: compliance
+ redis_slot: compliance
+ expiry: 84
+ aggregation: weekly
+- name: i_compliance_credential_inventory
+ category: compliance
+ redis_slot: compliance
+ expiry: 84
+ aggregation: weekly
+# Analytics category
+- name: g_analytics_contribution
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: g_analytics_insights
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: g_analytics_issues
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: g_analytics_productivity
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: g_analytics_valuestream
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: p_analytics_pipelines
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: p_analytics_code_reviews
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: p_analytics_valuestream
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: p_analytics_insights
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: p_analytics_issues
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: p_analytics_repo
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: i_analytics_cohorts
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
+- name: i_analytics_dev_ops_score
+ category: analytics
+ redis_slot: analytics
+ expiry: 84
+ aggregation: weekly
diff --git a/lib/gitlab/usage_data_counters/track_unique_actions.rb b/lib/gitlab/usage_data_counters/track_unique_actions.rb
index 9fb5a29748e..0df982572a4 100644
--- a/lib/gitlab/usage_data_counters/track_unique_actions.rb
+++ b/lib/gitlab/usage_data_counters/track_unique_actions.rb
@@ -4,7 +4,6 @@ module Gitlab
module UsageDataCounters
module TrackUniqueActions
KEY_EXPIRY_LENGTH = 29.days
- FEATURE_FLAG = :track_unique_actions
WIKI_ACTION = :wiki_action
DESIGN_ACTION = :design_action
@@ -27,24 +26,22 @@ module Gitlab
}).freeze
class << self
- def track_action(event_action:, event_target:, author_id:, time: Time.zone.now)
+ def track_event(event_action:, event_target:, author_id:, time: Time.zone.now)
return unless Gitlab::CurrentSettings.usage_ping_enabled
- return unless Feature.enabled?(FEATURE_FLAG)
return unless valid_target?(event_target)
return unless valid_action?(event_action)
transformed_target = transform_target(event_target)
transformed_action = transform_action(event_action, transformed_target)
+ target_key = key(transformed_action, time)
- add_event(transformed_action, author_id, time)
+ Gitlab::Redis::HLL.add(key: target_key, value: author_id, expiry: KEY_EXPIRY_LENGTH)
end
- def count_unique_events(event_action:, date_from:, date_to:)
+ def count_unique(event_action:, date_from:, date_to:)
keys = (date_from.to_date..date_to.to_date).map { |date| key(event_action, date) }
- Gitlab::Redis::SharedState.with do |redis|
- redis.pfcount(*keys)
- end
+ Gitlab::Redis::HLL.count(keys: keys)
end
private
@@ -69,17 +66,6 @@ module Gitlab
year_day = date.strftime('%G-%j')
"#{year_day}-{#{event_action}}"
end
-
- def add_event(event_action, author_id, date)
- target_key = key(event_action, date)
-
- Gitlab::Redis::SharedState.with do |redis|
- redis.multi do |multi|
- multi.pfadd(target_key, author_id)
- multi.expire(target_key, KEY_EXPIRY_LENGTH)
- end
- end
- end
end
end
end
diff --git a/lib/gitlab/usage_data_counters/wiki_page_counter.rb b/lib/gitlab/usage_data_counters/wiki_page_counter.rb
index 9cfe0be5bab..6c3fe842344 100644
--- a/lib/gitlab/usage_data_counters/wiki_page_counter.rb
+++ b/lib/gitlab/usage_data_counters/wiki_page_counter.rb
@@ -2,7 +2,7 @@
module Gitlab::UsageDataCounters
class WikiPageCounter < BaseCounter
- KNOWN_EVENTS = %w[create update delete].freeze
+ KNOWN_EVENTS = %w[view create update delete].freeze
PREFIX = 'wiki_pages'
end
end
diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb
index 1551548d9b4..1c6ddc2e70f 100644
--- a/lib/gitlab/user_access.rb
+++ b/lib/gitlab/user_access.rb
@@ -5,15 +5,16 @@ module Gitlab
extend Gitlab::Cache::RequestCache
request_cache_key do
- [user&.id, project&.id]
+ [user&.id, container&.to_global_id]
end
- attr_reader :user
- attr_accessor :project
+ attr_reader :user, :push_ability
+ attr_accessor :container
- def initialize(user, project: nil)
+ def initialize(user, container: nil, push_ability: :push_code)
@user = user
- @project = project
+ @container = container
+ @push_ability = push_ability
end
def can_do_action?(action)
@@ -21,7 +22,7 @@ module Gitlab
permission_cache[action] =
permission_cache.fetch(action) do
- user.can?(action, project)
+ user.can?(action, container)
end
end
@@ -42,20 +43,20 @@ module Gitlab
request_cache def can_create_tag?(ref)
return false unless can_access_git?
- if protected?(ProtectedTag, project, ref)
+ if protected?(ProtectedTag, ref)
protected_tag_accessible_to?(ref, action: :create)
else
- user.can?(:admin_tag, project)
+ user.can?(:admin_tag, container)
end
end
request_cache def can_delete_branch?(ref)
return false unless can_access_git?
- if protected?(ProtectedBranch, project, ref)
- user.can?(:push_to_delete_protected_branch, project)
+ if protected?(ProtectedBranch, ref)
+ user.can?(:push_to_delete_protected_branch, container)
else
- user.can?(:push_code, project)
+ can_push?
end
end
@@ -64,36 +65,36 @@ module Gitlab
end
request_cache def can_push_to_branch?(ref)
- return false unless can_access_git?
- return false unless project
-
- # Checking for an internal project to prevent an infinite loop:
- # https://gitlab.com/gitlab-org/gitlab/issues/36805
- if project.internal?
- return false unless user.can?(:push_code, project)
- else
- return false if !user.can?(:push_code, project) && !project.branch_allows_collaboration?(user, ref)
- end
+ return false unless can_access_git? && container && can_collaborate?(ref)
+ return true unless protected?(ProtectedBranch, ref)
- if protected?(ProtectedBranch, project, ref)
- protected_branch_accessible_to?(ref, action: :push)
- else
- true
- end
+ protected_branch_accessible_to?(ref, action: :push)
end
request_cache def can_merge_to_branch?(ref)
return false unless can_access_git?
- if protected?(ProtectedBranch, project, ref)
+ if protected?(ProtectedBranch, ref)
protected_branch_accessible_to?(ref, action: :merge)
else
- user.can?(:push_code, project)
+ can_push?
end
end
private
+ def can_push?
+ user.can?(push_ability, container)
+ end
+
+ def can_collaborate?(ref)
+ assert_project!
+
+ # Checking for an internal project or group to prevent an infinite loop:
+ # https://gitlab.com/gitlab-org/gitlab/issues/36805
+ can_push? || (!project.internal? && project.branch_allows_collaboration?(user, ref))
+ end
+
def permission_cache
@permission_cache ||= {}
end
@@ -103,6 +104,8 @@ module Gitlab
end
def protected_branch_accessible_to?(ref, action:)
+ assert_project!
+
ProtectedBranch.protected_ref_accessible_to?(
ref, user,
project: project,
@@ -111,6 +114,8 @@ module Gitlab
end
def protected_tag_accessible_to?(ref, action:)
+ assert_project!
+
ProtectedTag.protected_ref_accessible_to?(
ref, user,
project: project,
@@ -118,8 +123,22 @@ module Gitlab
protected_refs: project.protected_tags)
end
- request_cache def protected?(kind, project, refs)
+ request_cache def protected?(kind, refs)
+ assert_project!
+
kind.protected?(project, refs)
end
+
+ def project
+ container
+ end
+
+ # Any method that assumes that it is operating on a project should make this
+ # explicit by calling `#assert_project!`.
+ # TODO: remove when we make this class polymorphic enough not to care about projects
+ # See: https://gitlab.com/gitlab-org/gitlab/-/issues/227635
+ def assert_project!
+ raise "No project! #{project.inspect} is not a Project" unless project.is_a?(::Project)
+ end
end
end
diff --git a/lib/gitlab/user_access_snippet.rb b/lib/gitlab/user_access_snippet.rb
index dcd45f9350d..3d1ec800091 100644
--- a/lib/gitlab/user_access_snippet.rb
+++ b/lib/gitlab/user_access_snippet.rb
@@ -2,6 +2,7 @@
module Gitlab
class UserAccessSnippet < UserAccess
+ extend ::Gitlab::Utils::Override
extend ::Gitlab::Cache::RequestCache
# TODO: apply override check https://gitlab.com/gitlab-org/gitlab/issues/205677
@@ -9,11 +10,10 @@ module Gitlab
[user&.id, snippet&.id]
end
- attr_reader :snippet
+ alias_method :snippet, :container
def initialize(user, snippet: nil)
- @user = user
- @snippet = snippet
+ super(user, container: snippet)
@project = snippet&.project
end
@@ -43,13 +43,9 @@ module Gitlab
def can_push_to_branch?(ref)
return true if snippet_migration?
-
- super
-
return false unless snippet
- return false unless can_do_action?(:update_snippet)
- true
+ can_do_action?(:update_snippet)
end
def can_merge_to_branch?(ref)
@@ -59,5 +55,8 @@ module Gitlab
def snippet_migration?
user&.migration_bot? && snippet
end
+
+ override :project
+ attr_reader :project
end
end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index 8f5c1eda456..e2d93e7cd29 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -7,30 +7,50 @@ module Gitlab
# Ensure that the relative path will not traverse outside the base directory
# We url decode the path to avoid passing invalid paths forward in url encoded format.
- # We are ok to pass some double encoded paths to File.open since they won't resolve.
# Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580
# It also checks for ALT_SEPARATOR aka '\' (forward slash)
- def check_path_traversal!(path, allowed_absolute: false)
- path = CGI.unescape(path)
-
- if path.start_with?("..#{File::SEPARATOR}", "..#{File::ALT_SEPARATOR}") ||
- path.include?("#{File::SEPARATOR}..#{File::SEPARATOR}") ||
- path.end_with?("#{File::SEPARATOR}..") ||
- (!allowed_absolute && Pathname.new(path).absolute?)
+ def check_path_traversal!(path)
+ path = decode_path(path)
+ path_regex = /(\A(\.{1,2})\z|\A\.\.[\/\\]|[\/\\]\.\.\z|[\/\\]\.\.[\/\\]|\n)/
+ if path.match?(path_regex)
raise PathTraversalAttackError.new('Invalid path')
end
path
end
+ def allowlisted?(absolute_path, allowlist)
+ path = absolute_path.downcase
+
+ allowlist.map(&:downcase).any? do |allowed_path|
+ path.start_with?(allowed_path)
+ end
+ end
+
+ def check_allowed_absolute_path!(path, allowlist)
+ return unless Pathname.new(path).absolute?
+ return if allowlisted?(path, allowlist)
+
+ raise StandardError, "path #{path} is not allowed"
+ end
+
+ def decode_path(encoded_path)
+ decoded = CGI.unescape(encoded_path)
+ if decoded != CGI.unescape(decoded)
+ raise StandardError, "path #{encoded_path} is not allowed"
+ end
+
+ decoded
+ end
+
def force_utf8(str)
str.dup.force_encoding(Encoding::UTF_8)
end
def ensure_utf8_size(str, bytes:)
raise ArgumentError, 'Empty string provided!' if str.empty?
- raise ArgumentError, 'Negative string size provided!' if bytes.negative?
+ raise ArgumentError, 'Negative string size provided!' if bytes < 0
truncated = str.each_char.each_with_object(+'') do |char, object|
if object.bytesize + char.bytesize > bytes
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb
index 625e1076a54..36046ca14bf 100644
--- a/lib/gitlab/utils/usage_data.rb
+++ b/lib/gitlab/utils/usage_data.rb
@@ -93,7 +93,7 @@ module Gitlab
end
def with_finished_at(key, &block)
- yield.merge(key => Time.now)
+ yield.merge(key => Time.current)
end
private
diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb
index 5d241b9b4e9..9dc687f7740 100644
--- a/lib/gitlab/view/presenter/base.rb
+++ b/lib/gitlab/view/presenter/base.rb
@@ -30,6 +30,18 @@ module Gitlab
Gitlab::UrlBuilder.instance
end
+ def is_a?(type)
+ super || subject.is_a?(type)
+ end
+
+ def web_url
+ url_builder.build(subject)
+ end
+
+ def web_path
+ url_builder.build(subject, only_path: true)
+ end
+
class_methods do
def presenter?
true
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 6d935bb8828..e3b1cb3d016 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -156,6 +156,18 @@ module Gitlab
]
end
+ def send_scaled_image(location, width)
+ params = {
+ 'Location' => location,
+ 'Width' => width
+ }
+
+ [
+ SEND_DATA_HEADER,
+ "send-scaled-img:#{encode(params)}"
+ ]
+ end
+
def channel_websocket(channel)
details = {
'Channel' => {
diff --git a/lib/object_storage/config.rb b/lib/object_storage/config.rb
new file mode 100644
index 00000000000..d0777914cb5
--- /dev/null
+++ b/lib/object_storage/config.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module ObjectStorage
+ class Config
+ attr_reader :options
+
+ def initialize(options)
+ @options = options.to_hash.deep_symbolize_keys
+ end
+
+ def credentials
+ @credentials ||= options[:connection] || {}
+ end
+
+ def storage_options
+ @storage_options ||= options[:storage_options] || {}
+ end
+
+ def enabled?
+ options[:enabled]
+ end
+
+ def bucket
+ options[:remote_directory]
+ end
+
+ def consolidated_settings?
+ options.fetch(:consolidated_settings, false)
+ end
+
+ # AWS-specific options
+ def aws?
+ provider == 'AWS'
+ end
+
+ def use_iam_profile?
+ Gitlab::Utils.to_boolean(credentials[:use_iam_profile], default: false)
+ end
+
+ def use_path_style?
+ Gitlab::Utils.to_boolean(credentials[:path_style], default: false)
+ end
+
+ def server_side_encryption
+ storage_options[:server_side_encryption]
+ end
+
+ def server_side_encryption_kms_key_id
+ storage_options[:server_side_encryption_kms_key_id]
+ end
+
+ def provider
+ credentials[:provider].to_s
+ end
+ # End AWS-specific options
+
+ def google?
+ provider == 'Google'
+ end
+
+ def azure?
+ provider == 'AzureRM'
+ end
+
+ def fog_attributes
+ @fog_attributes ||= begin
+ return {} unless enabled? && aws?
+ return {} unless server_side_encryption.present?
+
+ aws_server_side_encryption_headers.compact
+ end
+ end
+
+ private
+
+ def aws_server_side_encryption_headers
+ {
+ 'x-amz-server-side-encryption' => server_side_encryption,
+ 'x-amz-server-side-encryption-aws-kms-key-id' => server_side_encryption_kms_key_id
+ }
+ end
+ end
+end
diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb
index 76f92f62e9c..90199114f2c 100644
--- a/lib/object_storage/direct_upload.rb
+++ b/lib/object_storage/direct_upload.rb
@@ -22,20 +22,20 @@ module ObjectStorage
MAXIMUM_MULTIPART_PARTS = 100
MINIMUM_MULTIPART_SIZE = 5.megabytes
- attr_reader :credentials, :bucket_name, :object_name
- attr_reader :has_length, :maximum_size, :consolidated_settings
+ attr_reader :config, :credentials, :bucket_name, :object_name
+ attr_reader :has_length, :maximum_size
- def initialize(credentials, bucket_name, object_name, has_length:, maximum_size: nil, consolidated_settings: false)
+ def initialize(config, object_name, has_length:, maximum_size: nil)
unless has_length
raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size
end
- @credentials = credentials
- @bucket_name = bucket_name
+ @config = config
+ @credentials = config.credentials
+ @bucket_name = config.bucket
@object_name = object_name
@has_length = has_length
@maximum_size = maximum_size
- @consolidated_settings = consolidated_settings
end
def to_hash
@@ -62,8 +62,16 @@ module ObjectStorage
end
def workhorse_client_hash
- return {} unless aws?
+ if config.aws?
+ workhorse_aws_hash
+ elsif config.azure?
+ workhorse_azure_hash
+ else
+ {}
+ end
+ end
+ def workhorse_aws_hash
{
UseWorkhorseClient: use_workhorse_s3_client?,
RemoteTempObjectID: object_name,
@@ -73,8 +81,25 @@ module ObjectStorage
Bucket: bucket_name,
Region: credentials[:region],
Endpoint: credentials[:endpoint],
- PathStyle: credentials.fetch(:path_style, false),
- UseIamProfile: credentials.fetch(:use_iam_profile, false)
+ PathStyle: config.use_path_style?,
+ UseIamProfile: config.use_iam_profile?,
+ ServerSideEncryption: config.server_side_encryption,
+ SSEKMSKeyID: config.server_side_encryption_kms_key_id
+ }.compact
+ }
+ }
+ end
+
+ def workhorse_azure_hash
+ {
+ # Azure requires Workhorse client because direct uploads can't
+ # use pre-signed URLs without buffering the whole file to disk.
+ UseWorkhorseClient: true,
+ RemoteTempObjectID: object_name,
+ ObjectStorage: {
+ Provider: 'AzureRM',
+ GoCloudConfig: {
+ URL: "azblob://#{bucket_name}"
}
}
}
@@ -82,7 +107,7 @@ module ObjectStorage
def use_workhorse_s3_client?
return false unless Feature.enabled?(:use_workhorse_s3_client, default_enabled: true)
- return false unless credentials.fetch(:use_iam_profile, false) || consolidated_settings
+ return false unless config.use_iam_profile? || config.consolidated_settings?
# The Golang AWS SDK does not support V2 signatures
return false unless credentials.fetch(:aws_signature_version, 4).to_i >= 4
@@ -95,7 +120,7 @@ module ObjectStorage
# Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
def get_url
- if google?
+ if config.google?
connection.get_object_https_url(bucket_name, object_name, expire_at)
else
connection.get_object_url(bucket_name, object_name, expire_at)
@@ -169,23 +194,15 @@ module ObjectStorage
].min
end
- def aws?
- provider == 'AWS'
- end
-
- def google?
- provider == 'Google'
- end
-
def requires_multipart_upload?
- aws? && !has_length
+ config.aws? && !has_length
end
def upload_id
return unless requires_multipart_upload?
strong_memoize(:upload_id) do
- new_upload = connection.initiate_multipart_upload(bucket_name, object_name)
+ new_upload = connection.initiate_multipart_upload(bucket_name, object_name, config.fog_attributes)
new_upload.body["UploadId"]
end
end
diff --git a/lib/product_analytics/event_params.rb b/lib/product_analytics/event_params.rb
index d938fe1f594..07e0bc8b43a 100644
--- a/lib/product_analytics/event_params.rb
+++ b/lib/product_analytics/event_params.rb
@@ -40,7 +40,12 @@ module ProductAnalytics
domain_userid: params['duid'],
user_fingerprint: params['fp'],
page_referrer: params['refr'],
- page_url: params['url']
+ page_url: params['url'],
+ se_category: params['se_ca'],
+ se_action: params['se_ac'],
+ se_label: params['se_la'],
+ se_property: params['se_pr'],
+ se_value: params['se_va']
}
end
diff --git a/lib/product_analytics/tracker.rb b/lib/product_analytics/tracker.rb
new file mode 100644
index 00000000000..d4a88b879f0
--- /dev/null
+++ b/lib/product_analytics/tracker.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ProductAnalytics
+ class Tracker
+ # The file is located in the /public directory
+ URL = Gitlab.config.gitlab.url + '/-/sp.js'
+
+ # The collector URL minus protocol and /i
+ COLLECTOR_URL = Gitlab.config.gitlab.url.sub(/\Ahttps?\:\/\//, '') + '/-/collector'
+ end
+end
diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb
index 08c8df9b044..0709f783f09 100644
--- a/lib/system_check/app/git_version_check.rb
+++ b/lib/system_check/app/git_version_check.rb
@@ -7,7 +7,7 @@ module SystemCheck
set_check_pass -> { "yes (#{self.current_version})" }
def self.required_version
- @required_version ||= Gitlab::VersionInfo.parse('2.22.0')
+ @required_version ||= Gitlab::VersionInfo.parse('2.24.0')
end
def self.current_version
diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb
index aca2972f287..aff0ee52e0d 100644
--- a/lib/system_check/app/redis_version_check.rb
+++ b/lib/system_check/app/redis_version_check.rb
@@ -5,8 +5,8 @@ require 'redis'
module SystemCheck
module App
class RedisVersionCheck < SystemCheck::BaseCheck
- MIN_REDIS_VERSION = '3.2.0'
- RECOMMENDED_REDIS_VERSION = '4.0.0'
+ MIN_REDIS_VERSION = '4.0.0'
+ RECOMMENDED_REDIS_VERSION = '4.0.0' # In future we may deprecate but still support Redis 4
set_name "Redis version >= #{RECOMMENDED_REDIS_VERSION}?"
@custom_error_message = ''
diff --git a/lib/system_check/sidekiq_check.rb b/lib/system_check/sidekiq_check.rb
index 2f5fc2cea89..4f1533a5b7d 100644
--- a/lib/system_check/sidekiq_check.rb
+++ b/lib/system_check/sidekiq_check.rb
@@ -32,7 +32,7 @@ module SystemCheck
def only_one_sidekiq_running
process_count = sidekiq_process_count
- return if process_count.zero?
+ return if process_count == 0
$stdout.print 'Number of Sidekiq processes ... '
diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake
index a192293fae6..31030d061f2 100644
--- a/lib/tasks/gettext.rake
+++ b/lib/tasks/gettext.rake
@@ -12,6 +12,14 @@ namespace :gettext do
)
end
+ # Disallow HTML from translatable strings
+ # See: https://docs.gitlab.com/ee/development/i18n/externalization.html#html
+ def html_todolist
+ return @html_todolist if defined?(@html_todolist)
+
+ @html_todolist = YAML.load_file(Rails.root.join('lib/gitlab/i18n/html_todo.yml'))
+ end
+
task :compile do
# See: https://gitlab.com/gitlab-org/gitlab-foss/issues/33014#note_31218998
FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot'))
@@ -54,11 +62,11 @@ namespace :gettext do
linters = files.map do |file|
locale = File.basename(File.dirname(file))
- Gitlab::I18n::PoLinter.new(file, locale)
+ Gitlab::I18n::PoLinter.new(po_path: file, html_todolist: html_todolist, locale: locale)
end
pot_file = Rails.root.join('locale/gitlab.pot')
- linters.unshift(Gitlab::I18n::PoLinter.new(pot_file))
+ linters.unshift(Gitlab::I18n::PoLinter.new(po_path: pot_file, html_todolist: html_todolist))
failed_linters = linters.select { |linter| linter.errors.any? }
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index e5e2faaa7df..b0f1ca39387 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -93,10 +93,19 @@ namespace :gitlab do
task create: :gitlab_environment do
puts_time "Dumping repositories ...".color(:blue)
+ max_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_CONCURRENCY', 1).to_i
+ max_storage_concurrency = ENV.fetch('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 1).to_i
+
if ENV["SKIP"] && ENV["SKIP"].include?("repositories")
puts_time "[SKIPPED]".color(:cyan)
+ elsif max_concurrency < 1 || max_storage_concurrency < 1
+ puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red)
+ exit 1
else
- Backup::Repository.new(progress).dump
+ Backup::Repository.new(progress).dump(
+ max_concurrency: max_concurrency,
+ max_storage_concurrency: max_storage_concurrency
+ )
puts_time "done".color(:green)
end
end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index fc55d9704d1..74cf3aad951 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -17,7 +17,7 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
command = []
_, status = Gitlab::Popen.popen(%w[which gmake])
- command << (status.zero? ? 'gmake' : 'make')
+ command << (status == 0 ? 'gmake' : 'make')
if Rails.env.test?
command.push(
diff --git a/lib/tasks/gitlab/snippets.rake b/lib/tasks/gitlab/snippets.rake
index c391cecfdbc..ed2e88692d5 100644
--- a/lib/tasks/gitlab/snippets.rake
+++ b/lib/tasks/gitlab/snippets.rake
@@ -13,7 +13,7 @@ namespace :gitlab do
raise "Please supply the list of ids through the SNIPPET_IDS env var"
end
- raise "Invalid limit value" if limit.zero?
+ raise "Invalid limit value" if limit == 0
if migration_running?
raise "There are already snippet migrations running. Please wait until they are finished."
@@ -37,7 +37,7 @@ namespace :gitlab do
def parse_snippet_ids!
ids = ENV['SNIPPET_IDS'].delete(' ').split(',').map do |id|
id.to_i.tap do |value|
- raise "Invalid id provided" if value.zero?
+ raise "Invalid id provided" if value == 0
end
end
@@ -68,10 +68,10 @@ namespace :gitlab do
# bundle exec rake gitlab:snippets:list_non_migrated LIMIT=50
desc 'GitLab | Show non migrated snippets'
task list_non_migrated: :environment do
- raise "Invalid limit value" if limit.zero?
+ raise "Invalid limit value" if limit == 0
non_migrated_count = non_migrated_snippets.count
- if non_migrated_count.zero?
+ if non_migrated_count == 0
puts "All snippets have been successfully migrated"
else
puts "There are #{non_migrated_count} snippets that haven't been migrated. Showing a batch of ids of those snippets:\n"
diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake
index 53343c8f8ff..15084a118b7 100644
--- a/lib/tasks/gitlab/workhorse.rake
+++ b/lib/tasks/gitlab/workhorse.rake
@@ -15,7 +15,7 @@ namespace :gitlab do
checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
_, status = Gitlab::Popen.popen(%w[which gmake])
- command = status.zero? ? 'gmake' : 'make'
+ command = status == 0 ? 'gmake' : 'make'
Dir.chdir(args.dir) do
run_command!([command])