summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-04-20 23:50:22 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2021-04-20 23:50:22 +0000
commit9dc93a4519d9d5d7be48ff274127136236a3adb3 (patch)
tree70467ae3692a0e35e5ea56bcb803eb512a10bedb /lib
parent4b0f34b6d759d6299322b3a54453e930c6121ff0 (diff)
downloadgitlab-ce-9dc93a4519d9d5d7be48ff274127136236a3adb3.tar.gz
Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43
Diffstat (limited to 'lib')
-rw-r--r--lib/api/admin/sidekiq.rb4
-rw-r--r--lib/api/api.rb4
-rw-r--r--lib/api/api_guard.rb4
-rw-r--r--lib/api/applications.rb2
-rw-r--r--lib/api/ci/pipelines.rb2
-rw-r--r--lib/api/ci/runner.rb4
-rw-r--r--lib/api/commits.rb6
-rw-r--r--lib/api/composer_packages.rb2
-rw-r--r--lib/api/concerns/packages/nuget_endpoints.rb2
-rw-r--r--lib/api/deploy_keys.rb2
-rw-r--r--lib/api/deployments.rb4
-rw-r--r--lib/api/entities/basic_project_details.rb9
-rw-r--r--lib/api/entities/clusters/agent.rb12
-rw-r--r--lib/api/entities/email.rb2
-rw-r--r--lib/api/entities/job_request/job_info.rb2
-rw-r--r--lib/api/entities/namespace_existence.rb9
-rw-r--r--lib/api/entities/project.rb5
-rw-r--r--lib/api/entities/project_import_failed_relation.rb6
-rw-r--r--lib/api/entities/user.rb4
-rw-r--r--lib/api/entities/user_preferences.rb9
-rw-r--r--lib/api/entities/user_public.rb1
-rw-r--r--lib/api/environments.rb2
-rw-r--r--lib/api/files.rb4
-rw-r--r--lib/api/generic_packages.rb4
-rw-r--r--lib/api/group_variables.rb36
-rw-r--r--lib/api/groups.rb10
-rw-r--r--lib/api/helpers.rb22
-rw-r--r--lib/api/helpers/authentication.rb5
-rw-r--r--lib/api/helpers/caching.rb137
-rw-r--r--lib/api/helpers/common_helpers.rb4
-rw-r--r--lib/api/helpers/graphql_helpers.rb4
-rw-r--r--lib/api/helpers/notes_helpers.rb6
-rw-r--r--lib/api/helpers/packages/conan/api_helpers.rb23
-rw-r--r--lib/api/helpers/packages/dependency_proxy_helpers.rb2
-rw-r--r--lib/api/helpers/packages_helpers.rb3
-rw-r--r--lib/api/helpers/runner.rb9
-rw-r--r--lib/api/helpers/services_helpers.rb10
-rw-r--r--lib/api/helpers/variables_helpers.rb27
-rw-r--r--lib/api/internal/base.rb61
-rw-r--r--lib/api/internal/kubernetes.rb6
-rw-r--r--lib/api/invitations.rb6
-rw-r--r--lib/api/issue_links.rb5
-rw-r--r--lib/api/issues.rb6
-rw-r--r--lib/api/jobs.rb30
-rw-r--r--lib/api/maven_packages.rb43
-rw-r--r--lib/api/members.rb8
-rw-r--r--lib/api/merge_request_diffs.rb2
-rw-r--r--lib/api/merge_requests.rb67
-rw-r--r--lib/api/milestone_responses.rb2
-rw-r--r--lib/api/namespaces.rb17
-rw-r--r--lib/api/project_import.rb2
-rw-r--r--lib/api/projects.rb53
-rw-r--r--lib/api/repositories.rb15
-rw-r--r--lib/api/resource_access_tokens.rb6
-rw-r--r--lib/api/rubygem_packages.rb12
-rw-r--r--lib/api/search.rb18
-rw-r--r--lib/api/settings.rb1
-rw-r--r--lib/api/tags.rb8
-rw-r--r--lib/api/triggers.rb2
-rw-r--r--lib/api/usage_data.rb17
-rw-r--r--lib/api/usage_data_non_sql_metrics.rb27
-rw-r--r--lib/api/usage_data_queries.rb27
-rw-r--r--lib/api/users.rb27
-rw-r--r--lib/api/v3/github.rb12
-rw-r--r--lib/api/variables.rb30
-rw-r--r--lib/atlassian/jira_connect/client.rb4
-rw-r--r--lib/atlassian/jira_connect/serializers/build_entity.rb10
-rw-r--r--lib/backup/repositories.rb2
-rw-r--r--lib/banzai/filter/abstract_reference_filter.rb446
-rw-r--r--lib/banzai/filter/alert_reference_filter.rb29
-rw-r--r--lib/banzai/filter/autolink_filter.rb2
-rw-r--r--lib/banzai/filter/commit_range_reference_filter.rb46
-rw-r--r--lib/banzai/filter/commit_reference_filter.rb86
-rw-r--r--lib/banzai/filter/commit_trailers_filter.rb2
-rw-r--r--lib/banzai/filter/design_reference_filter.rb107
-rw-r--r--lib/banzai/filter/epic_reference_filter.rb22
-rw-r--r--lib/banzai/filter/external_issue_reference_filter.rb118
-rw-r--r--lib/banzai/filter/feature_flag_reference_filter.rb33
-rw-r--r--lib/banzai/filter/gollum_tags_filter.rb13
-rw-r--r--lib/banzai/filter/issuable_reference_filter.rb19
-rw-r--r--lib/banzai/filter/issue_reference_filter.rb57
-rw-r--r--lib/banzai/filter/iteration_reference_filter.rb16
-rw-r--r--lib/banzai/filter/label_reference_filter.rb129
-rw-r--r--lib/banzai/filter/math_filter.rb2
-rw-r--r--lib/banzai/filter/merge_request_reference_filter.rb97
-rw-r--r--lib/banzai/filter/milestone_reference_filter.rb138
-rw-r--r--lib/banzai/filter/project_reference_filter.rb117
-rw-r--r--lib/banzai/filter/reference_filter.rb215
-rw-r--r--lib/banzai/filter/references/abstract_reference_filter.rb448
-rw-r--r--lib/banzai/filter/references/alert_reference_filter.rb31
-rw-r--r--lib/banzai/filter/references/commit_range_reference_filter.rb48
-rw-r--r--lib/banzai/filter/references/commit_reference_filter.rb88
-rw-r--r--lib/banzai/filter/references/design_reference_filter.rb109
-rw-r--r--lib/banzai/filter/references/epic_reference_filter.rb24
-rw-r--r--lib/banzai/filter/references/external_issue_reference_filter.rb120
-rw-r--r--lib/banzai/filter/references/feature_flag_reference_filter.rb35
-rw-r--r--lib/banzai/filter/references/issuable_reference_filter.rb21
-rw-r--r--lib/banzai/filter/references/issue_reference_filter.rb59
-rw-r--r--lib/banzai/filter/references/iteration_reference_filter.rb18
-rw-r--r--lib/banzai/filter/references/label_reference_filter.rb132
-rw-r--r--lib/banzai/filter/references/merge_request_reference_filter.rb99
-rw-r--r--lib/banzai/filter/references/milestone_reference_filter.rb140
-rw-r--r--lib/banzai/filter/references/project_reference_filter.rb119
-rw-r--r--lib/banzai/filter/references/reference_filter.rb217
-rw-r--r--lib/banzai/filter/references/snippet_reference_filter.rb31
-rw-r--r--lib/banzai/filter/references/user_reference_filter.rb182
-rw-r--r--lib/banzai/filter/references/vulnerability_reference_filter.rb24
-rw-r--r--lib/banzai/filter/repository_link_filter.rb2
-rw-r--r--lib/banzai/filter/snippet_reference_filter.rb29
-rw-r--r--lib/banzai/filter/spaced_link_filter.rb2
-rw-r--r--lib/banzai/filter/suggestion_filter.rb2
-rw-r--r--lib/banzai/filter/syntax_highlight_filter.rb2
-rw-r--r--lib/banzai/filter/user_reference_filter.rb180
-rw-r--r--lib/banzai/filter/vulnerability_reference_filter.rb22
-rw-r--r--lib/banzai/filter/wiki_link_filter/rewriter.rb5
-rw-r--r--lib/banzai/pipeline/gfm_pipeline.rb26
-rw-r--r--lib/banzai/pipeline/label_pipeline.rb2
-rw-r--r--lib/banzai/pipeline/single_line_pipeline.rb18
-rw-r--r--lib/banzai/pipeline/wiki_pipeline.rb2
-rw-r--r--lib/bulk_imports/clients/http.rb8
-rw-r--r--lib/bulk_imports/common/extractors/rest_extractor.rb45
-rw-r--r--lib/bulk_imports/common/transformers/user_reference_transformer.rb2
-rw-r--r--lib/bulk_imports/groups/graphql/get_labels_query.rb9
-rw-r--r--lib/bulk_imports/groups/graphql/get_members_query.rb9
-rw-r--r--lib/bulk_imports/groups/graphql/get_milestones_query.rb10
-rw-r--r--lib/bulk_imports/groups/pipelines/badges_pipeline.rb32
-rw-r--r--lib/bulk_imports/groups/pipelines/entity_finisher.rb35
-rw-r--r--lib/bulk_imports/groups/pipelines/labels_pipeline.rb12
-rw-r--r--lib/bulk_imports/groups/pipelines/members_pipeline.rb12
-rw-r--r--lib/bulk_imports/groups/pipelines/milestones_pipeline.rb12
-rw-r--r--lib/bulk_imports/groups/rest/get_badges_query.rb22
-rw-r--r--lib/bulk_imports/importers/group_importer.rb35
-rw-r--r--lib/bulk_imports/pipeline.rb4
-rw-r--r--lib/bulk_imports/pipeline/context.rb22
-rw-r--r--lib/bulk_imports/pipeline/extracted_data.rb7
-rw-r--r--lib/bulk_imports/pipeline/runner.rb42
-rw-r--r--lib/constraints/admin_constrainer.rb2
-rw-r--r--lib/container_registry/config.rb3
-rw-r--r--lib/container_registry/tag.rb3
-rw-r--r--lib/csv_builder.rb2
-rw-r--r--lib/declarative_policy/preferred_scope.rb3
-rw-r--r--lib/error_tracking/sentry_client/issue.rb4
-rw-r--r--lib/feature.rb4
-rw-r--r--lib/feature/active_support_cache_store_adapter.rb27
-rw-r--r--lib/file_size_validator.rb3
-rw-r--r--lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb26
-rw-r--r--lib/generators/gitlab/usage_metric_definition_generator.rb11
-rw-r--r--lib/gitlab.rb11
-rw-r--r--lib/gitlab/alert_management/payload/base.rb2
-rw-r--r--lib/gitlab/analytics/cycle_analytics/records_fetcher.rb54
-rw-r--r--lib/gitlab/analytics/unique_visits.rb4
-rw-r--r--lib/gitlab/application_context.rb7
-rw-r--r--lib/gitlab/auth/auth_finders.rb4
-rw-r--r--lib/gitlab/auth/ldap/adapter.rb2
-rw-r--r--lib/gitlab/auth/saml/origin_validator.rb2
-rw-r--r--lib/gitlab/background_migration/backfill_design_internal_ids.rb4
-rw-r--r--lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb2
-rw-r--r--lib/gitlab/background_migration/copy_column_using_background_migration_job.rb8
-rw-r--r--lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb2
-rw-r--r--lib/gitlab/background_migration/fix_projects_without_project_feature.rb2
-rw-r--r--lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb4
-rw-r--r--lib/gitlab/background_migration/fix_user_namespace_names.rb4
-rw-r--r--lib/gitlab/background_migration/fix_user_project_route_names.rb2
-rw-r--r--lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb19
-rw-r--r--lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb2
-rw-r--r--lib/gitlab/background_migration/populate_has_vulnerabilities.rb30
-rw-r--r--lib/gitlab/background_migration/populate_merge_request_assignees_table.rb2
-rw-r--r--lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb2
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb25
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb25
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb2
-rw-r--r--lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb60
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb20
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb4
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb2
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/design_management/design.rb3
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/epic.rb6
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/group.rb2
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/merge_request.rb7
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb1
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/namespace.rb15
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/note.rb4
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/project.rb48
-rw-r--r--lib/gitlab/background_migration/user_mentions/models/user.rb37
-rw-r--r--lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb4
-rw-r--r--lib/gitlab/batch_pop_queueing.rb3
-rw-r--r--lib/gitlab/bullet.rb16
-rw-r--r--lib/gitlab/bullet/exclusions.rb37
-rw-r--r--lib/gitlab/cache/ci/project_pipeline_status.rb4
-rw-r--r--lib/gitlab/changelog/config.rb24
-rw-r--r--lib/gitlab/chaos.rb6
-rw-r--r--lib/gitlab/ci/build/artifacts/metadata.rb4
-rw-r--r--lib/gitlab/ci/config.rb10
-rw-r--r--lib/gitlab/ci/config/entry/cache.rb6
-rw-r--r--lib/gitlab/ci/config/entry/processable.rb12
-rw-r--r--lib/gitlab/ci/config/entry/product/variables.rb3
-rw-r--r--lib/gitlab/ci/config/entry/variables.rb4
-rw-r--r--lib/gitlab/ci/config/external/mapper.rb12
-rw-r--r--lib/gitlab/ci/config/normalizer/matrix_strategy.rb5
-rw-r--r--lib/gitlab/ci/features.rb12
-rw-r--r--lib/gitlab/ci/jwt.rb16
-rw-r--r--lib/gitlab/ci/pipeline/chain/command.rb9
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/process.rb1
-rw-r--r--lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb22
-rw-r--r--lib/gitlab/ci/pipeline/chain/helpers.rb20
-rw-r--r--lib/gitlab/ci/pipeline/chain/metrics.rb2
-rw-r--r--lib/gitlab/ci/pipeline/chain/pipeline/process.rb4
-rw-r--r--lib/gitlab/ci/pipeline/chain/seed.rb20
-rw-r--r--lib/gitlab/ci/pipeline/chain/validate/external.rb85
-rw-r--r--lib/gitlab/ci/pipeline/metrics.rb72
-rw-r--r--lib/gitlab/ci/pipeline/seed/build.rb19
-rw-r--r--lib/gitlab/ci/pipeline/seed/context.rb18
-rw-r--r--lib/gitlab/ci/pipeline/seed/pipeline.rb6
-rw-r--r--lib/gitlab/ci/pipeline/seed/stage.rb7
-rw-r--r--lib/gitlab/ci/queue/metrics.rb35
-rw-r--r--lib/gitlab/ci/reports/codequality_reports.rb12
-rw-r--r--lib/gitlab/ci/reports/codequality_reports_comparer.rb5
-rw-r--r--lib/gitlab/ci/reports/test_failure_history.rb18
-rw-r--r--lib/gitlab/ci/runner_instructions.rb31
-rw-r--r--lib/gitlab/ci/status/build/failed.rb4
-rw-r--r--lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml7
-rw-r--r--lib/gitlab/ci/templates/Docker.gitlab-ci.yml42
-rw-r--r--lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml9
-rw-r--r--lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml91
-rw-r--r--lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml37
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml12
-rw-r--r--lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml270
-rw-r--r--lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml41
-rw-r--r--lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml99
-rw-r--r--lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml9
-rw-r--r--lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml25
-rw-r--r--lib/gitlab/ci/trace.rb33
-rw-r--r--lib/gitlab/ci/variables/helpers.rb16
-rw-r--r--lib/gitlab/ci/yaml_processor/result.rb17
-rw-r--r--lib/gitlab/composer/version_index.rb22
-rw-r--r--lib/gitlab/conan_token.rb2
-rw-r--r--lib/gitlab/contributor.rb4
-rw-r--r--lib/gitlab/crypto_helper.rb28
-rw-r--r--lib/gitlab/data_builder/build.rb6
-rw-r--r--lib/gitlab/data_builder/pipeline.rb1
-rw-r--r--lib/gitlab/database/as_with_materialized.rb36
-rw-r--r--lib/gitlab/database/background_migration/batch_metrics.rb33
-rw-r--r--lib/gitlab/database/background_migration/batched_migration.rb20
-rw-r--r--lib/gitlab/database/background_migration/batched_migration_runner.rb (renamed from lib/gitlab/database/background_migration/scheduler.rb)38
-rw-r--r--lib/gitlab/database/background_migration/batched_migration_wrapper.rb73
-rw-r--r--lib/gitlab/database/batch_count.rb39
-rw-r--r--lib/gitlab/database/bulk_update.rb2
-rw-r--r--lib/gitlab/database/count/reltuples_count_strategy.rb6
-rw-r--r--lib/gitlab/database/loose_index_scan_distinct_count.rb102
-rw-r--r--lib/gitlab/database/migration_helpers.rb160
-rw-r--r--lib/gitlab/database/migrations/background_migration_helpers.rb21
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb74
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb32
-rw-r--r--lib/gitlab/database/pg_class.rb23
-rw-r--r--lib/gitlab/database/postgres_hll/batch_distinct_counter.rb31
-rw-r--r--lib/gitlab/database/similarity_score.rb2
-rw-r--r--lib/gitlab/database/unidirectional_copy_trigger.rb97
-rw-r--r--lib/gitlab/diff/highlight.rb106
-rw-r--r--lib/gitlab/diff/highlight_cache.rb7
-rw-r--r--lib/gitlab/diff/inline_diff.rb1
-rw-r--r--lib/gitlab/diff/line.rb25
-rw-r--r--lib/gitlab/diff/suggestions_parser.rb7
-rw-r--r--lib/gitlab/downtime_check.rb73
-rw-r--r--lib/gitlab/downtime_check/message.rb41
-rw-r--r--lib/gitlab/error_tracking.rb10
-rw-r--r--lib/gitlab/error_tracking/processor/context_payload_processor.rb12
-rw-r--r--lib/gitlab/error_tracking/processor/grpc_error_processor.rb166
-rw-r--r--lib/gitlab/error_tracking/processor/sidekiq_processor.rb71
-rw-r--r--lib/gitlab/exclusive_lease.rb4
-rw-r--r--lib/gitlab/experimentation.rb4
-rw-r--r--lib/gitlab/external_authorization/access.rb3
-rw-r--r--lib/gitlab/external_authorization/cache.rb3
-rw-r--r--lib/gitlab/external_authorization/client.rb23
-rw-r--r--lib/gitlab/fogbugz_import/importer.rb6
-rw-r--r--lib/gitlab/git/blame.rb6
-rw-r--r--lib/gitlab/git/commit.rb2
-rw-r--r--lib/gitlab/git/diff_collection.rb49
-rw-r--r--lib/gitlab/git/merge_base.rb3
-rw-r--r--lib/gitlab/git/patches/commit_patches.rb5
-rw-r--r--lib/gitlab/git/repository.rb8
-rw-r--r--lib/gitlab/git/tag.rb4
-rw-r--r--lib/gitlab/git/wiki.rb13
-rw-r--r--lib/gitlab/git/wiki_file.rb24
-rw-r--r--lib/gitlab/git_access.rb5
-rw-r--r--lib/gitlab/gitaly_client.rb2
-rw-r--r--lib/gitlab/gitaly_client/attributes_bag.rb2
-rw-r--r--lib/gitlab/gitaly_client/blob_service.rb55
-rw-r--r--lib/gitlab/gitaly_client/call.rb14
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb2
-rw-r--r--lib/gitlab/gitaly_client/operation_service.rb2
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb5
-rw-r--r--lib/gitlab/gitaly_client/storage_settings.rb2
-rw-r--r--lib/gitlab/gitaly_client/wiki_file.rb11
-rw-r--r--lib/gitlab/gitaly_client/wiki_service.rb26
-rw-r--r--lib/gitlab/golang.rb29
-rw-r--r--lib/gitlab/gon_helper.rb2
-rw-r--r--lib/gitlab/grape_logging/loggers/context_logger.rb2
-rw-r--r--lib/gitlab/graphql/authorize.rb15
-rw-r--r--lib/gitlab/graphql/authorize/authorize_field_service.rb147
-rw-r--r--lib/gitlab/graphql/authorize/authorize_resource.rb44
-rw-r--r--lib/gitlab/graphql/authorize/connection_filter_extension.rb65
-rw-r--r--lib/gitlab/graphql/authorize/instrumentation.rb21
-rw-r--r--lib/gitlab/graphql/authorize/object_authorization.rb32
-rw-r--r--lib/gitlab/graphql/deprecation.rb116
-rw-r--r--lib/gitlab/graphql/docs/helper.rb134
-rw-r--r--lib/gitlab/graphql/docs/renderer.rb7
-rw-r--r--lib/gitlab/graphql/docs/templates/default.md.haml18
-rw-r--r--lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb3
-rw-r--r--lib/gitlab/graphql/loaders/batch_model_loader.rb3
-rw-r--r--lib/gitlab/graphql/loaders/full_path_model_loader.rb3
-rw-r--r--lib/gitlab/graphql/negatable_arguments.rb53
-rw-r--r--lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb6
-rw-r--r--lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb4
-rw-r--r--lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb4
-rw-r--r--lib/gitlab/graphql/pagination/keyset/query_builder.rb5
-rw-r--r--lib/gitlab/graphql/queries.rb4
-rw-r--r--lib/gitlab/graphql/query_analyzers/logger_analyzer.rb7
-rw-r--r--lib/gitlab/health_checks/gitaly_check.rb2
-rw-r--r--lib/gitlab/highlight.rb14
-rw-r--r--lib/gitlab/hook_data/user_builder.rb53
-rw-r--r--lib/gitlab/http_connection_adapter.rb14
-rw-r--r--lib/gitlab/import_export/base/relation_factory.rb2
-rw-r--r--lib/gitlab/import_export/project/import_export.yml2
-rw-r--r--lib/gitlab/import_export/uploads_manager.rb2
-rw-r--r--lib/gitlab/import_sources.rb2
-rw-r--r--lib/gitlab/instrumentation_helper.rb18
-rw-r--r--lib/gitlab/issuables_count_for_state.rb2
-rw-r--r--lib/gitlab/jira/dvcs.rb4
-rw-r--r--lib/gitlab/json.rb33
-rw-r--r--lib/gitlab/kas.rb2
-rw-r--r--lib/gitlab/kubernetes/deployment.rb2
-rw-r--r--lib/gitlab/language_detection.rb5
-rw-r--r--lib/gitlab/manifest_import/manifest.rb8
-rw-r--r--lib/gitlab/marker_range.rb6
-rw-r--r--lib/gitlab/markup_helper.rb2
-rw-r--r--lib/gitlab/metrics/background_transaction.rb20
-rw-r--r--lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb8
-rw-r--r--lib/gitlab/metrics/samplers/database_sampler.rb4
-rw-r--r--lib/gitlab/metrics/subscribers/active_record.rb21
-rw-r--r--lib/gitlab/metrics/subscribers/external_http.rb5
-rw-r--r--lib/gitlab/middleware/multipart.rb2
-rw-r--r--lib/gitlab/middleware/rack_multipart_tempfile_factory.rb25
-rw-r--r--lib/gitlab/middleware/same_site_cookies.rb2
-rw-r--r--lib/gitlab/object_hierarchy.rb46
-rw-r--r--lib/gitlab/pages.rb2
-rw-r--r--lib/gitlab/pages/migration_helper.rb53
-rw-r--r--lib/gitlab/pages/settings.rb22
-rw-r--r--lib/gitlab/pages/stores/local_store.rb15
-rw-r--r--lib/gitlab/pages_transfer.rb4
-rw-r--r--lib/gitlab/pagination/keyset/order.rb8
-rw-r--r--lib/gitlab/pagination/offset_header_builder.rb21
-rw-r--r--lib/gitlab/performance_bar/stats.rb15
-rw-r--r--lib/gitlab/phabricator_import.rb2
-rw-r--r--lib/gitlab/phabricator_import/issues/importer.rb3
-rw-r--r--lib/gitlab/phabricator_import/issues/task_importer.rb3
-rw-r--r--lib/gitlab/phabricator_import/project_creator.rb13
-rw-r--r--lib/gitlab/phabricator_import/user_finder.rb3
-rw-r--r--lib/gitlab/project_template.rb6
-rw-r--r--lib/gitlab/prometheus/adapter.rb4
-rw-r--r--lib/gitlab/prometheus/queries/matched_metric_query.rb2
-rw-r--r--lib/gitlab/prometheus_client.rb2
-rw-r--r--lib/gitlab/push_options.rb6
-rw-r--r--lib/gitlab/query_limiting.rb22
-rw-r--r--lib/gitlab/query_limiting/transaction.rb9
-rw-r--r--lib/gitlab/quick_actions/command_definition.rb11
-rw-r--r--lib/gitlab/quick_actions/issue_and_merge_request_actions.rb2
-rw-r--r--lib/gitlab/rack_attack/request.rb6
-rw-r--r--lib/gitlab/regex.rb6
-rw-r--r--lib/gitlab/relative_positioning/closed_range.rb3
-rw-r--r--lib/gitlab/relative_positioning/gap.rb3
-rw-r--r--lib/gitlab/repository_cache_adapter.rb9
-rw-r--r--lib/gitlab/repository_hash_cache.rb2
-rw-r--r--lib/gitlab/repository_set_cache.rb30
-rw-r--r--lib/gitlab/search_context.rb7
-rw-r--r--lib/gitlab/set_cache.rb13
-rw-r--r--lib/gitlab/setup_helper.rb12
-rw-r--r--lib/gitlab/sidekiq_cluster/cli.rb14
-rw-r--r--lib/gitlab/sidekiq_config.rb17
-rw-r--r--lib/gitlab/sidekiq_config/cli_methods.rb90
-rw-r--r--lib/gitlab/sidekiq_config/worker_matcher.rb86
-rw-r--r--lib/gitlab/sidekiq_logging/structured_logger.rb4
-rw-r--r--lib/gitlab/sidekiq_middleware.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/admin_mode/client.rb3
-rw-r--r--lib/gitlab/sidekiq_middleware/admin_mode/server.rb3
-rw-r--r--lib/gitlab/sidekiq_middleware/instrumentation_logger.rb23
-rw-r--r--lib/gitlab/sidekiq_middleware/metrics_helper.rb15
-rw-r--r--lib/gitlab/sidekiq_middleware/server_metrics.rb47
-rw-r--r--lib/gitlab/sidekiq_queue.rb2
-rw-r--r--lib/gitlab/slash_commands/base_command.rb4
-rw-r--r--lib/gitlab/slash_commands/presenters/issue_new.rb14
-rw-r--r--lib/gitlab/slash_commands/run.rb2
-rw-r--r--lib/gitlab/slug/environment.rb17
-rw-r--r--lib/gitlab/sql/cte.rb11
-rw-r--r--lib/gitlab/sql/recursive_cte.rb6
-rw-r--r--lib/gitlab/sql/set_operator.rb12
-rw-r--r--lib/gitlab/sql/union.rb4
-rw-r--r--lib/gitlab/static_site_editor/config/file_config.rb2
-rw-r--r--lib/gitlab/subscription_portal.rb7
-rw-r--r--lib/gitlab/template/base_template.rb4
-rw-r--r--lib/gitlab/tracking.rb25
-rw-r--r--lib/gitlab/tracking/destinations/snowplow.rb7
-rw-r--r--lib/gitlab/tracking/standard_context.rb13
-rw-r--r--lib/gitlab/untrusted_regexp.rb4
-rw-r--r--lib/gitlab/updated_notes_paginator.rb4
-rw-r--r--lib/gitlab/usage/docs/helper.rb4
-rw-r--r--lib/gitlab/usage/docs/templates/default.md.haml3
-rw-r--r--lib/gitlab/usage/metric_definition.rb32
-rw-r--r--lib/gitlab/usage/metrics/aggregates/aggregate.rb2
-rw-r--r--lib/gitlab/usage/metrics/names_suggestions/generator.rb155
-rw-r--r--lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb74
-rw-r--r--lib/gitlab/usage_data.rb92
-rw-r--r--lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml108
-rw-r--r--lib/gitlab/usage_data_counters/aggregated_metrics/common.yml72
-rw-r--r--lib/gitlab/usage_data_counters/base_counter.rb4
-rw-r--r--lib/gitlab/usage_data_counters/ci_template_unique_counter.rb4
-rw-r--r--lib/gitlab/usage_data_counters/hll_redis_counter.rb5
-rw-r--r--lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb112
-rw-r--r--lib/gitlab/usage_data_counters/known_events/ci_templates.yml15
-rw-r--r--lib/gitlab/usage_data_counters/known_events/common.yml42
-rw-r--r--lib/gitlab/usage_data_counters/known_events/epic_events.yml142
-rw-r--r--lib/gitlab/usage_data_counters/note_counter.rb6
-rw-r--r--lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb2
-rw-r--r--lib/gitlab/usage_data_non_sql_metrics.rb35
-rw-r--r--lib/gitlab/usage_data_queries.rb14
-rw-r--r--lib/gitlab/utils.rb2
-rw-r--r--lib/gitlab/utils/usage_data.rb15
-rw-r--r--lib/gitlab/uuid.rb4
-rw-r--r--lib/gitlab/web_ide/config/entry/terminal.rb18
-rw-r--r--lib/gitlab/word_diff/chunk_collection.rb21
-rw-r--r--lib/gitlab/word_diff/parser.rb2
-rw-r--r--lib/learn_gitlab.rb6
-rw-r--r--lib/peek/views/active_record.rb30
-rw-r--r--lib/quality/test_level.rb142
-rw-r--r--lib/rouge/formatters/html_gitlab.rb5
-rw-r--r--lib/spam/concerns/has_spam_action_response_fields.rb9
-rw-r--r--lib/support/nginx/gitlab4
-rw-r--r--lib/support/nginx/gitlab-ssl4
-rw-r--r--lib/system_check/app/git_version_check.rb2
-rw-r--r--lib/system_check/app/redis_version_check.rb4
-rw-r--r--lib/tasks/brakeman.rake13
-rw-r--r--lib/tasks/cache.rake2
-rw-r--r--lib/tasks/db_obsolete_ignored_columns.rake2
-rw-r--r--lib/tasks/downtime_check.rake14
-rw-r--r--lib/tasks/gitlab/db.rake19
-rw-r--r--lib/tasks/gitlab/docs/redirect.rake57
-rw-r--r--lib/tasks/gitlab/gitaly.rake31
-rw-r--r--lib/tasks/gitlab/graphql.rake4
-rw-r--r--lib/tasks/gitlab/pages.rake40
-rw-r--r--lib/tasks/gitlab/test.rake17
-rw-r--r--lib/tasks/spec.rake20
-rw-r--r--lib/tasks/test.rake13
456 files changed, 7496 insertions, 4352 deletions
diff --git a/lib/api/admin/sidekiq.rb b/lib/api/admin/sidekiq.rb
index 7e561783685..d91d4a0d4d5 100644
--- a/lib/api/admin/sidekiq.rb
+++ b/lib/api/admin/sidekiq.rb
@@ -12,11 +12,11 @@ module API
namespace 'queues' do
desc 'Drop jobs matching the given metadata from the Sidekiq queue'
params do
- Labkit::Context::KNOWN_KEYS.each do |key|
+ Gitlab::ApplicationContext::KNOWN_KEYS.each do |key|
optional key, type: String, allow_blank: false
end
- at_least_one_of(*Labkit::Context::KNOWN_KEYS)
+ at_least_one_of(*Gitlab::ApplicationContext::KNOWN_KEYS)
end
delete ':queue_name' do
result =
diff --git a/lib/api/api.rb b/lib/api/api.rb
index f83a36068dd..a287ffbfcd8 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -59,7 +59,7 @@ module API
project: -> { @project },
namespace: -> { @group },
runner: -> { @current_runner || @runner },
- caller_id: route.origin,
+ caller_id: api_endpoint.endpoint_id,
remote_ip: request.ip,
feature_category: feature_category
)
@@ -293,6 +293,8 @@ module API
mount ::API::Triggers
mount ::API::Unleash
mount ::API::UsageData
+ mount ::API::UsageDataQueries
+ mount ::API::UsageDataNonSqlMetrics
mount ::API::UserCounts
mount ::API::Users
mount ::API::Variables
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index 8641271f2df..8822a30d4a1 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -55,7 +55,7 @@ module API
user = find_user_from_sources
return unless user
- if user.is_a?(User) && Feature.enabled?(:user_mode_in_session)
+ if user.is_a?(User) && Gitlab::CurrentSettings.admin_mode
# Sessions are enforced to be unavailable for API calls, so ignore them for admin mode
Gitlab::Auth::CurrentUserMode.bypass_session!(user.id)
end
@@ -236,7 +236,7 @@ module API
def after
# Use a Grape middleware since the Grape `after` blocks might run
# before we are finished rendering the `Grape::Entity` classes
- Gitlab::Auth::CurrentUserMode.reset_bypass_session! if Feature.enabled?(:user_mode_in_session)
+ Gitlab::Auth::CurrentUserMode.reset_bypass_session! if Gitlab::CurrentSettings.admin_mode
# Explicit nil is needed or the api call return value will be overwritten
nil
diff --git a/lib/api/applications.rb b/lib/api/applications.rb
index b883f83cc19..be482272b20 100644
--- a/lib/api/applications.rb
+++ b/lib/api/applications.rb
@@ -41,6 +41,8 @@ module API
desc 'Delete an application'
delete ':id' do
application = ApplicationsFinder.new(params).execute
+ break not_found!('Application') unless application
+
application.destroy
no_content!
diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb
index fa75d012613..339c0e779f9 100644
--- a/lib/api/ci/pipelines.rb
+++ b/lib/api/ci/pipelines.rb
@@ -70,7 +70,7 @@ module API
optional :variables, Array, desc: 'Array of variables available in the pipeline'
end
post ':id/pipeline' do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42124')
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20711')
authorize! :create_pipeline, user_project
diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb
index 80d5e80e21e..c5249f1377b 100644
--- a/lib/api/ci/runner.rb
+++ b/lib/api/ci/runner.rb
@@ -245,7 +245,7 @@ module API
job = authenticate_job!
- result = ::Ci::CreateJobArtifactsService.new(job).authorize(artifact_type: params[:artifact_type], filesize: params[:filesize])
+ result = ::Ci::JobArtifacts::CreateService.new(job).authorize(artifact_type: params[:artifact_type], filesize: params[:filesize])
if result[:status] == :success
content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
@@ -284,7 +284,7 @@ module API
artifacts = params[:file]
metadata = params[:metadata]
- result = ::Ci::CreateJobArtifactsService.new(job).execute(artifacts, params, metadata_file: metadata)
+ result = ::Ci::JobArtifacts::CreateService.new(job).execute(artifacts, params, metadata_file: metadata)
if result[:status] == :success
status :created
diff --git a/lib/api/commits.rb b/lib/api/commits.rb
index a24848082a9..bd9f83ac24c 100644
--- a/lib/api/commits.rb
+++ b/lib/api/commits.rb
@@ -186,16 +186,14 @@ module API
use :pagination
requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag'
end
- # rubocop: disable CodeReuse/ActiveRecord
get ':id/repository/commits/:sha/comments', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do
commit = user_project.commit(params[:sha])
not_found! 'Commit' unless commit
- notes = commit.notes.order(:created_at)
+ notes = commit.notes.with_api_entity_associations.fresh
present paginate(notes), with: Entities::CommitNote
end
- # rubocop: enable CodeReuse/ActiveRecord
desc 'Cherry pick commit into a branch' do
detail 'This feature was introduced in GitLab 8.15'
@@ -372,7 +370,7 @@ module API
current_user,
project_id: user_project.id,
commit_sha: commit.sha
- ).execute
+ ).execute.with_api_entity_associations
present paginate(commit_merge_requests), with: Entities::MergeRequestBasic
end
diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb
index bd8d9b68858..115a6b8ac4f 100644
--- a/lib/api/composer_packages.rb
+++ b/lib/api/composer_packages.rb
@@ -161,6 +161,8 @@ module API
not_found! unless metadata
+ track_package_event('pull_package', :composer)
+
send_git_archive unauthorized_user_project.repository, ref: metadata.target_sha, format: 'zip', append_sha: true
end
end
diff --git a/lib/api/concerns/packages/nuget_endpoints.rb b/lib/api/concerns/packages/nuget_endpoints.rb
index 53b778875fc..5364eeb1880 100644
--- a/lib/api/concerns/packages/nuget_endpoints.rb
+++ b/lib/api/concerns/packages/nuget_endpoints.rb
@@ -95,7 +95,7 @@ module API
# https://docs.microsoft.com/en-us/nuget/api/search-query-service-resource
params do
- requires :q, type: String, desc: 'The search term'
+ optional :q, type: String, desc: 'The search term'
optional :skip, type: Integer, desc: 'The number of results to skip', default: 0, regexp: NON_NEGATIVE_INTEGER_REGEX
optional :take, type: Integer, desc: 'The number of results to return', default: Kaminari.config.default_per_page, regexp: POSITIVE_INTEGER_REGEX
optional :prerelease, type: ::Grape::API::Boolean, desc: 'Include prerelease versions', default: true
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index 0a541620c3a..9f0f569b711 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -44,7 +44,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
get ":id/deploy_keys" do
- keys = user_project.deploy_keys_projects.preload(:deploy_key)
+ keys = user_project.deploy_keys_projects.preload(deploy_key: :user)
present paginate(keys), with: Entities::DeployKeysProject
end
diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb
index d0c842bb19d..0a6ecf2919c 100644
--- a/lib/api/deployments.rb
+++ b/lib/api/deployments.rb
@@ -36,7 +36,9 @@ module API
get ':id/deployments' do
authorize! :read_deployment, user_project
- deployments = DeploymentsFinder.new(params.merge(project: user_project)).execute
+ deployments =
+ DeploymentsFinder.new(params.merge(project: user_project))
+ .execute.with_api_entity_associations
present paginate(deployments), with: Entities::Deployment
end
diff --git a/lib/api/entities/basic_project_details.rb b/lib/api/entities/basic_project_details.rb
index cf0b32bed26..2de49d6ed40 100644
--- a/lib/api/entities/basic_project_details.rb
+++ b/lib/api/entities/basic_project_details.rb
@@ -8,11 +8,10 @@ module API
expose :default_branch, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) }
# Avoids an N+1 query: https://github.com/mbleigh/acts-as-taggable-on/issues/91#issuecomment-168273770
expose :tag_list do |project|
- # project.tags.order(:name).pluck(:name) is the most suitable option
- # to avoid loading all the ActiveRecord objects but, if we use it here
- # it override the preloaded associations and makes a query
- # (fixed in https://github.com/rails/rails/pull/25976).
- project.tags.map(&:name).sort
+ # Tags is a preloaded association. If we perform then sorting
+ # through the database, it will trigger a new query, ending up
+ # in an N+1 if we have several projects
+ project.tags.pluck(:name).sort # rubocop:disable CodeReuse/ActiveRecord
end
expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url
diff --git a/lib/api/entities/clusters/agent.rb b/lib/api/entities/clusters/agent.rb
new file mode 100644
index 00000000000..3b4538b81c2
--- /dev/null
+++ b/lib/api/entities/clusters/agent.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Clusters
+ class Agent < Grape::Entity
+ expose :id
+ expose :project, with: Entities::ProjectIdentity, as: :config_project
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/email.rb b/lib/api/entities/email.rb
index 5ba425def3d..46ebc458bcd 100644
--- a/lib/api/entities/email.rb
+++ b/lib/api/entities/email.rb
@@ -3,7 +3,7 @@
module API
module Entities
class Email < Grape::Entity
- expose :id, :email
+ expose :id, :email, :confirmed_at
end
end
end
diff --git a/lib/api/entities/job_request/job_info.rb b/lib/api/entities/job_request/job_info.rb
index 09c13aa8471..a4bcc9726d0 100644
--- a/lib/api/entities/job_request/job_info.rb
+++ b/lib/api/entities/job_request/job_info.rb
@@ -4,7 +4,7 @@ module API
module Entities
module JobRequest
class JobInfo < Grape::Entity
- expose :name, :stage
+ expose :id, :name, :stage
expose :project_id, :project_name
end
end
diff --git a/lib/api/entities/namespace_existence.rb b/lib/api/entities/namespace_existence.rb
new file mode 100644
index 00000000000..d93078ecdac
--- /dev/null
+++ b/lib/api/entities/namespace_existence.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class NamespaceExistence < Grape::Entity
+ expose :exists, :suggests
+ end
+ end
+end
diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb
index e332e5e40fa..690bc5d419d 100644
--- a/lib/api/entities/project.rb
+++ b/lib/api/entities/project.rb
@@ -127,15 +127,16 @@ module API
# as `:tags` are defined as: `has_many :tags, through: :taggings`
# N+1 is solved then by using `subject.tags.map(&:name)`
# MR describing the solution: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20555
- super(projects_relation).preload(:group)
+ super(projects_relation).preload(group: :namespace_settings)
.preload(:ci_cd_settings)
.preload(:project_setting)
.preload(:container_expiration_policy)
.preload(:auto_devops)
+ .preload(:service_desk_setting)
.preload(project_group_links: { group: :route },
fork_network: :root_project,
fork_network_member: :forked_from_project,
- forked_from_project: [:route, :forks, :tags, namespace: :route])
+ forked_from_project: [:route, :forks, :tags, :group, :project_feature, namespace: [:route, :owner]])
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/api/entities/project_import_failed_relation.rb b/lib/api/entities/project_import_failed_relation.rb
index 16b26ad0efa..b8f842c1646 100644
--- a/lib/api/entities/project_import_failed_relation.rb
+++ b/lib/api/entities/project_import_failed_relation.rb
@@ -3,7 +3,11 @@
module API
module Entities
class ProjectImportFailedRelation < Grape::Entity
- expose :id, :created_at, :exception_class, :exception_message, :source
+ expose :id, :created_at, :exception_class, :source
+
+ expose :exception_message do |_|
+ nil
+ end
expose :relation_key, as: :relation_name
end
diff --git a/lib/api/entities/user.rb b/lib/api/entities/user.rb
index 248a86751d2..3ce6d03e236 100644
--- a/lib/api/entities/user.rb
+++ b/lib/api/entities/user.rb
@@ -11,10 +11,10 @@ module API
work_information(user)
end
expose :followers, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user|
- user.followers.count
+ user.followers.size
end
expose :following, if: ->(user, opts) { Ability.allowed?(opts[:current_user], :read_user_profile, user) } do |user|
- user.followees.count
+ user.followees.size
end
end
end
diff --git a/lib/api/entities/user_preferences.rb b/lib/api/entities/user_preferences.rb
new file mode 100644
index 00000000000..7a6df9b6c59
--- /dev/null
+++ b/lib/api/entities/user_preferences.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class UserPreferences < Grape::Entity
+ expose :id, :user_id, :view_diffs_file_by_file
+ end
+ end
+end
diff --git a/lib/api/entities/user_public.rb b/lib/api/entities/user_public.rb
index 15e9b905bef..685adb1dd10 100644
--- a/lib/api/entities/user_public.rb
+++ b/lib/api/entities/user_public.rb
@@ -14,6 +14,7 @@ module API
expose :two_factor_enabled?, as: :two_factor_enabled
expose :external
expose :private_profile
+ expose :commit_email
end
end
end
diff --git a/lib/api/environments.rb b/lib/api/environments.rb
index 3e1e430c2f9..b606b2e814d 100644
--- a/lib/api/environments.rb
+++ b/lib/api/environments.rb
@@ -26,7 +26,7 @@ module API
get ':id/environments' do
authorize! :read_environment, user_project
- environments = ::EnvironmentsFinder.new(user_project, current_user, params).find
+ environments = ::EnvironmentsFinder.new(user_project, current_user, params).execute
present paginate(environments), with: Entities::Environment, current_user: current_user
end
diff --git a/lib/api/files.rb b/lib/api/files.rb
index cb73bde73f5..f3de7fbe96b 100644
--- a/lib/api/files.rb
+++ b/lib/api/files.rb
@@ -113,7 +113,7 @@ module API
desc 'Get raw file metadata from repository'
params do
requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
- requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false
+ optional :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false
end
head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do
assign_file_vars!
@@ -124,7 +124,7 @@ module API
desc 'Get raw file contents from the repository'
params do
requires :file_path, type: String, file_path: true, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb'
- requires :ref, type: String, desc: 'The name of branch, tag commit', allow_blank: false
+ optional :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false
end
get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do
assign_file_vars!
diff --git a/lib/api/generic_packages.rb b/lib/api/generic_packages.rb
index 3d0ba97b51a..cce55fa92d9 100644
--- a/lib/api/generic_packages.rb
+++ b/lib/api/generic_packages.rb
@@ -62,7 +62,7 @@ module API
authorize_upload!(project)
bad_request!('File is too large') if max_file_size_exceeded?
- track_event('push_package')
+ ::Gitlab::Tracking.event(self.options[:for].name, 'push_package')
create_package_file_params = declared_params.merge(build: current_authenticated_job)
::Packages::Generic::CreatePackageFileService
@@ -94,7 +94,7 @@ module API
package = ::Packages::Generic::PackageFinder.new(project).execute!(params[:package_name], params[:package_version])
package_file = ::Packages::PackageFileFinder.new(package, params[:file_name]).execute!
- track_event('pull_package')
+ ::Gitlab::Tracking.event(self.options[:for].name, 'pull_package')
present_carrierwave_file!(package_file.file)
end
diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb
index 09744fbeda2..8d52a0a5b4e 100644
--- a/lib/api/group_variables.rb
+++ b/lib/api/group_variables.rb
@@ -8,6 +8,8 @@ module API
before { authorize! :admin_group, user_group }
feature_category :continuous_integration
+ helpers Helpers::VariablesHelpers
+
params do
requires :id, type: String, desc: 'The ID of a group'
end
@@ -30,16 +32,13 @@ module API
params do
requires :key, type: String, desc: 'The key of the variable'
end
- # rubocop: disable CodeReuse/ActiveRecord
get ':id/variables/:key' do
- key = params[:key]
- variable = user_group.variables.find_by(key: key)
+ variable = find_variable(user_group, params)
break not_found!('GroupVariable') unless variable
present variable, with: Entities::Ci::Variable
end
- # rubocop: enable CodeReuse/ActiveRecord
desc 'Create a new variable in a group' do
success Entities::Ci::Variable
@@ -50,12 +49,19 @@ module API
optional :protected, type: String, desc: 'Whether the variable is protected'
optional :masked, type: String, desc: 'Whether the variable is masked'
optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var'
+
+ use :optional_group_variable_params_ee
end
post ':id/variables' do
+ filtered_params = filter_variable_parameters(
+ user_group,
+ declared_params(include_missing: false)
+ )
+
variable = ::Ci::ChangeVariableService.new(
container: user_group,
current_user: current_user,
- params: { action: :create, variable_params: declared_params(include_missing: false) }
+ params: { action: :create, variable_params: filtered_params }
).execute
if variable.valid?
@@ -74,13 +80,19 @@ module API
optional :protected, type: String, desc: 'Whether the variable is protected'
optional :masked, type: String, desc: 'Whether the variable is masked'
optional :variable_type, type: String, values: ::Ci::GroupVariable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file'
+
+ use :optional_group_variable_params_ee
end
- # rubocop: disable CodeReuse/ActiveRecord
put ':id/variables/:key' do
+ filtered_params = filter_variable_parameters(
+ user_group,
+ declared_params(include_missing: false)
+ )
+
variable = ::Ci::ChangeVariableService.new(
container: user_group,
current_user: current_user,
- params: { action: :update, variable_params: declared_params(include_missing: false) }
+ params: { action: :update, variable_params: filtered_params }
).execute
if variable.valid?
@@ -91,7 +103,6 @@ module API
rescue ::ActiveRecord::RecordNotFound
not_found!('GroupVariable')
end
- # rubocop: enable CodeReuse/ActiveRecord
desc 'Delete an existing variable from a group' do
success Entities::Ci::Variable
@@ -99,21 +110,18 @@ module API
params do
requires :key, type: String, desc: 'The key of the variable'
end
- # rubocop: disable CodeReuse/ActiveRecord
delete ':id/variables/:key' do
- variable = user_group.variables.find_by!(key: params[:key])
+ variable = find_variable(user_group, params)
+ break not_found!('GroupVariable') unless variable
destroy_conditionally!(variable) do |target_variable|
::Ci::ChangeVariableService.new(
container: user_group,
current_user: current_user,
- params: { action: :destroy, variable_params: declared_params(include_missing: false) }
+ params: { action: :destroy, variable: variable }
).execute
end
- rescue ::ActiveRecord::RecordNotFound
- not_found!('GroupVariable')
end
- # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 26fa00d6186..912813d5bb7 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -52,9 +52,7 @@ module API
groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
order_options = { params[:order_by] => params[:sort] }
order_options["id"] ||= "asc"
- groups = groups.reorder(order_options)
-
- groups
+ groups.reorder(order_options)
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -112,7 +110,6 @@ module API
end
def delete_group(group)
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/46285')
destroy_conditionally!(group) do |group|
::Groups::DestroyService.new(group, current_user).async_execute
end
@@ -141,6 +138,10 @@ module API
def authorize_group_creation!
authorize! :create_group
end
+
+ def check_subscription!(group)
+ render_api_error!("This group can't be removed because it is linked to a subscription.", :bad_request) if group.paid?
+ end
end
resource :groups do
@@ -239,6 +240,7 @@ module API
delete ":id" do
group = find_group!(params[:id])
authorize! :admin_group, group
+ check_subscription! group
delete_group(group)
end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 9db4a03c5b9..2d8a4f60e2a 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -3,6 +3,7 @@
module API
module Helpers
include Gitlab::Utils
+ include Helpers::Caching
include Helpers::Pagination
include Helpers::PaginationStrategies
@@ -48,7 +49,11 @@ module API
# Returns the job associated with the token provided for
# authentication, if any
def current_authenticated_job
- @current_authenticated_job
+ if try(:namespace_inheritable, :authentication)
+ ci_build_from_namespace_inheritable
+ else
+ @current_authenticated_job # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ end
end
# rubocop:disable Gitlab/ModuleWithInstanceVariables
@@ -539,17 +544,6 @@ module API
end
end
- def track_event(action = action_name, **args)
- category = args.delete(:category) || self.options[:for].name
- raise "invalid category" unless category
-
- ::Gitlab::Tracking.event(category, action.to_s, **args)
- rescue => error
- Gitlab::AppLogger.warn(
- "Tracking event failed for action: #{action}, category: #{category}, message: #{error.message}"
- )
- end
-
def increment_counter(event_name)
feature_name = "usage_data_#{event_name}"
return unless Feature.enabled?(feature_name)
@@ -564,10 +558,6 @@ module API
def increment_unique_values(event_name, values)
return unless values.present?
- feature_flag = "usage_data_#{event_name}"
-
- return unless Feature.enabled?(feature_flag, default_enabled: true)
-
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: values)
rescue => error
Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}")
diff --git a/lib/api/helpers/authentication.rb b/lib/api/helpers/authentication.rb
index a6cfe930190..da11f07485b 100644
--- a/lib/api/helpers/authentication.rb
+++ b/lib/api/helpers/authentication.rb
@@ -52,6 +52,11 @@ module API
token&.user
end
+ def ci_build_from_namespace_inheritable
+ token = token_from_namespace_inheritable
+ token if token.is_a?(::Ci::Build)
+ end
+
private
def find_token_from_raw_credentials(token_types, raw)
diff --git a/lib/api/helpers/caching.rb b/lib/api/helpers/caching.rb
new file mode 100644
index 00000000000..d0f22109879
--- /dev/null
+++ b/lib/api/helpers/caching.rb
@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+# Grape helpers for caching.
+#
+# This module helps introduce standardised caching into the Grape API
+# in a similar manner to the standard Grape DSL.
+
+module API
+ module Helpers
+ module Caching
+ # @return [ActiveSupport::Duration]
+ DEFAULT_EXPIRY = 1.day
+
+ # @return [ActiveSupport::Cache::Store]
+ def cache
+ Rails.cache
+ end
+
+ # This is functionally equivalent to the standard `#present` used in
+ # Grape endpoints, but the JSON for the object, or for each object of
+ # a collection, will be cached.
+ #
+ # With a collection all the keys will be fetched in a single call and the
+ # Entity rendered for those missing from the cache, which are then written
+ # back into it.
+ #
+ # Both the single object, and all objects inside a collection, must respond
+ # to `#cache_key`.
+ #
+ # To override the Grape formatter we return a custom wrapper in
+ # `Gitlab::Json::PrecompiledJson` which tells the `Gitlab::Json::GrapeFormatter`
+ # to export the string without conversion.
+ #
+ # A cache context can be supplied to add more context to the cache key. This
+ # defaults to including the `current_user` in every key for safety, unless overridden.
+ #
+ # @param obj_or_collection [Object, Enumerable<Object>] the object or objects to render
+ # @param with [Grape::Entity] the entity to use for rendering
+ # @param cache_context [Proc] a proc to call for each object to provide more context to the cache key
+ # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry
+ # @param presenter_args [Hash] keyword arguments to be passed to the entity
+ # @return [Gitlab::Json::PrecompiledJson]
+ def present_cached(obj_or_collection, with:, cache_context: -> (_) { current_user.cache_key }, expires_in: DEFAULT_EXPIRY, **presenter_args)
+ json =
+ if obj_or_collection.is_a?(Enumerable)
+ cached_collection(
+ obj_or_collection,
+ presenter: with,
+ presenter_args: presenter_args,
+ context: cache_context,
+ expires_in: expires_in
+ )
+ else
+ cached_object(
+ obj_or_collection,
+ presenter: with,
+ presenter_args: presenter_args,
+ context: cache_context,
+ expires_in: expires_in
+ )
+ end
+
+ body Gitlab::Json::PrecompiledJson.new(json)
+ end
+
+ private
+
+ # Optionally uses a `Proc` to add context to a cache key
+ #
+ # @param object [Object] must respond to #cache_key
+ # @param context [Proc] a proc that will be called with the object as an argument, and which should return a
+ # string or array of strings to be combined into the cache key
+ # @return [String]
+ def contextual_cache_key(object, context)
+ return object.cache_key if context.nil?
+
+ [object.cache_key, context.call(object)].flatten.join(":")
+ end
+
+ # Used for fetching or rendering a single object
+ #
+ # @param object [Object] the object to render
+ # @param presenter [Grape::Entity]
+ # @param presenter_args [Hash] keyword arguments to be passed to the entity
+ # @param context [Proc]
+ # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry
+ # @return [String]
+ def cached_object(object, presenter:, presenter_args:, context:, expires_in:)
+ cache.fetch(contextual_cache_key(object, context), expires_in: expires_in) do
+ Gitlab::Json.dump(presenter.represent(object, **presenter_args).as_json)
+ end
+ end
+
+ # Used for fetching or rendering multiple objects
+ #
+ # @param objects [Enumerable<Object>] the objects to render
+ # @param presenter [Grape::Entity]
+ # @param presenter_args [Hash] keyword arguments to be passed to the entity
+ # @param context [Proc]
+ # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry
+ # @return [Array<String>]
+ def cached_collection(collection, presenter:, presenter_args:, context:, expires_in:)
+ json = fetch_multi(collection, context: context, expires_in: expires_in) do |obj|
+ Gitlab::Json.dump(presenter.represent(obj, **presenter_args).as_json)
+ end
+
+ json.values
+ end
+
+ # An adapted version of ActiveSupport::Cache::Store#fetch_multi.
+ #
+ # The original method only provides the missing key to the block,
+ # not the missing object, so we have to create a map of cache keys
+ # to the objects to allow us to pass the object to the missing value
+ # block.
+ #
+ # The result is that this is functionally identical to `#fetch`.
+ def fetch_multi(*objs, context:, **kwargs)
+ objs.flatten!
+ map = multi_key_map(objs, context: context)
+
+ cache.fetch_multi(*map.keys, **kwargs) do |key|
+ yield map[key]
+ end
+ end
+
+ # @param objects [Enumerable<Object>] objects which _must_ respond to `#cache_key`
+ # @param context [Proc] a proc that can be called to help generate each cache key
+ # @return [Hash]
+ def multi_key_map(objects, context:)
+ objects.index_by do |object|
+ contextual_cache_key(object, context)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb
index a44fd4b0a5b..8940cf87f82 100644
--- a/lib/api/helpers/common_helpers.rb
+++ b/lib/api/helpers/common_helpers.rb
@@ -32,6 +32,10 @@ module API
end
end.compact.to_set
end
+
+ def endpoint_id
+ "#{request.request_method} #{route.origin}"
+ end
end
end
end
diff --git a/lib/api/helpers/graphql_helpers.rb b/lib/api/helpers/graphql_helpers.rb
index 3ddef0c16b3..4f7f85bd69d 100644
--- a/lib/api/helpers/graphql_helpers.rb
+++ b/lib/api/helpers/graphql_helpers.rb
@@ -6,8 +6,8 @@ module API
# against the graphql API. Helper code for the graphql server implementation
# should be in app/graphql/ or lib/gitlab/graphql/
module GraphqlHelpers
- def run_graphql!(query:, context: {}, transform: nil)
- result = GitlabSchema.execute(query, context: context)
+ def run_graphql!(query:, context: {}, variables: nil, transform: nil)
+ result = GitlabSchema.execute(query, variables: variables, context: context)
if transform
transform.call(result)
diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb
index 71a18524104..cb938bc8a14 100644
--- a/lib/api/helpers/notes_helpers.rb
+++ b/lib/api/helpers/notes_helpers.rb
@@ -116,7 +116,7 @@ module API
end
def create_note(noteable, opts)
- whitelist_query_limiting
+ disable_query_limiting
authorize!(:create_note, noteable)
parent = noteable_parent(noteable)
@@ -144,8 +144,8 @@ module API
present discussion, with: Entities::Discussion
end
- def whitelist_query_limiting
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/-/issues/211538')
+ def disable_query_limiting
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/211538')
end
end
end
diff --git a/lib/api/helpers/packages/conan/api_helpers.rb b/lib/api/helpers/packages/conan/api_helpers.rb
index d5f5448fd42..b18f52b5be6 100644
--- a/lib/api/helpers/packages/conan/api_helpers.rb
+++ b/lib/api/helpers/packages/conan/api_helpers.rb
@@ -14,7 +14,8 @@ module API
package,
current_user,
project,
- conan_package_reference: params[:conan_package_reference]
+ conan_package_reference: params[:conan_package_reference],
+ id: params[:id]
)
render_api_error!("No recipe manifest found", 404) if yield(presenter).empty?
@@ -31,19 +32,15 @@ module API
end
def recipe_upload_urls
- { upload_urls: Hash[
- file_names.select(&method(:recipe_file?)).map do |file_name|
- [file_name, build_recipe_file_upload_url(file_name)]
- end
- ] }
+ { upload_urls: file_names.select(&method(:recipe_file?)).to_h do |file_name|
+ [file_name, build_recipe_file_upload_url(file_name)]
+ end }
end
def package_upload_urls
- { upload_urls: Hash[
- file_names.select(&method(:package_file?)).map do |file_name|
- [file_name, build_package_file_upload_url(file_name)]
- end
- ] }
+ { upload_urls: file_names.select(&method(:package_file?)).to_h do |file_name|
+ [file_name, build_package_file_upload_url(file_name)]
+ end }
end
def recipe_file?(file_name)
@@ -212,10 +209,8 @@ module API
end
def find_personal_access_token
- personal_access_token = find_personal_access_token_from_conan_jwt ||
+ find_personal_access_token_from_conan_jwt ||
find_personal_access_token_from_http_basic_auth
-
- personal_access_token
end
def find_user_from_job_token
diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb
index 577ba97d68a..989c4e1761b 100644
--- a/lib/api/helpers/packages/dependency_proxy_helpers.rb
+++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb
@@ -10,7 +10,7 @@ module API
def redirect_registry_request(forward_to_registry, package_type, options)
if forward_to_registry && redirect_registry_request_available?
- track_event("#{package_type}_request_forward")
+ ::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward")
redirect(registry_url(package_type, options))
else
yield
diff --git a/lib/api/helpers/packages_helpers.rb b/lib/api/helpers/packages_helpers.rb
index e1898d28ef7..2221eec0f82 100644
--- a/lib/api/helpers/packages_helpers.rb
+++ b/lib/api/helpers/packages_helpers.rb
@@ -50,7 +50,8 @@ module API
def track_package_event(event_name, scope, **args)
::Packages::CreateEventService.new(nil, current_user, event_name: event_name, scope: scope).execute
- track_event(event_name, **args)
+ category = args.delete(:category) || self.options[:for].name
+ ::Gitlab::Tracking.event(category, event_name.to_s, **args)
end
end
end
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 39586483990..688cd2da994 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -38,10 +38,17 @@ module API
end
end
+ # HTTP status codes to terminate the job on GitLab Runner:
+ # - 403
def authenticate_job!(require_running: true)
job = current_job
- not_found! unless job
+ # 404 is not returned here because we want to terminate the job if it's
+ # running. A 404 can be returned from anywhere in the networking stack which is why
+ # we are explicit about a 403, we should improve this in
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/327703
+ forbidden! unless job
+
forbidden! unless job_token_valid?(job)
forbidden!('Project has been deleted!') if job.project.nil? || job.project.pending_delete?
diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb
index ed3d694f006..2f2ad88c942 100644
--- a/lib/api/helpers/services_helpers.rb
+++ b/lib/api/helpers/services_helpers.rb
@@ -394,7 +394,7 @@ module API
required: true,
name: :external_wiki_url,
type: String,
- desc: 'The URL of the external Wiki'
+ desc: 'The URL of the external wiki'
}
],
'flowdock' => [
@@ -543,9 +543,15 @@ module API
},
{
required: false,
+ name: :jira_issue_transition_automatic,
+ type: Boolean,
+ desc: 'Enable automatic issue transitions'
+ },
+ {
+ required: false,
name: :jira_issue_transition_id,
type: String,
- desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the Jira workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`'
+ desc: 'The ID of one or more transitions for custom issue transitions'
},
{
required: false,
diff --git a/lib/api/helpers/variables_helpers.rb b/lib/api/helpers/variables_helpers.rb
new file mode 100644
index 00000000000..e2b3372fc33
--- /dev/null
+++ b/lib/api/helpers/variables_helpers.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module API
+ module Helpers
+ module VariablesHelpers
+ extend ActiveSupport::Concern
+ extend Grape::API::Helpers
+
+ params :optional_group_variable_params_ee do
+ end
+
+ def filter_variable_parameters(_, params)
+ params # Overridden in EE
+ end
+
+ def find_variable(owner, params)
+ variables = ::Ci::VariablesFinder.new(owner, params).execute.to_a
+
+ return variables.first unless variables.many? # rubocop: disable CodeReuse/ActiveRecord
+
+ conflict!("There are multiple variables with provided parameters. Please use 'filter[environment_scope]'")
+ end
+ end
+ end
+end
+
+API::Helpers::VariablesHelpers.prepend_if_ee('EE::API::Helpers::VariablesHelpers')
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index a3fee49cd8f..4dcfc0cf7eb 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -15,7 +15,7 @@ module API
Gitlab::ApplicationContext.push(
user: -> { actor&.user },
project: -> { project },
- caller_id: route.origin,
+ caller_id: api_endpoint.endpoint_id,
remote_ip: request.ip,
feature_category: feature_category
)
@@ -23,7 +23,7 @@ module API
helpers ::API::Helpers::InternalHelpers
- UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result'.freeze
+ UNKNOWN_CHECK_RESULT_ERROR = 'Unknown check result'
VALID_PAT_SCOPES = Set.new(
Gitlab::Auth::API_SCOPES + Gitlab::Auth::REPOSITORY_SCOPES + Gitlab::Auth::REGISTRY_SCOPES
@@ -52,20 +52,20 @@ module API
actor.update_last_used_at!
check_result = begin
- Gitlab::Auth::CurrentUserMode.bypass_session!(actor.user&.id) do
- access_check!(actor, params)
- end
- rescue Gitlab::GitAccess::ForbiddenError => e
- # The return code needs to be 401. If we return 403
- # the custom message we return won't be shown to the user
- # and, instead, the default message 'GitLab: API is not accessible'
- # will be displayed
- return response_with_status(code: 401, success: false, message: e.message)
- rescue Gitlab::GitAccess::TimeoutError => e
- return response_with_status(code: 503, success: false, message: e.message)
- rescue Gitlab::GitAccess::NotFoundError => e
- return response_with_status(code: 404, success: false, message: e.message)
- end
+ with_admin_mode_bypass!(actor.user&.id) do
+ access_check!(actor, params)
+ end
+ rescue Gitlab::GitAccess::ForbiddenError => e
+ # The return code needs to be 401. If we return 403
+ # the custom message we return won't be shown to the user
+ # and, instead, the default message 'GitLab: API is not accessible'
+ # will be displayed
+ return response_with_status(code: 401, success: false, message: e.message)
+ rescue Gitlab::GitAccess::TimeoutError => e
+ return response_with_status(code: 503, success: false, message: e.message)
+ rescue Gitlab::GitAccess::NotFoundError => e
+ return response_with_status(code: 404, success: false, message: e.message)
+ end
log_user_activity(actor.user)
@@ -109,9 +109,7 @@ module API
end
end
- def validate_actor_key(actor, key_id)
- return 'Could not find a user without a key' unless key_id
-
+ def validate_actor(actor)
return 'Could not find the given key' unless actor.key
'Could not find a user for the given key' unless actor.user
@@ -120,6 +118,19 @@ module API
def two_factor_otp_check
{ success: false, message: 'Feature is not available' }
end
+
+ def with_admin_mode_bypass!(actor_id)
+ return yield unless Gitlab::CurrentSettings.admin_mode
+
+ Gitlab::Auth::CurrentUserMode.bypass_session!(actor_id) do
+ yield
+ end
+ end
+
+ # Overridden in EE
+ def geo_proxy
+ {}
+ end
end
namespace 'internal' do
@@ -193,7 +204,7 @@ module API
actor.update_last_used_at!
user = actor.user
- error_message = validate_actor_key(actor, params[:key_id])
+ error_message = validate_actor(actor)
if params[:user_id] && user.nil?
break { success: false, message: 'Could not find the given user' }
@@ -222,7 +233,7 @@ module API
actor.update_last_used_at!
user = actor.user
- error_message = validate_actor_key(actor, params[:key_id])
+ error_message = validate_actor(actor)
break { success: false, message: 'Deploy keys cannot be used to create personal access tokens' } if actor.key.is_a?(DeployKey)
@@ -295,7 +306,7 @@ module API
actor.update_last_used_at!
user = actor.user
- error_message = validate_actor_key(actor, params[:key_id])
+ error_message = validate_actor(actor)
if error_message
{ success: false, message: error_message }
@@ -314,6 +325,12 @@ module API
two_factor_otp_check
end
+
+ # Workhorse calls this to determine if it is a Geo secondary site
+ # that should proxy requests. FOSS can quickly return empty data.
+ get '/geo_proxy', feature_category: :geo_replication do
+ geo_proxy
+ end
end
end
end
diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb
index 87ad79d601f..af2c53dd778 100644
--- a/lib/api/internal/kubernetes.rb
+++ b/lib/api/internal/kubernetes.rb
@@ -13,7 +13,7 @@ module API
helpers do
def authenticate_gitlab_kas_request!
- unauthorized! unless Gitlab::Kas.verify_api_request(headers)
+ render_api_error!('KAS JWT authentication invalid', 401) unless Gitlab::Kas.verify_api_request(headers)
end
def agent_token
@@ -51,9 +51,11 @@ module API
end
def check_agent_token
- forbidden! unless agent_token
+ unauthorized! unless agent_token
forbidden! unless Gitlab::Kas.included_in_gitlab_com_rollout?(agent.project)
+
+ agent_token.track_usage
end
end
diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb
index 52c32b4d1cf..0d562cc18f8 100644
--- a/lib/api/invitations.rb
+++ b/lib/api/invitations.rb
@@ -25,11 +25,11 @@ module API
optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY'
end
post ":id/invitations" do
- source = find_source(source_type, params[:id])
+ params[:source] = find_source(source_type, params[:id])
- authorize_admin_source!(source_type, source)
+ authorize_admin_source!(source_type, params[:source])
- ::Members::InviteService.new(current_user, params).execute(source)
+ ::Members::InviteService.new(current_user, params).execute
end
desc 'Get a list of group or project invitations viewable by the authenticated user' do
diff --git a/lib/api/issue_links.rb b/lib/api/issue_links.rb
index e938dbbae87..1cd5bde224b 100644
--- a/lib/api/issue_links.rb
+++ b/lib/api/issue_links.rb
@@ -18,7 +18,10 @@ module API
end
get ':id/issues/:issue_iid/links' do
source_issue = find_project_issue(params[:issue_iid])
- related_issues = source_issue.related_issues(current_user)
+ related_issues = source_issue.related_issues(current_user) do |issues|
+ issues.with_api_entity_associations.preload_awardable
+ end
+ related_issues.each { |issue| issue.lazy_subscription(current_user, user_project) } # preload subscriptions
present related_issues,
with: Entities::RelatedIssue,
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index 13dac1c174c..4f2ac73c0d3 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -242,7 +242,7 @@ module API
use :issue_params
end
post ':id/issues' do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42320')
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/21140')
check_rate_limit! :issues_create, [current_user]
@@ -288,7 +288,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
put ':id/issues/:issue_iid' do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42322')
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20775')
issue = user_project.issues.find_by!(iid: params.delete(:issue_iid))
authorize! :update_issue, issue
@@ -346,7 +346,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
post ':id/issues/:issue_iid/move' do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42323')
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20776')
issue = user_project.issues.find_by(iid: params[:issue_iid])
not_found!('Issue') unless issue
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index 7390219b60e..54951f9bd01 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -6,8 +6,6 @@ module API
before { authenticate! }
- feature_category :continuous_integration
-
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
params do
requires :id, type: String, desc: 'The ID of a project'
@@ -40,7 +38,7 @@ module API
use :pagination
end
# rubocop: disable CodeReuse/ActiveRecord
- get ':id/jobs' do
+ get ':id/jobs', feature_category: :continuous_integration do
authorize_read_builds!
builds = user_project.builds.order('id DESC')
@@ -57,7 +55,7 @@ module API
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
end
- get ':id/jobs/:job_id' do
+ get ':id/jobs/:job_id', feature_category: :continuous_integration do
authorize_read_builds!
build = find_build!(params[:job_id])
@@ -72,7 +70,7 @@ module API
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
end
- get ':id/jobs/:job_id/trace' do
+ get ':id/jobs/:job_id/trace', feature_category: :continuous_integration do
authorize_read_builds!
build = find_build!(params[:job_id])
@@ -94,7 +92,7 @@ module API
params do
requires :job_id, type: Integer, desc: 'The ID of a job'
end
- post ':id/jobs/:job_id/cancel' do
+ post ':id/jobs/:job_id/cancel', feature_category: :continuous_integration do
authorize_update_builds!
build = find_build!(params[:job_id])
@@ -111,7 +109,7 @@ module API
params do
requires :job_id, type: Integer, desc: 'The ID of a build'
end
- post ':id/jobs/:job_id/retry' do
+ post ':id/jobs/:job_id/retry', feature_category: :continuous_integration do
authorize_update_builds!
build = find_build!(params[:job_id])
@@ -129,7 +127,7 @@ module API
params do
requires :job_id, type: Integer, desc: 'The ID of a build'
end
- post ':id/jobs/:job_id/erase' do
+ post ':id/jobs/:job_id/erase', feature_category: :continuous_integration do
authorize_update_builds!
build = find_build!(params[:job_id])
@@ -148,7 +146,7 @@ module API
requires :job_id, type: Integer, desc: 'The ID of a Job'
end
- post ":id/jobs/:job_id/play" do
+ post ":id/jobs/:job_id/play", feature_category: :continuous_integration do
authorize_read_builds!
job = find_job!(params[:job_id])
@@ -174,10 +172,8 @@ module API
success Entities::Ci::Job
end
route_setting :authentication, job_token_allowed: true
- get do
- # current_authenticated_job will be nil if user is using
- # a valid authentication that is not CI_JOB_TOKEN
- not_found!('Job') unless current_authenticated_job
+ get '', feature_category: :continuous_integration do
+ validate_current_authenticated_job
present current_authenticated_job, with: Entities::Ci::Job
end
@@ -196,6 +192,14 @@ module API
builds.where(status: available_statuses && scope)
end
# rubocop: enable CodeReuse/ActiveRecord
+
+ def validate_current_authenticated_job
+ # current_authenticated_job will be nil if user is using
+ # a valid authentication (like PRIVATE-TOKEN) that is not CI_JOB_TOKEN
+ not_found!('Job') unless current_authenticated_job
+ end
end
end
end
+
+API::Jobs.prepend_if_ee('EE::API::Jobs')
diff --git a/lib/api/maven_packages.rb b/lib/api/maven_packages.rb
index 4a5b2ead163..bd1d984719e 100644
--- a/lib/api/maven_packages.rb
+++ b/lib/api/maven_packages.rb
@@ -23,6 +23,15 @@ module API
helpers ::API::Helpers::PackagesHelpers
helpers do
+ def path_exists?(path)
+ # return true when FF disabled so that processing the request is not stopped
+ return true unless Feature.enabled?(:check_maven_path_first)
+ return false if path.blank?
+
+ Packages::Maven::Metadatum.with_path(path)
+ .exists?
+ end
+
def extract_format(file_name)
name, _, format = file_name.rpartition('.')
@@ -77,6 +86,22 @@ module API
request.head? &&
file.fog_credentials[:provider] == 'AWS'
end
+
+ def fetch_package(file_name:, project: nil, group: nil)
+ order_by_package_file = false
+ if Feature.enabled?(:maven_packages_group_level_improvements, default_enabled: :yaml)
+ order_by_package_file = file_name.include?(::Packages::Maven::Metadata.filename) &&
+ !params[:path].include?(::Packages::Maven::FindOrCreatePackageService::SNAPSHOT_TERM)
+ end
+
+ ::Packages::Maven::PackageFinder.new(
+ params[:path],
+ current_user,
+ project: project,
+ group: group,
+ order_by_package_file: order_by_package_file
+ ).execute!
+ end
end
desc 'Download the maven package file at instance level' do
@@ -88,6 +113,9 @@ module API
end
route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
get 'packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
+ # return a similar failure to authorize_read_package!(project)
+ forbidden! unless path_exists?(params[:path])
+
file_name, format = extract_format(params[:file_name])
# To avoid name collision we require project path and project package be the same.
@@ -97,8 +125,7 @@ module API
authorize_read_package!(project)
- package = ::Packages::Maven::PackageFinder
- .new(params[:path], current_user, project: project).execute!
+ package = fetch_package(file_name: file_name, project: project)
package_file = ::Packages::PackageFileFinder
.new(package, file_name).execute!
@@ -127,14 +154,16 @@ module API
end
route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
get ':id/-/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
+ # return a similar failure to group = find_group(params[:id])
+ not_found!('Group') unless path_exists?(params[:path])
+
file_name, format = extract_format(params[:file_name])
group = find_group(params[:id])
not_found!('Group') unless can?(current_user, :read_group, group)
- package = ::Packages::Maven::PackageFinder
- .new(params[:path], current_user, group: group).execute!
+ package = fetch_package(file_name: file_name, group: group)
authorize_read_package!(package.project)
@@ -167,12 +196,14 @@ module API
end
route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true
get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do
+ # return a similar failure to user_project
+ not_found!('Project') unless path_exists?(params[:path])
+
authorize_read_package!(user_project)
file_name, format = extract_format(params[:file_name])
- package = ::Packages::Maven::PackageFinder
- .new(params[:path], current_user, project: user_project).execute!
+ package = fetch_package(file_name: file_name, project: user_project)
package_file = ::Packages::PackageFileFinder
.new(package, file_name).execute!
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 42f608102b3..aaf0e3e1927 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -100,9 +100,9 @@ module API
authorize_admin_source!(source_type, source)
if params[:user_id].to_s.include?(',')
- create_service_params = params.except(:user_id).merge({ user_ids: params[:user_id] })
+ create_service_params = params.except(:user_id).merge({ user_ids: params[:user_id], source: source })
- ::Members::CreateService.new(current_user, create_service_params).execute(source)
+ ::Members::CreateService.new(current_user, create_service_params).execute
elsif params[:user_id].present?
member = source.members.find_by(user_id: params[:user_id])
conflict!('Member already exists') if member
@@ -155,6 +155,8 @@ module API
desc 'Removes a user from a group or project.'
params do
requires :user_id, type: Integer, desc: 'The user ID of the member'
+ optional :skip_subresources, type: Boolean, default: false,
+ desc: 'Flag indicating if the deletion of direct memberships of the removed member in subgroups and projects should be skipped'
optional :unassign_issuables, type: Boolean, default: false,
desc: 'Flag indicating if the removed member should be unassigned from any issues or merge requests within given group or project'
end
@@ -164,7 +166,7 @@ module API
member = source_members(source).find_by!(user_id: params[:user_id])
destroy_conditionally!(member) do
- ::Members::DestroyService.new(current_user).execute(member, unassign_issuables: params[:unassign_issuables])
+ ::Members::DestroyService.new(current_user).execute(member, skip_subresources: params[:skip_subresources], unassign_issuables: params[:unassign_issuables])
end
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb
index 97a6c7075b3..470f78a7dc2 100644
--- a/lib/api/merge_request_diffs.rb
+++ b/lib/api/merge_request_diffs.rb
@@ -45,7 +45,7 @@ module API
merge_request = find_merge_request_with_access(params[:merge_request_iid])
- present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull
+ present_cached merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull, cache_context: nil
end
end
end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index 5051c1a5529..613de514ffa 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -8,11 +8,20 @@ module API
before { authenticate_non_get! }
- feature_category :code_review
-
helpers Helpers::MergeRequestsHelpers
helpers Helpers::SSEHelpers
+ # These endpoints are defined in `TimeTrackingEndpoints` and is shared by
+ # API::Issues. In order to be able to define the feature category of these
+ # endpoints, we need to define them at the top-level by route.
+ feature_category :code_review, [
+ '/projects/:id/merge_requests/:merge_request_iid/time_estimate',
+ '/projects/:id/merge_requests/:merge_request_iid/reset_time_estimate',
+ '/projects/:id/merge_requests/:merge_request_iid/add_spent_time',
+ '/projects/:id/merge_requests/:merge_request_iid/reset_spent_time',
+ '/projects/:id/merge_requests/:merge_request_iid/time_stats'
+ ]
+
# EE::API::MergeRequests would override the following helpers
helpers do
params :optional_params_ee do
@@ -125,7 +134,7 @@ module API
use :merge_requests_params
use :optional_scope_param
end
- get do
+ get feature_category: :code_review do
authenticate! unless params[:scope] == 'all'
merge_requests = find_merge_requests
@@ -145,7 +154,7 @@ module API
optional :non_archived, type: Boolean, desc: 'Return merge requests from non archived projects',
default: true
end
- get ":id/merge_requests" do
+ get ":id/merge_requests", feature_category: :code_review do
merge_requests = find_merge_requests(group_id: user_group.id, include_subgroups: true)
present merge_requests, serializer_options_for(merge_requests).merge(group: user_group)
@@ -184,7 +193,7 @@ module API
use :merge_requests_params
optional :iids, type: Array[Integer], coerce_with: ::API::Validations::Types::CommaSeparatedToIntegerArray.coerce, desc: 'The IID array of merge requests'
end
- get ":id/merge_requests" do
+ get ":id/merge_requests", feature_category: :code_review do
authorize! :read_merge_request, user_project
merge_requests = find_merge_requests(project_id: user_project.id)
@@ -206,8 +215,8 @@ module API
desc: 'The target project of the merge request defaults to the :id of the project'
use :optional_params
end
- post ":id/merge_requests" do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42316')
+ post ":id/merge_requests", feature_category: :code_review do
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20770')
authorize! :create_merge_request_from, user_project
@@ -228,7 +237,7 @@ module API
params do
requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request'
end
- delete ":id/merge_requests/:merge_request_iid" do
+ delete ":id/merge_requests/:merge_request_iid", feature_category: :code_review do
merge_request = find_project_merge_request(params[:merge_request_iid])
authorize!(:destroy_merge_request, merge_request)
@@ -247,7 +256,7 @@ module API
desc 'Get a single merge request' do
success Entities::MergeRequest
end
- get ':id/merge_requests/:merge_request_iid' do
+ get ':id/merge_requests/:merge_request_iid', feature_category: :code_review do
not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project)
merge_request = find_merge_request_with_access(params[:merge_request_iid])
@@ -265,7 +274,7 @@ module API
desc 'Get the participants of a merge request' do
success Entities::UserBasic
end
- get ':id/merge_requests/:merge_request_iid/participants' do
+ get ':id/merge_requests/:merge_request_iid/participants', feature_category: :code_review do
not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project)
merge_request = find_merge_request_with_access(params[:merge_request_iid])
@@ -278,7 +287,7 @@ module API
desc 'Get the commits of a merge request' do
success Entities::Commit
end
- get ':id/merge_requests/:merge_request_iid/commits' do
+ get ':id/merge_requests/:merge_request_iid/commits', feature_category: :code_review do
not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project)
merge_request = find_merge_request_with_access(params[:merge_request_iid])
@@ -293,7 +302,7 @@ module API
desc 'Get the context commits of a merge request' do
success Entities::Commit
end
- get ':id/merge_requests/:merge_request_iid/context_commits' do
+ get ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do
merge_request = find_merge_request_with_access(params[:merge_request_iid])
project = merge_request.project
@@ -311,7 +320,7 @@ module API
desc 'create context commits of merge request' do
success Entities::Commit
end
- post ':id/merge_requests/:merge_request_iid/context_commits' do
+ post ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do
commit_ids = params[:commits]
if commit_ids.size > CONTEXT_COMMITS_POST_LIMIT
@@ -339,7 +348,7 @@ module API
requires :commits, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, allow_blank: false, desc: 'List of context commits sha'
end
desc 'remove context commits of merge request'
- delete ':id/merge_requests/:merge_request_iid/context_commits' do
+ delete ':id/merge_requests/:merge_request_iid/context_commits', feature_category: :code_review do
commit_ids = params[:commits]
merge_request = find_merge_request_with_access(params[:merge_request_iid])
project = merge_request.project
@@ -361,7 +370,7 @@ module API
desc 'Show the merge request changes' do
success Entities::MergeRequestChanges
end
- get ':id/merge_requests/:merge_request_iid/changes' do
+ get ':id/merge_requests/:merge_request_iid/changes', feature_category: :code_review do
not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project)
merge_request = find_merge_request_with_access(params[:merge_request_iid])
@@ -376,7 +385,7 @@ module API
desc 'Get the merge request pipelines' do
success Entities::Ci::PipelineBasic
end
- get ':id/merge_requests/:merge_request_iid/pipelines' do
+ get ':id/merge_requests/:merge_request_iid/pipelines', feature_category: :continuous_integration do
pipelines = merge_request_pipelines_with_access
not_found!("Merge Request") unless can?(current_user, :read_merge_request, user_project)
@@ -387,7 +396,7 @@ module API
desc 'Create a pipeline for merge request' do
success ::API::Entities::Ci::Pipeline
end
- post ':id/merge_requests/:merge_request_iid/pipelines' do
+ post ':id/merge_requests/:merge_request_iid/pipelines', feature_category: :continuous_integration do
pipeline = ::MergeRequests::CreatePipelineService
.new(user_project, current_user, allow_duplicate: true)
.execute(find_merge_request_with_access(params[:merge_request_iid]))
@@ -415,8 +424,8 @@ module API
use :optional_params
at_least_one_of(*::API::MergeRequests.update_params_at_least_one_of)
end
- put ':id/merge_requests/:merge_request_iid' do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42318')
+ put ':id/merge_requests/:merge_request_iid', feature_category: :code_review do
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20772')
merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request)
@@ -424,7 +433,13 @@ module API
mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params.has_key?(:remove_source_branch)
mr_params = convert_parameters_from_legacy_format(mr_params)
- merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request)
+ service = if mr_params.one? && (mr_params.keys & %i[assignee_id assignee_ids]).one?
+ ::MergeRequests::UpdateAssigneesService
+ else
+ ::MergeRequests::UpdateService
+ end
+
+ merge_request = service.new(user_project, current_user, mr_params).execute(merge_request)
handle_merge_request_errors!(merge_request)
@@ -444,8 +459,8 @@ module API
optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch'
optional :squash, type: Grape::API::Boolean, desc: 'When true, the commits will be squashed into a single commit on merge'
end
- put ':id/merge_requests/:merge_request_iid/merge' do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42317')
+ put ':id/merge_requests/:merge_request_iid/merge', feature_category: :code_review do
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796')
merge_request = find_project_merge_request(params[:merge_request_iid])
@@ -485,7 +500,7 @@ module API
end
desc 'Returns the up to date merge-ref HEAD commit'
- get ':id/merge_requests/:merge_request_iid/merge_ref' do
+ get ':id/merge_requests/:merge_request_iid/merge_ref', feature_category: :code_review do
merge_request = find_project_merge_request(params[:merge_request_iid])
result = ::MergeRequests::MergeabilityCheckService.new(merge_request).execute(recheck: true)
@@ -500,7 +515,7 @@ module API
desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do
success Entities::MergeRequest
end
- post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds' do
+ post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds', feature_category: :code_review do
merge_request = find_project_merge_request(params[:merge_request_iid])
unauthorized! unless merge_request.can_cancel_auto_merge?(current_user)
@@ -514,7 +529,7 @@ module API
params do
optional :skip_ci, type: Boolean, desc: 'Do not create CI pipeline'
end
- put ':id/merge_requests/:merge_request_iid/rebase' do
+ put ':id/merge_requests/:merge_request_iid/rebase', feature_category: :code_review do
merge_request = find_project_merge_request(params[:merge_request_iid])
authorize_push_to_merge_request!(merge_request)
@@ -533,7 +548,7 @@ module API
params do
use :pagination
end
- get ':id/merge_requests/:merge_request_iid/closes_issues' do
+ get ':id/merge_requests/:merge_request_iid/closes_issues', feature_category: :code_review do
merge_request = find_merge_request_with_access(params[:merge_request_iid])
issues = ::Kaminari.paginate_array(merge_request.visible_closing_issues_for(current_user))
issues = paginate(issues)
diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb
index b337b992841..d75ed3a48d7 100644
--- a/lib/api/milestone_responses.rb
+++ b/lib/api/milestone_responses.rb
@@ -80,7 +80,7 @@ module API
params = build_finder_params(milestone, parent)
- issuables = finder_klass.new(current_user, params).execute
+ issuables = finder_klass.new(current_user, params).execute.with_api_entity_associations
present paginate(issuables), with: entity, current_user: current_user
end
diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb
index 25a901c18b6..465d2f23e9d 100644
--- a/lib/api/namespaces.rb
+++ b/lib/api/namespaces.rb
@@ -56,6 +56,23 @@ module API
present user_namespace, with: Entities::Namespace, current_user: current_user
end
+
+ desc 'Get existence of a namespace including alternative suggestions' do
+ success Entities::NamespaceExistence
+ end
+ params do
+ requires :namespace, type: String, desc: "Namespace's path"
+ optional :parent_id, type: Integer, desc: "The ID of the parent namespace. If no ID is specified, only top-level namespaces are considered."
+ end
+ get ':namespace/exists', requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ namespace_path = params[:namespace]
+
+ exists = Namespace.by_parent(params[:parent_id]).filter_by_path(namespace_path).exists?
+ suggestions = exists ? [Namespace.clean_path(namespace_path)] : []
+
+ present :exists, exists
+ present :suggests, suggestions
+ end
end
end
end
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index 15b06cea385..5f3a574eeee 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -67,7 +67,7 @@ module API
check_rate_limit! :project_import, [current_user, :project_import]
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42437')
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20823')
validate_file!
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 19b63c28f89..92f6970e6fc 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -13,6 +13,8 @@ module API
feature_category :projects, ['/projects/:id/custom_attributes', '/projects/:id/custom_attributes/:key']
+ PROJECT_ATTACHMENT_SIZE_EXEMPT = 1.gigabyte
+
helpers do
# EE::API::Projects would override this method
def apply_filters(projects)
@@ -52,6 +54,29 @@ module API
accepted!
end
+
+ def exempt_from_global_attachment_size?(user_project)
+ list = ::Gitlab::RackAttack::UserAllowlist.new(ENV['GITLAB_UPLOAD_API_ALLOWLIST'])
+ list.include?(user_project.id)
+ end
+
+ # Temporarily introduced for upload API: https://gitlab.com/gitlab-org/gitlab/-/issues/325788
+ def project_attachment_size(user_project)
+ return PROJECT_ATTACHMENT_SIZE_EXEMPT if exempt_from_global_attachment_size?(user_project)
+ return user_project.max_attachment_size if Feature.enabled?(:enforce_max_attachment_size_upload_api, user_project)
+
+ PROJECT_ATTACHMENT_SIZE_EXEMPT
+ end
+
+ # This is to help determine which projects to use in https://gitlab.com/gitlab-org/gitlab/-/issues/325788
+ def log_if_upload_exceed_max_size(user_project, file)
+ return if file.size <= user_project.max_attachment_size
+
+ if file.size > user_project.max_attachment_size
+ allowed = exempt_from_global_attachment_size?(user_project)
+ Gitlab::AppLogger.info({ message: "File exceeds maximum size", file_bytes: file.size, project_id: user_project.id, project_path: user_project.full_path, upload_allowed: allowed })
+ end
+ end
end
helpers do
@@ -215,7 +240,7 @@ module API
use :create_params
end
post do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/21139')
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/issues/21139')
attrs = declared_params(include_missing: false)
attrs = translate_params_for_compatibility(attrs)
filter_attributes_using_license!(attrs)
@@ -248,7 +273,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
post "user/:user_id", feature_category: :projects do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/21139')
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/issues/21139')
authenticated_as_admin!
user = User.find_by(id: params.delete(:user_id))
not_found!('User') unless user
@@ -310,7 +335,7 @@ module API
optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the fork'
end
post ':id/fork', feature_category: :source_code_management do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42284')
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20759')
not_found! unless can?(current_user, :fork_project, user_project)
@@ -460,7 +485,7 @@ module API
get ':id/languages', feature_category: :source_code_management do
::Projects::RepositoryLanguagesService
.new(user_project, current_user)
- .execute.map { |lang| [lang.name, lang.share] }.to_h
+ .execute.to_h { |lang| [lang.name, lang.share] }
end
desc 'Delete a project'
@@ -545,13 +570,27 @@ module API
end
# rubocop: enable CodeReuse/ActiveRecord
+ desc 'Workhorse authorize the file upload' do
+ detail 'This feature was introduced in GitLab 13.11'
+ end
+ post ':id/uploads/authorize', feature_category: :not_owned do
+ require_gitlab_workhorse!
+
+ status 200
+ content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
+ FileUploader.workhorse_authorize(has_length: false, maximum_size: project_attachment_size(user_project))
+ end
+
desc 'Upload a file'
params do
- # TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
- requires :file, type: File, desc: 'The file to be uploaded' # rubocop:disable Scalability/FileUploads
+ requires :file, types: [Rack::Multipart::UploadedFile, ::API::Validations::Types::WorkhorseFile], desc: 'The attachment file to be uploaded'
end
post ":id/uploads", feature_category: :not_owned do
- upload = UploadService.new(user_project, params[:file]).execute
+ log_if_upload_exceed_max_size(user_project, params[:file])
+
+ service = UploadService.new(user_project, params[:file])
+ service.override_max_attachment_size = project_attachment_size(user_project)
+ upload = service.execute
present upload, with: Entities::ProjectUpload
end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index f6ffeeea829..033cc6744b0 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -116,10 +116,23 @@ module API
params do
requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison'
requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison'
+ optional :from_project_id, type: String, desc: 'The project to compare from'
optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false
end
get ':id/repository/compare' do
- compare = CompareService.new(user_project, params[:to]).execute(user_project, params[:from], straight: params[:straight])
+ if params[:from_project_id].present?
+ target_project = MergeRequestTargetProjectFinder
+ .new(current_user: current_user, source_project: user_project, project_feature: :repository)
+ .execute(include_routes: true).find_by_id(params[:from_project_id])
+
+ if target_project.blank?
+ render_api_error!("Target project id:#{params[:from_project_id]} is not a fork of project id:#{params[:id]}", 400)
+ end
+ else
+ target_project = user_project
+ end
+
+ compare = CompareService.new(user_project, params[:to]).execute(target_project, params[:from], straight: params[:straight])
if compare
present compare, with: Entities::Compare
diff --git a/lib/api/resource_access_tokens.rb b/lib/api/resource_access_tokens.rb
index 99c278be8e7..705e4778c83 100644
--- a/lib/api/resource_access_tokens.rb
+++ b/lib/api/resource_access_tokens.rb
@@ -19,7 +19,7 @@ module API
get ":id/access_tokens" do
resource = find_source(source_type, params[:id])
- next unauthorized! unless has_permission_to_read?(resource)
+ next unauthorized! unless current_user.can?(:read_resource_access_tokens, resource)
tokens = PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).execute
@@ -85,10 +85,6 @@ module API
def find_token(resource, token_id)
PersonalAccessTokensFinder.new({ user: resource.bots, impersonation: false }).find_by_id(token_id)
end
-
- def has_permission_to_read?(resource)
- can?(current_user, :project_bot_access, resource) || can?(current_user, :admin_resource_access_tokens, resource)
- end
end
end
end
diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb
index 8d2d4586d8d..1d17148e0df 100644
--- a/lib/api/rubygem_packages.rb
+++ b/lib/api/rubygem_packages.rb
@@ -99,6 +99,8 @@ module API
track_package_event('push_package', :rubygems)
+ package_file = nil
+
ActiveRecord::Base.transaction do
package = ::Packages::CreateTemporaryPackageService.new(
user_project, current_user, declared_params.merge(build: current_authenticated_job)
@@ -109,12 +111,18 @@ module API
file_name: PACKAGE_FILENAME
}
- ::Packages::CreatePackageFileService.new(
+ package_file = ::Packages::CreatePackageFileService.new(
package, file_params.merge(build: current_authenticated_job)
).execute
end
- created!
+ if package_file
+ ::Packages::Rubygems::ExtractionWorker.perform_async(package_file.id) # rubocop:disable CodeReuse/Worker
+
+ created!
+ else
+ bad_request!('Package creation failed')
+ end
rescue ObjectStorage::RemoteStoreError => e
Gitlab::ErrorTracking.track_exception(e, extra: { file_name: params[:file_name], project_id: user_project.id })
diff --git a/lib/api/search.rb b/lib/api/search.rb
index f0ffe6ba443..8fabf379d49 100644
--- a/lib/api/search.rb
+++ b/lib/api/search.rb
@@ -22,13 +22,15 @@ module API
users: Entities::UserBasic
}.freeze
- SCOPE_PRELOAD_METHOD = {
- merge_requests: :with_api_entity_associations,
- projects: :with_api_entity_associations,
- issues: :with_api_entity_associations,
- milestones: :with_api_entity_associations,
- commits: :with_api_commit_entity_associations
- }.freeze
+ def scope_preload_method
+ {
+ merge_requests: :with_api_entity_associations,
+ projects: :with_api_entity_associations,
+ issues: :with_api_entity_associations,
+ milestones: :with_api_entity_associations,
+ commits: :with_api_commit_entity_associations
+ }.freeze
+ end
def search(additional_params = {})
search_params = {
@@ -60,7 +62,7 @@ module API
end
def preload_method
- SCOPE_PRELOAD_METHOD[params[:scope].to_sym]
+ scope_preload_method[params[:scope].to_sym]
end
def verify_search_scope!(resource:)
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 64a72b4cb7f..95d0c525ced 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -30,6 +30,7 @@ module API
success Entities::ApplicationSetting
end
params do
+ optional :admin_mode, type: Boolean, desc: 'Require admin users to re-authenticate for administrative (i.e. potentially dangerous) operations'
optional :admin_notification_email, type: String, desc: 'Deprecated: Use :abuse_notification_email instead. Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.'
optional :abuse_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.'
optional :after_sign_up_text, type: String, desc: 'Text shown after sign up'
diff --git a/lib/api/tags.rb b/lib/api/tags.rb
index 7636c45bdac..e77d7e34de3 100644
--- a/lib/api/tags.rb
+++ b/lib/api/tags.rb
@@ -28,7 +28,13 @@ module API
sort: "#{params[:order_by]}_#{params[:sort]}",
search: params[:search]).execute
- present paginate(::Kaminari.paginate_array(tags)), with: Entities::Tag, project: user_project
+ paginated_tags = paginate(::Kaminari.paginate_array(tags))
+
+ if Feature.enabled?(:api_caching_tags, user_project, type: :development)
+ present_cached paginated_tags, with: Entities::Tag, project: user_project, cache_context: -> (_tag) { user_project.cache_key }
+ else
+ present paginated_tags, with: Entities::Tag, project: user_project
+ end
end
desc 'Get a single repository tag' do
diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb
index aebbc95cbea..84c51e5aeac 100644
--- a/lib/api/triggers.rb
+++ b/lib/api/triggers.rb
@@ -21,7 +21,7 @@ module API
optional :variables, type: Hash, desc: 'The list of variables to be injected into build'
end
post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-foss/issues/42283')
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20758')
forbidden! if gitlab_pipeline_hook_request?
diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb
index c7d63f8d6ac..7deec15dcac 100644
--- a/lib/api/usage_data.rb
+++ b/lib/api/usage_data.rb
@@ -2,24 +2,22 @@
module API
class UsageData < ::API::Base
- before { authenticate! }
+ before { authenticate_non_get! }
feature_category :usage_ping
namespace 'usage_data' do
before do
- not_found! unless Feature.enabled?(:usage_data_api, default_enabled: true)
+ not_found! unless Feature.enabled?(:usage_data_api, default_enabled: :yaml, type: :ops)
forbidden!('Invalid CSRF token is provided') unless verified_request?
end
desc 'Track usage data events' do
detail 'This feature was introduced in GitLab 13.4.'
end
-
params do
requires :event, type: String, desc: 'The event name that should be tracked'
end
-
post 'increment_counter' do
event_name = params[:event]
@@ -31,7 +29,6 @@ module API
params do
requires :event, type: String, desc: 'The event name that should be tracked'
end
-
post 'increment_unique_users' do
event_name = params[:event]
@@ -39,6 +36,16 @@ module API
status :ok
end
+
+ desc 'Get a list of all metric definitions' do
+ detail 'This feature was introduced in GitLab 13.11.'
+ end
+ get 'metric_definitions' do
+ content_type 'application/yaml'
+ env['api.format'] = :binary
+
+ Gitlab::Usage::MetricDefinition.dump_metrics_yaml
+ end
end
end
end
diff --git a/lib/api/usage_data_non_sql_metrics.rb b/lib/api/usage_data_non_sql_metrics.rb
new file mode 100644
index 00000000000..63a14a223f5
--- /dev/null
+++ b/lib/api/usage_data_non_sql_metrics.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module API
+ class UsageDataNonSqlMetrics < ::API::Base
+ before { authenticated_as_admin! }
+
+ feature_category :usage_ping
+
+ namespace 'usage_data' do
+ before do
+ not_found! unless Feature.enabled?(:usage_data_non_sql_metrics, default_enabled: :yaml, type: :ops)
+ end
+
+ desc 'Get Non SQL usage ping metrics' do
+ detail 'This feature was introduced in GitLab 13.11.'
+ end
+
+ get 'non_sql_metrics' do
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/325534')
+
+ data = Gitlab::UsageDataNonSqlMetrics.uncached_data
+
+ present data
+ end
+ end
+ end
+end
diff --git a/lib/api/usage_data_queries.rb b/lib/api/usage_data_queries.rb
new file mode 100644
index 00000000000..0ad9ad7650c
--- /dev/null
+++ b/lib/api/usage_data_queries.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module API
+ class UsageDataQueries < ::API::Base
+ before { authenticated_as_admin! }
+
+ feature_category :usage_ping
+
+ namespace 'usage_data' do
+ before do
+ not_found! unless Feature.enabled?(:usage_data_queries_api, default_enabled: :yaml, type: :ops)
+ end
+
+ desc 'Get raw SQL queries for usage data SQL metrics' do
+ detail 'This feature was introduced in GitLab 13.11.'
+ end
+
+ get 'queries' do
+ Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/325534')
+
+ queries = Gitlab::UsageDataQueries.uncached_data
+
+ present queries
+ end
+ end
+ end
+end
diff --git a/lib/api/users.rb b/lib/api/users.rb
index b2f99bb18dc..078ba7542a3 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -231,7 +231,7 @@ module API
optional :password, type: String, desc: 'The password of the new user'
optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token'
optional :skip_confirmation, type: Boolean, desc: 'Flag indicating the account is confirmed'
- at_least_one_of :password, :reset_password
+ at_least_one_of :password, :reset_password, :force_random_password
requires :name, type: String, desc: 'The name of the user'
requires :username, type: String, desc: 'The username of the user'
optional :force_random_password, type: Boolean, desc: 'Flag indicating a random password will be set'
@@ -571,8 +571,6 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
delete ":id", feature_category: :users do
- Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/issues/20757')
-
authenticated_as_admin!
user = User.find_by(id: params[:id])
@@ -998,6 +996,29 @@ module API
present paginate(current_user.emails), with: Entities::Email
end
+ desc "Update the current user's preferences" do
+ success Entities::UserPreferences
+ detail 'This feature was introduced in GitLab 13.10.'
+ end
+ params do
+ requires :view_diffs_file_by_file, type: Boolean, desc: 'Flag indicating the user sees only one file diff per page'
+ end
+ put "preferences", feature_category: :users do
+ authenticate!
+
+ preferences = current_user.user_preference
+
+ attrs = declared_params(include_missing: false)
+
+ service = ::UserPreferences::UpdateService.new(current_user, attrs).execute
+
+ if service.success?
+ present preferences, with: Entities::UserPreferences
+ else
+ render_api_error!('400 Bad Request', 400)
+ end
+ end
+
desc 'Get a single email address owned by the currently authenticated user' do
success Entities::Email
end
diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb
index 2d25e76626a..29e4a79110f 100644
--- a/lib/api/v3/github.rb
+++ b/lib/api/v3/github.rb
@@ -18,7 +18,7 @@ module API
# Used to differentiate Jira Cloud requests from Jira Server requests
# Jira Cloud user agent format: Jira DVCS Connector Vertigo/version
# Jira Server user agent format: Jira DVCS Connector/version
- JIRA_DVCS_CLOUD_USER_AGENT = 'Jira DVCS Connector Vertigo'.freeze
+ JIRA_DVCS_CLOUD_USER_AGENT = 'Jira DVCS Connector Vertigo'
include PaginationParams
@@ -75,11 +75,14 @@ module API
# rubocop: enable CodeReuse/ActiveRecord
def authorized_merge_requests
- MergeRequestsFinder.new(current_user, authorized_only: !current_user.admin?).execute
+ MergeRequestsFinder.new(current_user, authorized_only: !current_user.admin?)
+ .execute.with_jira_integration_associations
end
def authorized_merge_requests_for_project(project)
- MergeRequestsFinder.new(current_user, authorized_only: !current_user.admin?, project_id: project.id).execute
+ MergeRequestsFinder
+ .new(current_user, authorized_only: !current_user.admin?, project_id: project.id)
+ .execute.with_jira_integration_associations
end
# rubocop: disable CodeReuse/ActiveRecord
@@ -194,16 +197,13 @@ module API
# Self-hosted Jira (tested on 7.11.1) requests this endpoint right
# after fetching branches.
- # rubocop: disable CodeReuse/ActiveRecord
get ':namespace/:project/events' do
user_project = find_project_with_access(params)
merge_requests = authorized_merge_requests_for_project(user_project)
- merge_requests = merge_requests.preload(:author, :assignees, :metrics, source_project: :namespace, target_project: :namespace)
present paginate(merge_requests), with: ::API::Github::Entities::PullRequestEvent
end
- # rubocop: enable CodeReuse/ActiveRecord
params do
use :project_full_path
diff --git a/lib/api/variables.rb b/lib/api/variables.rb
index 94fa98b7a14..8b0745c6b5b 100644
--- a/lib/api/variables.rb
+++ b/lib/api/variables.rb
@@ -9,21 +9,7 @@ module API
feature_category :continuous_integration
- helpers do
- def filter_variable_parameters(params)
- # This method exists so that EE can more easily filter out certain
- # parameters, without having to modify the source code directly.
- params
- end
-
- def find_variable(params)
- variables = ::Ci::VariablesFinder.new(user_project, params).execute.to_a
-
- return variables.first unless variables.many? # rubocop: disable CodeReuse/ActiveRecord
-
- conflict!("There are multiple variables with provided parameters. Please use 'filter[environment_scope]'")
- end
- end
+ helpers Helpers::VariablesHelpers
params do
requires :id, type: String, desc: 'The ID of a project'
@@ -49,7 +35,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
get ':id/variables/:key' do
- variable = find_variable(params)
+ variable = find_variable(user_project, params)
not_found!('Variable') unless variable
present variable, with: Entities::Ci::Variable
@@ -71,7 +57,7 @@ module API
variable = ::Ci::ChangeVariableService.new(
container: user_project,
current_user: current_user,
- params: { action: :create, variable_params: filter_variable_parameters(declared_params(include_missing: false)) }
+ params: { action: :create, variable_params: declared_params(include_missing: false) }
).execute
if variable.valid?
@@ -95,17 +81,13 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
put ':id/variables/:key' do
- variable = find_variable(params)
+ variable = find_variable(user_project, params)
not_found!('Variable') unless variable
- variable_params = filter_variable_parameters(
- declared_params(include_missing: false)
- .except(:key, :filter)
- )
variable = ::Ci::ChangeVariableService.new(
container: user_project,
current_user: current_user,
- params: { action: :update, variable: variable, variable_params: variable_params }
+ params: { action: :update, variable: variable, variable_params: declared_params(include_missing: false).except(:key, :filter) }
).execute
if variable.valid?
@@ -125,7 +107,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
delete ':id/variables/:key' do
- variable = find_variable(params)
+ variable = find_variable(user_project, params)
not_found!('Variable') unless variable
::Ci::ChangeVariableService.new(
diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb
index 6f87b7b7d3c..ea83076c49b 100644
--- a/lib/atlassian/jira_connect/client.rb
+++ b/lib/atlassian/jira_connect/client.rb
@@ -141,9 +141,9 @@ module Atlassian
def user_notes_count(merge_requests)
return unless merge_requests
- Note.count_for_collection(merge_requests.map(&:id), 'MergeRequest').map do |count_group|
+ Note.count_for_collection(merge_requests.map(&:id), 'MergeRequest').to_h do |count_group|
[count_group.noteable_id, count_group.count]
- end.to_h
+ end
end
def jwt_token(http_method, uri)
diff --git a/lib/atlassian/jira_connect/serializers/build_entity.rb b/lib/atlassian/jira_connect/serializers/build_entity.rb
index 8372d2a62da..a3434c529a4 100644
--- a/lib/atlassian/jira_connect/serializers/build_entity.rb
+++ b/lib/atlassian/jira_connect/serializers/build_entity.rb
@@ -25,11 +25,11 @@ module Atlassian
# extract Jira issue keys from either the source branch/ref or the
# merge request title.
@issue_keys ||= begin
- pipeline.all_merge_requests.flat_map do |mr|
- src = "#{mr.source_branch} #{mr.title}"
- JiraIssueKeyExtractor.new(src).issue_keys
- end.uniq
- end
+ pipeline.all_merge_requests.flat_map do |mr|
+ src = "#{mr.source_branch} #{mr.title}"
+ JiraIssueKeyExtractor.new(src).issue_keys
+ end.uniq
+ end
end
private
diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb
index 79b7b2c61f2..627bb44331b 100644
--- a/lib/backup/repositories.rb
+++ b/lib/backup/repositories.rb
@@ -249,7 +249,7 @@ module Backup
progress.puts " * #{display_repo_path} ... "
if repository.empty?
- progress.puts " * #{display_repo_path} ... " + "[SKIPPED]".color(:cyan)
+ progress.puts " * #{display_repo_path} ... " + "[EMPTY] [SKIPPED]".color(:cyan)
return
end
diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb
deleted file mode 100644
index 2448c2c2bb2..00000000000
--- a/lib/banzai/filter/abstract_reference_filter.rb
+++ /dev/null
@@ -1,446 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # Issues, Merge Requests, Snippets, Commits and Commit Ranges share
- # similar functionality in reference filtering.
- class AbstractReferenceFilter < ReferenceFilter
- include CrossProjectReference
-
- # REFERENCE_PLACEHOLDER is used for re-escaping HTML text except found
- # reference (which we replace with placeholder during re-scaping). The
- # random number helps ensure it's pretty close to unique. Since it's a
- # transitory value (it never gets saved) we can initialize once, and it
- # doesn't matter if it changes on a restart.
- REFERENCE_PLACEHOLDER = "_reference_#{SecureRandom.hex(16)}_"
- REFERENCE_PLACEHOLDER_PATTERN = %r{#{REFERENCE_PLACEHOLDER}(\d+)}.freeze
-
- def self.object_class
- # Implement in child class
- # Example: MergeRequest
- end
-
- def self.object_name
- @object_name ||= object_class.name.underscore
- end
-
- def self.object_sym
- @object_sym ||= object_name.to_sym
- end
-
- # Public: Find references in text (like `!123` for merge requests)
- #
- # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches|
- # object = find_object(project_ref, id)
- # "<a href=...>#{object.to_reference}</a>"
- # end
- #
- # text - String text to search.
- #
- # Yields the String match, the Integer referenced object ID, an optional String
- # of the external project reference, and all of the matchdata.
- #
- # Returns a String replaced with the return of the block.
- def self.references_in(text, pattern = object_class.reference_pattern)
- text.gsub(pattern) do |match|
- if ident = identifier($~)
- yield match, ident, $~[:project], $~[:namespace], $~
- else
- match
- end
- end
- end
-
- def self.identifier(match_data)
- symbol = symbol_from_match(match_data)
-
- parse_symbol(symbol, match_data) if object_class.reference_valid?(symbol)
- end
-
- def identifier(match_data)
- self.class.identifier(match_data)
- end
-
- def self.symbol_from_match(match)
- key = object_sym
- match[key] if match.names.include?(key.to_s)
- end
-
- # Transform a symbol extracted from the text to a meaningful value
- # In most cases these will be integers, so we call #to_i by default
- #
- # This method has the contract that if a string `ref` refers to a
- # record `record`, then `parse_symbol(ref) == record_identifier(record)`.
- def self.parse_symbol(symbol, match_data)
- symbol.to_i
- end
-
- # We assume that most classes are identifying records by ID.
- #
- # This method has the contract that if a string `ref` refers to a
- # record `record`, then `class.parse_symbol(ref) == record_identifier(record)`.
- def record_identifier(record)
- record.id
- end
-
- def object_class
- self.class.object_class
- end
-
- def object_sym
- self.class.object_sym
- end
-
- def references_in(*args, &block)
- self.class.references_in(*args, &block)
- end
-
- # Implement in child class
- # Example: project.merge_requests.find
- def find_object(parent_object, id)
- end
-
- # Override if the link reference pattern produces a different ID (global
- # ID vs internal ID, for instance) to the regular reference pattern.
- def find_object_from_link(parent_object, id)
- find_object(parent_object, id)
- end
-
- # Implement in child class
- # Example: project_merge_request_url
- def url_for_object(object, parent_object)
- end
-
- def find_object_cached(parent_object, id)
- cached_call(:banzai_find_object, id, path: [object_class, parent_object.id]) do
- find_object(parent_object, id)
- end
- end
-
- def find_object_from_link_cached(parent_object, id)
- cached_call(:banzai_find_object_from_link, id, path: [object_class, parent_object.id]) do
- find_object_from_link(parent_object, id)
- end
- end
-
- def from_ref_cached(ref)
- cached_call("banzai_#{parent_type}_refs".to_sym, ref) do
- parent_from_ref(ref)
- end
- end
-
- def url_for_object_cached(object, parent_object)
- cached_call(:banzai_url_for_object, object, path: [object_class, parent_object.id]) do
- url_for_object(object, parent_object)
- end
- end
-
- def call
- return doc unless project || group || user
-
- ref_pattern = object_class.reference_pattern
- link_pattern = object_class.link_reference_pattern
-
- # Compile often used regexps only once outside of the loop
- ref_pattern_anchor = /\A#{ref_pattern}\z/
- link_pattern_start = /\A#{link_pattern}/
- link_pattern_anchor = /\A#{link_pattern}\z/
-
- nodes.each_with_index do |node, index|
- if text_node?(node) && ref_pattern
- replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
- object_link_filter(content, ref_pattern)
- end
-
- elsif element_node?(node)
- yield_valid_link(node) do |link, inner_html|
- if ref_pattern && link =~ ref_pattern_anchor
- replace_link_node_with_href(node, index, link) do
- object_link_filter(link, ref_pattern, link_content: inner_html)
- end
-
- next
- end
-
- next unless link_pattern
-
- if link == inner_html && inner_html =~ link_pattern_start
- replace_link_node_with_text(node, index) do
- object_link_filter(inner_html, link_pattern, link_reference: true)
- end
-
- next
- end
-
- if link =~ link_pattern_anchor
- replace_link_node_with_href(node, index, link) do
- object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true)
- end
-
- next
- end
- end
- end
- end
-
- doc
- end
-
- # Replace references (like `!123` for merge requests) in text with links
- # to the referenced object's details page.
- #
- # text - String text to replace references in.
- # pattern - Reference pattern to match against.
- # link_content - Original content of the link being replaced.
- # link_reference - True if this was using the link reference pattern,
- # false otherwise.
- #
- # Returns a String with references replaced with links. All links
- # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling.
- def object_link_filter(text, pattern, link_content: nil, link_reference: false)
- references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches|
- parent_path = if parent_type == :group
- full_group_path(namespace_ref)
- else
- full_project_path(namespace_ref, project_ref)
- end
-
- parent = from_ref_cached(parent_path)
-
- if parent
- object =
- if link_reference
- find_object_from_link_cached(parent, id)
- else
- find_object_cached(parent, id)
- end
- end
-
- if object
- title = object_link_title(object, matches)
- klass = reference_class(object_sym)
-
- data_attributes = data_attributes_for(link_content || match, parent, object,
- link_content: !!link_content,
- link_reference: link_reference)
- data = data_attribute(data_attributes)
-
- url =
- if matches.names.include?("url") && matches[:url]
- matches[:url]
- else
- url_for_object_cached(object, parent)
- end
-
- content = link_content || object_link_text(object, matches)
-
- link = %(<a href="#{url}" #{data}
- title="#{escape_once(title)}"
- class="#{klass}">#{content}</a>)
-
- wrap_link(link, object)
- else
- match
- end
- end
- end
-
- def wrap_link(link, object)
- link
- end
-
- def data_attributes_for(text, parent, object, link_content: false, link_reference: false)
- object_parent_type = parent.is_a?(Group) ? :group : :project
-
- {
- original: escape_html_entities(text),
- link: link_content,
- link_reference: link_reference,
- object_parent_type => parent.id,
- object_sym => object.id
- }
- end
-
- def object_link_text_extras(object, matches)
- extras = []
-
- if matches.names.include?("anchor") && matches[:anchor] && matches[:anchor] =~ /\A\#note_(\d+)\z/
- extras << "comment #{Regexp.last_match(1)}"
- end
-
- extension = matches[:extension] if matches.names.include?("extension")
-
- extras << extension if extension
-
- extras
- end
-
- def object_link_title(object, matches)
- object.title
- end
-
- def object_link_text(object, matches)
- parent = project || group || user
- text = object.reference_link_text(parent)
-
- extras = object_link_text_extras(object, matches)
- text += " (#{extras.join(", ")})" if extras.any?
-
- text
- end
-
- # Returns a Hash containing all object references (e.g. issue IDs) per the
- # project they belong to.
- def references_per_parent
- @references_per ||= {}
-
- @references_per[parent_type] ||= begin
- refs = Hash.new { |hash, key| hash[key] = Set.new }
- regex = [
- object_class.link_reference_pattern,
- object_class.reference_pattern
- ].compact.reduce { |a, b| Regexp.union(a, b) }
-
- nodes.each do |node|
- node.to_html.scan(regex) do
- path = if parent_type == :project
- full_project_path($~[:namespace], $~[:project])
- else
- full_group_path($~[:group])
- end
-
- if ident = identifier($~)
- refs[path] << ident
- end
- end
- end
-
- refs
- end
- end
-
- # Returns a Hash containing referenced projects grouped per their full
- # path.
- def parent_per_reference
- @per_reference ||= {}
-
- @per_reference[parent_type] ||= begin
- refs = Set.new
-
- references_per_parent.each do |ref, _|
- refs << ref
- end
-
- find_for_paths(refs.to_a).index_by(&:full_path)
- end
- end
-
- def relation_for_paths(paths)
- klass = parent_type.to_s.camelize.constantize
- result = klass.where_full_path_in(paths)
- return result if parent_type == :group
-
- result.includes(:namespace) if parent_type == :project
- end
-
- # Returns projects for the given paths.
- def find_for_paths(paths)
- if Gitlab::SafeRequestStore.active?
- cache = refs_cache
- to_query = paths - cache.keys
-
- unless to_query.empty?
- records = relation_for_paths(to_query)
-
- found = []
- records.each do |record|
- ref = record.full_path
- get_or_set_cache(cache, ref) { record }
- found << ref
- end
-
- not_found = to_query - found
- not_found.each do |ref|
- get_or_set_cache(cache, ref) { nil }
- end
- end
-
- cache.slice(*paths).values.compact
- else
- relation_for_paths(paths)
- end
- end
-
- def current_parent_path
- @current_parent_path ||= parent&.full_path
- end
-
- def current_project_namespace_path
- @current_project_namespace_path ||= project&.namespace&.full_path
- end
-
- def records_per_parent
- @_records_per_project ||= {}
-
- @_records_per_project[object_class.to_s.underscore] ||= begin
- hash = Hash.new { |h, k| h[k] = {} }
-
- parent_per_reference.each do |path, parent|
- record_ids = references_per_parent[path]
-
- parent_records(parent, record_ids).each do |record|
- hash[parent][record_identifier(record)] = record
- end
- end
-
- hash
- end
- end
-
- private
-
- def full_project_path(namespace, project_ref)
- return current_parent_path unless project_ref
-
- namespace_ref = namespace || current_project_namespace_path
- "#{namespace_ref}/#{project_ref}"
- end
-
- def refs_cache
- Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {}
- end
-
- def parent_type
- :project
- end
-
- def parent
- parent_type == :project ? project : group
- end
-
- def full_group_path(group_ref)
- return current_parent_path unless group_ref
-
- group_ref
- end
-
- def unescape_html_entities(text)
- CGI.unescapeHTML(text.to_s)
- end
-
- def escape_html_entities(text)
- CGI.escapeHTML(text.to_s)
- end
-
- def escape_with_placeholders(text, placeholder_data)
- escaped = escape_html_entities(text)
-
- escaped.gsub(REFERENCE_PLACEHOLDER_PATTERN) do |match|
- placeholder_data[Regexp.last_match(1).to_i]
- end
- end
- end
- end
-end
-
-Banzai::Filter::AbstractReferenceFilter.prepend_if_ee('EE::Banzai::Filter::AbstractReferenceFilter')
diff --git a/lib/banzai/filter/alert_reference_filter.rb b/lib/banzai/filter/alert_reference_filter.rb
deleted file mode 100644
index 228a4159c99..00000000000
--- a/lib/banzai/filter/alert_reference_filter.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- class AlertReferenceFilter < IssuableReferenceFilter
- self.reference_type = :alert
-
- def self.object_class
- AlertManagement::Alert
- end
-
- def self.object_sym
- :alert
- end
-
- def parent_records(parent, ids)
- parent.alert_management_alerts.where(iid: ids.to_a)
- end
-
- def url_for_object(alert, project)
- ::Gitlab::Routing.url_helpers.details_project_alert_management_url(
- project,
- alert.iid,
- only_path: context[:only_path]
- )
- end
- end
- end
-end
diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb
index d569711431c..a86c1bb2892 100644
--- a/lib/banzai/filter/autolink_filter.rb
+++ b/lib/banzai/filter/autolink_filter.rb
@@ -43,7 +43,7 @@ module Banzai
TEXT_QUERY = %Q(descendant-or-self::text()[
not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')})
and contains(., '://')
- ]).freeze
+ ])
PUNCTUATION_PAIRS = {
"'" => "'",
diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb
deleted file mode 100644
index d6b46236a49..00000000000
--- a/lib/banzai/filter/commit_range_reference_filter.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces commit range references with links.
- #
- # This filter supports cross-project references.
- class CommitRangeReferenceFilter < AbstractReferenceFilter
- self.reference_type = :commit_range
-
- def self.object_class
- CommitRange
- end
-
- def self.references_in(text, pattern = CommitRange.reference_pattern)
- text.gsub(pattern) do |match|
- yield match, $~[:commit_range], $~[:project], $~[:namespace], $~
- end
- end
-
- def initialize(*args)
- super
-
- @commit_map = {}
- end
-
- def find_object(project, id)
- return unless project.is_a?(Project)
-
- range = CommitRange.new(id, project)
-
- range.valid_commits? ? range : nil
- end
-
- def url_for_object(range, project)
- h = Gitlab::Routing.url_helpers
- h.project_compare_url(project,
- range.to_param.merge(only_path: context[:only_path]))
- end
-
- def object_link_title(range, matches)
- nil
- end
- end
- end
-end
diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb
deleted file mode 100644
index 3df003a88fa..00000000000
--- a/lib/banzai/filter/commit_reference_filter.rb
+++ /dev/null
@@ -1,86 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces commit references with links.
- #
- # This filter supports cross-project references.
- class CommitReferenceFilter < AbstractReferenceFilter
- self.reference_type = :commit
-
- def self.object_class
- Commit
- end
-
- def self.references_in(text, pattern = Commit.reference_pattern)
- text.gsub(pattern) do |match|
- yield match, $~[:commit], $~[:project], $~[:namespace], $~
- end
- end
-
- def find_object(project, id)
- return unless project.is_a?(Project) && project.valid_repo?
-
- _, record = records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) }
-
- record
- end
-
- def referenced_merge_request_commit_shas
- return [] unless noteable.is_a?(MergeRequest)
-
- @referenced_merge_request_commit_shas ||= begin
- referenced_shas = references_per_parent.values.reduce(:|).to_a
- noteable.all_commit_shas.select do |sha|
- referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) }
- end
- end
- end
-
- # The default behaviour is `#to_i` - we just pass the hash through.
- def self.parse_symbol(sha_hash, _match)
- sha_hash
- end
-
- def url_for_object(commit, project)
- h = Gitlab::Routing.url_helpers
-
- if referenced_merge_request_commit_shas.include?(commit.id)
- h.diffs_project_merge_request_url(project,
- noteable,
- commit_id: commit.id,
- only_path: only_path?)
- else
- h.project_commit_url(project,
- commit,
- only_path: only_path?)
- end
- end
-
- def object_link_text_extras(object, matches)
- extras = super
-
- path = matches[:path] if matches.names.include?("path")
- if path == '/builds'
- extras.unshift "builds"
- end
-
- extras
- end
-
- private
-
- def parent_records(parent, ids)
- parent.commits_by(oids: ids.to_a)
- end
-
- def noteable
- context[:noteable]
- end
-
- def only_path?
- context[:only_path]
- end
- end
- end
-end
diff --git a/lib/banzai/filter/commit_trailers_filter.rb b/lib/banzai/filter/commit_trailers_filter.rb
index 5288db3b0cb..a615abc1989 100644
--- a/lib/banzai/filter/commit_trailers_filter.rb
+++ b/lib/banzai/filter/commit_trailers_filter.rb
@@ -36,7 +36,7 @@ module Banzai
next if html == content
- node.replace(html)
+ node.replace("\n\n#{html}")
end
doc
diff --git a/lib/banzai/filter/design_reference_filter.rb b/lib/banzai/filter/design_reference_filter.rb
deleted file mode 100644
index 1754fec93d4..00000000000
--- a/lib/banzai/filter/design_reference_filter.rb
+++ /dev/null
@@ -1,107 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- class DesignReferenceFilter < AbstractReferenceFilter
- class Identifier
- include Comparable
- attr_reader :issue_iid, :filename
-
- def initialize(issue_iid:, filename:)
- @issue_iid = issue_iid
- @filename = filename
- end
-
- def as_composite_id(id_for_iid)
- id = id_for_iid[issue_iid]
- return unless id
-
- { issue_id: id, filename: filename }
- end
-
- def <=>(other)
- return unless other.is_a?(Identifier)
-
- [issue_iid, filename] <=> [other.issue_iid, other.filename]
- end
- alias_method :eql?, :==
-
- def hash
- [issue_iid, filename].hash
- end
- end
-
- self.reference_type = :design
-
- def find_object(project, identifier)
- records_per_parent[project][identifier]
- end
-
- def parent_records(project, identifiers)
- return [] unless project.design_management_enabled?
-
- iids = identifiers.map(&:issue_iid).to_set
- issues = project.issues.where(iid: iids)
- id_for_iid = issues.index_by(&:iid).transform_values(&:id)
- issue_by_id = issues.index_by(&:id)
-
- designs(identifiers, id_for_iid).each do |d|
- issue = issue_by_id[d.issue_id]
- # optimisation: assign values we have already fetched
- d.project = project
- d.issue = issue
- end
- end
-
- def relation_for_paths(paths)
- super.includes(:route, :namespace, :group)
- end
-
- def parent_type
- :project
- end
-
- # optimisation to reuse the parent_per_reference query information
- def parent_from_ref(ref)
- parent_per_reference[ref || current_parent_path]
- end
-
- def url_for_object(design, project)
- path_options = { vueroute: design.filename }
- Gitlab::Routing.url_helpers.designs_project_issue_path(project, design.issue, path_options)
- end
-
- def data_attributes_for(_text, _project, design, **_kwargs)
- super.merge(issue: design.issue_id)
- end
-
- def self.object_class
- ::DesignManagement::Design
- end
-
- def self.object_sym
- :design
- end
-
- def self.parse_symbol(raw, match_data)
- filename = match_data[:url_filename]
- iid = match_data[:issue].to_i
- Identifier.new(filename: CGI.unescape(filename), issue_iid: iid)
- end
-
- def record_identifier(design)
- Identifier.new(filename: design.filename, issue_iid: design.issue.iid)
- end
-
- private
-
- def designs(identifiers, id_for_iid)
- identifiers
- .map { |identifier| identifier.as_composite_id(id_for_iid) }
- .compact
- .in_groups_of(100, false) # limitation of by_issue_id_and_filename, so we batch
- .flat_map { |ids| DesignManagement::Design.by_issue_id_and_filename(ids) }
- end
- end
- end
-end
diff --git a/lib/banzai/filter/epic_reference_filter.rb b/lib/banzai/filter/epic_reference_filter.rb
deleted file mode 100644
index 70a6cb0a6dc..00000000000
--- a/lib/banzai/filter/epic_reference_filter.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # The actual filter is implemented in the EE mixin
- class EpicReferenceFilter < IssuableReferenceFilter
- self.reference_type = :epic
-
- def self.object_class
- Epic
- end
-
- private
-
- def group
- context[:group] || context[:project]&.group
- end
- end
- end
-end
-
-Banzai::Filter::EpicReferenceFilter.prepend_if_ee('EE::Banzai::Filter::EpicReferenceFilter')
diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb
deleted file mode 100644
index fcf4863ab4f..00000000000
--- a/lib/banzai/filter/external_issue_reference_filter.rb
+++ /dev/null
@@ -1,118 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces external issue tracker references with links.
- # References are ignored if the project doesn't use an external issue
- # tracker.
- #
- # This filter does not support cross-project references.
- class ExternalIssueReferenceFilter < ReferenceFilter
- self.reference_type = :external_issue
-
- # Public: Find `JIRA-123` issue references in text
- #
- # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue|
- # "<a href=...>##{issue}</a>"
- # end
- #
- # text - String text to search.
- #
- # Yields the String match and the String issue reference.
- #
- # Returns a String replaced with the return of the block.
- def self.references_in(text, pattern)
- text.gsub(pattern) do |match|
- yield match, $~[:issue]
- end
- end
-
- def call
- # Early return if the project isn't using an external tracker
- return doc if project.nil? || default_issues_tracker?
-
- ref_pattern = issue_reference_pattern
- ref_start_pattern = /\A#{ref_pattern}\z/
-
- nodes.each_with_index do |node, index|
- if text_node?(node)
- replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
- issue_link_filter(content)
- end
-
- elsif element_node?(node)
- yield_valid_link(node) do |link, inner_html|
- if link =~ ref_start_pattern
- replace_link_node_with_href(node, index, link) do
- issue_link_filter(link, link_content: inner_html)
- end
- end
- end
- end
- end
-
- doc
- end
-
- private
-
- # Replace `JIRA-123` issue references in text with links to the referenced
- # issue's details page.
- #
- # text - String text to replace references in.
- # link_content - Original content of the link being replaced.
- #
- # Returns a String with `JIRA-123` references replaced with links. All
- # links have `gfm` and `gfm-issue` class names attached for styling.
- def issue_link_filter(text, link_content: nil)
- self.class.references_in(text, issue_reference_pattern) do |match, id|
- url = url_for_issue(id)
- klass = reference_class(:issue)
- data = data_attribute(project: project.id, external_issue: id)
- content = link_content || match
-
- %(<a href="#{url}" #{data}
- title="#{escape_once(issue_title)}"
- class="#{klass}">#{content}</a>)
- end
- end
-
- def url_for_issue(issue_id)
- return '' if project.nil?
-
- url = if only_path?
- project.external_issue_tracker.issue_path(issue_id)
- else
- project.external_issue_tracker.issue_url(issue_id)
- end
-
- # Ensure we return a valid URL to prevent possible XSS.
- URI.parse(url).to_s
- rescue URI::InvalidURIError
- ''
- end
-
- def default_issues_tracker?
- external_issues_cached(:default_issues_tracker?)
- end
-
- def issue_reference_pattern
- external_issues_cached(:external_issue_reference_pattern)
- end
-
- def project
- context[:project]
- end
-
- def issue_title
- "Issue in #{project.external_issue_tracker.title}"
- end
-
- def external_issues_cached(attribute)
- cached_attributes = Gitlab::SafeRequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} }
- cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend
- cached_attributes[project.id][attribute]
- end
- end
- end
-end
diff --git a/lib/banzai/filter/feature_flag_reference_filter.rb b/lib/banzai/filter/feature_flag_reference_filter.rb
deleted file mode 100644
index c11576901ce..00000000000
--- a/lib/banzai/filter/feature_flag_reference_filter.rb
+++ /dev/null
@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- class FeatureFlagReferenceFilter < IssuableReferenceFilter
- self.reference_type = :feature_flag
-
- def self.object_class
- Operations::FeatureFlag
- end
-
- def self.object_sym
- :feature_flag
- end
-
- def parent_records(parent, ids)
- parent.operations_feature_flags.where(iid: ids.to_a)
- end
-
- def url_for_object(feature_flag, project)
- ::Gitlab::Routing.url_helpers.edit_project_feature_flag_url(
- project,
- feature_flag.iid,
- only_path: context[:only_path]
- )
- end
-
- def object_link_title(object, matches)
- object.name
- end
- end
- end
-end
diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb
index 8a7d3c49ffb..6de9f2b86f6 100644
--- a/lib/banzai/filter/gollum_tags_filter.rb
+++ b/lib/banzai/filter/gollum_tags_filter.rb
@@ -98,14 +98,15 @@ module Banzai
return unless image?(content)
- if url?(content)
- path = content
- elsif file = wiki.find_file(content, load_content: false)
- path = ::File.join(wiki_base_path, file.path)
- end
+ path =
+ if url?(content)
+ content
+ elsif file = wiki.find_file(content, load_content: false)
+ file.path
+ end
if path
- content_tag(:img, nil, data: { src: path }, class: 'gfm')
+ content_tag(:img, nil, src: path, class: 'gfm')
end
end
diff --git a/lib/banzai/filter/issuable_reference_filter.rb b/lib/banzai/filter/issuable_reference_filter.rb
deleted file mode 100644
index b91ba9f7256..00000000000
--- a/lib/banzai/filter/issuable_reference_filter.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- class IssuableReferenceFilter < AbstractReferenceFilter
- def record_identifier(record)
- record.iid.to_i
- end
-
- def find_object(parent, iid)
- records_per_parent[parent][iid]
- end
-
- def parent_from_ref(ref)
- parent_per_reference[ref || current_parent_path]
- end
- end
- end
-end
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
deleted file mode 100644
index 216418ee5fa..00000000000
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces issue references with links. References to
- # issues that do not exist are ignored.
- #
- # This filter supports cross-project references.
- #
- # When external issues tracker like Jira is activated we should not
- # use issue reference pattern, but we should still be able
- # to reference issues from other GitLab projects.
- class IssueReferenceFilter < IssuableReferenceFilter
- self.reference_type = :issue
-
- def self.object_class
- Issue
- end
-
- def url_for_object(issue, project)
- return issue_path(issue, project) if only_path?
-
- issue_url(issue, project)
- end
-
- def parent_records(parent, ids)
- parent.issues.where(iid: ids.to_a)
- end
-
- def object_link_text_extras(issue, matches)
- super + design_link_extras(issue, matches.named_captures['path'])
- end
-
- private
-
- def issue_path(issue, project)
- Gitlab::Routing.url_helpers.namespace_project_issue_path(namespace_id: project.namespace, project_id: project, id: issue.iid)
- end
-
- def issue_url(issue, project)
- Gitlab::Routing.url_helpers.namespace_project_issue_url(namespace_id: project.namespace, project_id: project, id: issue.iid)
- end
-
- def design_link_extras(issue, path)
- if path == '/designs' && read_designs?(issue)
- ['designs']
- else
- []
- end
- end
-
- def read_designs?(issue)
- issue.project.design_management_enabled?
- end
- end
- end
-end
diff --git a/lib/banzai/filter/iteration_reference_filter.rb b/lib/banzai/filter/iteration_reference_filter.rb
deleted file mode 100644
index 9d2b533e6da..00000000000
--- a/lib/banzai/filter/iteration_reference_filter.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # The actual filter is implemented in the EE mixin
- class IterationReferenceFilter < AbstractReferenceFilter
- self.reference_type = :iteration
-
- def self.object_class
- Iteration
- end
- end
- end
-end
-
-Banzai::Filter::IterationReferenceFilter.prepend_if_ee('EE::Banzai::Filter::IterationReferenceFilter')
diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb
deleted file mode 100644
index a4d3e352051..00000000000
--- a/lib/banzai/filter/label_reference_filter.rb
+++ /dev/null
@@ -1,129 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces label references with links.
- class LabelReferenceFilter < AbstractReferenceFilter
- self.reference_type = :label
-
- def self.object_class
- Label
- end
-
- def find_object(parent_object, id)
- find_labels(parent_object).find(id)
- end
-
- def references_in(text, pattern = Label.reference_pattern)
- labels = {}
- unescaped_html = unescape_html_entities(text).gsub(pattern) do |match|
- namespace, project = $~[:namespace], $~[:project]
- project_path = full_project_path(namespace, project)
- label = find_label_cached(project_path, $~[:label_id], $~[:label_name])
-
- if label
- labels[label.id] = yield match, label.id, project, namespace, $~
- "#{REFERENCE_PLACEHOLDER}#{label.id}"
- else
- match
- end
- end
-
- return text if labels.empty?
-
- escape_with_placeholders(unescaped_html, labels)
- end
-
- def find_label_cached(parent_ref, label_id, label_name)
- cached_call(:banzai_find_label_cached, label_name&.tr('"', '') || label_id, path: [object_class, parent_ref]) do
- find_label(parent_ref, label_id, label_name)
- end
- end
-
- def find_label(parent_ref, label_id, label_name)
- parent = parent_from_ref(parent_ref)
- return unless parent
-
- label_params = label_params(label_id, label_name)
- find_labels(parent).find_by(label_params)
- end
-
- def find_labels(parent)
- params = if parent.is_a?(Group)
- { group_id: parent.id,
- include_ancestor_groups: true,
- only_group_labels: true }
- else
- { project: parent,
- include_ancestor_groups: true }
- end
-
- LabelsFinder.new(nil, params).execute(skip_authorization: true)
- end
-
- # Parameters to pass to `Label.find_by` based on the given arguments
- #
- # id - Integer ID to pass. If present, returns {id: id}
- # name - String name to pass. If `id` is absent, finds by name without
- # surrounding quotes.
- #
- # Returns a Hash.
- def label_params(id, name)
- if name
- { name: name.tr('"', '') }
- else
- { id: id.to_i }
- end
- end
-
- def url_for_object(label, parent)
- label_url_method =
- if context[:label_url_method]
- context[:label_url_method]
- elsif parent.is_a?(Project)
- :project_issues_url
- end
-
- return unless label_url_method
-
- Gitlab::Routing.url_helpers.public_send(label_url_method, parent, label_name: label.name, only_path: context[:only_path]) # rubocop:disable GitlabSecurity/PublicSend
- end
-
- def object_link_text(object, matches)
- label_suffix = ''
- parent = project || group
-
- if project || full_path_ref?(matches)
- project_path = full_project_path(matches[:namespace], matches[:project])
- parent_from_ref = from_ref_cached(project_path)
- reference = parent_from_ref.to_human_reference(parent)
-
- label_suffix = " <i>in #{ERB::Util.html_escape(reference)}</i>" if reference.present?
- end
-
- presenter = object.present(issuable_subject: parent)
- LabelsHelper.render_colored_label(presenter, suffix: label_suffix)
- end
-
- def wrap_link(link, label)
- presenter = label.present(issuable_subject: project || group)
- LabelsHelper.wrap_label_html(link, small: true, label: presenter)
- end
-
- def full_path_ref?(matches)
- matches[:namespace] && matches[:project]
- end
-
- def reference_class(type, tooltip: true)
- super + ' gl-link gl-label-link'
- end
-
- def object_link_title(object, matches)
- presenter = object.present(issuable_subject: project || group)
- LabelsHelper.label_tooltip_title(presenter)
- end
- end
- end
-end
-
-Banzai::Filter::LabelReferenceFilter.prepend_if_ee('EE::Banzai::Filter::LabelReferenceFilter')
diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb
index c915f0ee35b..2247984b86d 100644
--- a/lib/banzai/filter/math_filter.rb
+++ b/lib/banzai/filter/math_filter.rb
@@ -39,7 +39,7 @@ module Banzai
end
end
- doc.css('pre.code.math').each do |el|
+ doc.css('pre.code.language-math').each do |el|
el[STYLE_ATTRIBUTE] = 'display'
el[:class] += " #{TAG_CLASS}"
end
diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb
deleted file mode 100644
index 0b8bd17a71b..00000000000
--- a/lib/banzai/filter/merge_request_reference_filter.rb
+++ /dev/null
@@ -1,97 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces merge request references with links. References
- # to merge requests that do not exist are ignored.
- #
- # This filter supports cross-project references.
- class MergeRequestReferenceFilter < IssuableReferenceFilter
- self.reference_type = :merge_request
-
- def self.object_class
- MergeRequest
- end
-
- def url_for_object(mr, project)
- h = Gitlab::Routing.url_helpers
- h.project_merge_request_url(project, mr,
- only_path: context[:only_path])
- end
-
- def object_link_title(object, matches)
- # The method will return `nil` if object is not a commit
- # allowing for properly handling the extended MR Tooltip
- object_link_commit_title(object, matches)
- end
-
- def object_link_text_extras(object, matches)
- extras = super
-
- if commit_ref = object_link_commit_ref(object, matches)
- klass = reference_class(:commit, tooltip: false)
- commit_ref_tag = %(<span class="#{klass}">#{commit_ref}</span>)
-
- return extras.unshift(commit_ref_tag)
- end
-
- path = matches[:path] if matches.names.include?("path")
-
- case path
- when '/diffs'
- extras.unshift "diffs"
- when '/commits'
- extras.unshift "commits"
- when '/builds'
- extras.unshift "builds"
- end
-
- extras
- end
-
- def parent_records(parent, ids)
- parent.merge_requests
- .where(iid: ids.to_a)
- .includes(target_project: :namespace)
- end
-
- def reference_class(object_sym, options = {})
- super(object_sym, tooltip: false)
- end
-
- def data_attributes_for(text, parent, object, **data)
- super.merge(project_path: parent.full_path, iid: object.iid, mr_title: object.title)
- end
-
- private
-
- def object_link_commit_title(object, matches)
- object_link_commit(object, matches)&.title
- end
-
- def object_link_commit_ref(object, matches)
- object_link_commit(object, matches)&.short_id
- end
-
- def object_link_commit(object, matches)
- return unless matches.names.include?('query') && query = matches[:query]
-
- # Removes leading "?". CGI.parse expects "arg1&arg2&arg3"
- params = CGI.parse(query.sub(/^\?/, ''))
-
- return unless commit_sha = params['commit_id']&.first
-
- if commit = find_commit_by_sha(object, commit_sha)
- Commit.from_hash(commit.to_hash, object.project)
- end
- end
-
- def find_commit_by_sha(object, commit_sha)
- @all_commits ||= {}
- @all_commits[object.id] ||= object.all_commits
-
- @all_commits[object.id].find { |commit| commit.sha == commit_sha }
- end
- end
- end
-end
diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb
deleted file mode 100644
index 126208db935..00000000000
--- a/lib/banzai/filter/milestone_reference_filter.rb
+++ /dev/null
@@ -1,138 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces milestone references with links.
- class MilestoneReferenceFilter < AbstractReferenceFilter
- include Gitlab::Utils::StrongMemoize
-
- self.reference_type = :milestone
-
- def self.object_class
- Milestone
- end
-
- # Links to project milestones contain the IID, but when we're handling
- # 'regular' references, we need to use the global ID to disambiguate
- # between group and project milestones.
- def find_object(parent, id)
- return unless valid_context?(parent)
-
- find_milestone_with_finder(parent, id: id)
- end
-
- def find_object_from_link(parent, iid)
- return unless valid_context?(parent)
-
- find_milestone_with_finder(parent, iid: iid)
- end
-
- def valid_context?(parent)
- strong_memoize(:valid_context) do
- group_context?(parent) || project_context?(parent)
- end
- end
-
- def group_context?(parent)
- strong_memoize(:group_context) do
- parent.is_a?(Group)
- end
- end
-
- def project_context?(parent)
- strong_memoize(:project_context) do
- parent.is_a?(Project)
- end
- end
-
- def references_in(text, pattern = Milestone.reference_pattern)
- # We'll handle here the references that follow the `reference_pattern`.
- # Other patterns (for example, the link pattern) are handled by the
- # default implementation.
- return super(text, pattern) if pattern != Milestone.reference_pattern
-
- milestones = {}
- unescaped_html = unescape_html_entities(text).gsub(pattern) do |match|
- milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name])
-
- if milestone
- milestones[milestone.id] = yield match, milestone.id, $~[:project], $~[:namespace], $~
- "#{REFERENCE_PLACEHOLDER}#{milestone.id}"
- else
- match
- end
- end
-
- return text if milestones.empty?
-
- escape_with_placeholders(unescaped_html, milestones)
- end
-
- def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name)
- project_path = full_project_path(namespace_ref, project_ref)
-
- # Returns group if project is not found by path
- parent = parent_from_ref(project_path)
-
- return unless parent
-
- milestone_params = milestone_params(milestone_id, milestone_name)
-
- find_milestone_with_finder(parent, milestone_params)
- end
-
- def milestone_params(iid, name)
- if name
- { name: name.tr('"', '') }
- else
- { iid: iid.to_i }
- end
- end
-
- def find_milestone_with_finder(parent, params)
- finder_params = milestone_finder_params(parent, params[:iid].present?)
-
- MilestonesFinder.new(finder_params).find_by(params)
- end
-
- def milestone_finder_params(parent, find_by_iid)
- { order: nil, state: 'all' }.tap do |params|
- params[:project_ids] = parent.id if project_context?(parent)
-
- # We don't support IID lookups because IIDs can clash between
- # group/project milestones and group/subgroup milestones.
- params[:group_ids] = self_and_ancestors_ids(parent) unless find_by_iid
- end
- end
-
- def self_and_ancestors_ids(parent)
- if group_context?(parent)
- parent.self_and_ancestors.select(:id)
- elsif project_context?(parent)
- parent.group&.self_and_ancestors&.select(:id)
- end
- end
-
- def url_for_object(milestone, project)
- Gitlab::Routing
- .url_helpers
- .milestone_url(milestone, only_path: context[:only_path])
- end
-
- def object_link_text(object, matches)
- milestone_link = escape_once(super)
- reference = object.project&.to_reference_base(project)
-
- if reference.present?
- "#{milestone_link} <i>in #{reference}</i>".html_safe
- else
- milestone_link
- end
- end
-
- def object_link_title(object, matches)
- nil
- end
- end
- end
-end
diff --git a/lib/banzai/filter/project_reference_filter.rb b/lib/banzai/filter/project_reference_filter.rb
deleted file mode 100644
index 50e23460cb8..00000000000
--- a/lib/banzai/filter/project_reference_filter.rb
+++ /dev/null
@@ -1,117 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces project references with links.
- class ProjectReferenceFilter < ReferenceFilter
- self.reference_type = :project
-
- # Public: Find `namespace/project>` project references in text
- #
- # ProjectReferenceFilter.references_in(text) do |match, project|
- # "<a href=...>#{project}></a>"
- # end
- #
- # text - String text to search.
- #
- # Yields the String match, and the String project name.
- #
- # Returns a String replaced with the return of the block.
- def self.references_in(text)
- text.gsub(Project.markdown_reference_pattern) do |match|
- yield match, "#{$~[:namespace]}/#{$~[:project]}"
- end
- end
-
- def call
- ref_pattern = Project.markdown_reference_pattern
- ref_pattern_start = /\A#{ref_pattern}\z/
-
- nodes.each_with_index do |node, index|
- if text_node?(node)
- replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
- project_link_filter(content)
- end
- elsif element_node?(node)
- yield_valid_link(node) do |link, inner_html|
- if link =~ ref_pattern_start
- replace_link_node_with_href(node, index, link) do
- project_link_filter(link, link_content: inner_html)
- end
- end
- end
- end
- end
-
- doc
- end
-
- # Replace `namespace/project>` project references in text with links to the referenced
- # project page.
- #
- # text - String text to replace references in.
- # link_content - Original content of the link being replaced.
- #
- # Returns a String with `namespace/project>` references replaced with links. All links
- # have `gfm` and `gfm-project` class names attached for styling.
- def project_link_filter(text, link_content: nil)
- self.class.references_in(text) do |match, project_path|
- cached_call(:banzai_url_for_object, match, path: [Project, project_path.downcase]) do
- if project = projects_hash[project_path.downcase]
- link_to_project(project, link_content: link_content) || match
- else
- match
- end
- end
- end
- end
-
- # Returns a Hash containing all Project objects for the project
- # references in the current document.
- #
- # The keys of this Hash are the project paths, the values the
- # corresponding Project objects.
- def projects_hash
- @projects ||= Project.eager_load(:route, namespace: [:route])
- .where_full_path_in(projects)
- .index_by(&:full_path)
- .transform_keys(&:downcase)
- end
-
- # Returns all projects referenced in the current document.
- def projects
- refs = Set.new
-
- nodes.each do |node|
- node.to_html.scan(Project.markdown_reference_pattern) do
- refs << "#{$~[:namespace]}/#{$~[:project]}"
- end
- end
-
- refs.to_a
- end
-
- private
-
- def urls
- Gitlab::Routing.url_helpers
- end
-
- def link_class
- reference_class(:project)
- end
-
- def link_to_project(project, link_content: nil)
- url = urls.project_url(project, only_path: context[:only_path])
- data = data_attribute(project: project.id)
- content = link_content || project.to_reference
-
- link_tag(url, data, content, project.name)
- end
-
- def link_tag(url, data, link_content, title)
- %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>)
- end
- end
- end
-end
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
deleted file mode 100644
index d22a0e0b504..00000000000
--- a/lib/banzai/filter/reference_filter.rb
+++ /dev/null
@@ -1,215 +0,0 @@
-# frozen_string_literal: true
-
-# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/reference.js
-module Banzai
- module Filter
- # Base class for GitLab Flavored Markdown reference filters.
- #
- # References within <pre>, <code>, <a>, and <style> elements are ignored.
- #
- # Context options:
- # :project (required) - Current project, ignored if reference is cross-project.
- # :only_path - Generate path-only links.
- class ReferenceFilter < HTML::Pipeline::Filter
- include RequestStoreReferenceCache
- include OutputSafety
-
- class << self
- attr_accessor :reference_type
-
- def call(doc, context = nil, result = nil)
- new(doc, context, result).call_and_update_nodes
- end
- end
-
- def initialize(doc, context = nil, result = nil)
- super
-
- @new_nodes = {}
- @nodes = self.result[:reference_filter_nodes]
- end
-
- def call_and_update_nodes
- with_update_nodes { call }
- end
-
- # Returns a data attribute String to attach to a reference link
- #
- # attributes - Hash, where the key becomes the data attribute name and the
- # value is the data attribute value
- #
- # Examples:
- #
- # data_attribute(project: 1, issue: 2)
- # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\""
- #
- # data_attribute(project: 3, merge_request: 4)
- # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\""
- #
- # Returns a String
- def data_attribute(attributes = {})
- attributes = attributes.reject { |_, v| v.nil? }
-
- attributes[:reference_type] ||= self.class.reference_type
- attributes[:container] ||= 'body'
- attributes[:placement] ||= 'top'
- attributes.delete(:original) if context[:no_original_data]
- attributes.map do |key, value|
- %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}")
- end.join(' ')
- end
-
- def ignore_ancestor_query
- @ignore_ancestor_query ||= begin
- parents = %w(pre code a style)
- parents << 'blockquote' if context[:ignore_blockquotes]
-
- parents.map { |n| "ancestor::#{n}" }.join(' or ')
- end
- end
-
- def project
- context[:project]
- end
-
- def group
- context[:group]
- end
-
- def user
- context[:user]
- end
-
- def skip_project_check?
- context[:skip_project_check]
- end
-
- def reference_class(type, tooltip: true)
- gfm_klass = "gfm gfm-#{type}"
-
- return gfm_klass unless tooltip
-
- "#{gfm_klass} has-tooltip"
- end
-
- # Ensure that a :project key exists in context
- #
- # Note that while the key might exist, its value could be nil!
- def validate
- needs :project unless skip_project_check?
- end
-
- # Iterates over all <a> and text() nodes in a document.
- #
- # Nodes are skipped whenever their ancestor is one of the nodes returned
- # by `ignore_ancestor_query`. Link tags are not processed if they have a
- # "gfm" class or the "href" attribute is empty.
- def each_node
- return to_enum(__method__) unless block_given?
-
- doc.xpath(query).each do |node|
- yield node
- end
- end
-
- # Returns an Array containing all HTML nodes.
- def nodes
- @nodes ||= each_node.to_a
- end
-
- # Yields the link's URL and inner HTML whenever the node is a valid <a> tag.
- def yield_valid_link(node)
- link = unescape_link(node.attr('href').to_s)
- inner_html = node.inner_html
-
- return unless link.force_encoding('UTF-8').valid_encoding?
-
- yield link, inner_html
- end
-
- def unescape_link(href)
- CGI.unescape(href)
- end
-
- def replace_text_when_pattern_matches(node, index, pattern)
- return unless node.text =~ pattern
-
- content = node.to_html
- html = yield content
-
- replace_text_with_html(node, index, html) unless html == content
- end
-
- def replace_link_node_with_text(node, index)
- html = yield
-
- replace_text_with_html(node, index, html) unless html == node.text
- end
-
- def replace_link_node_with_href(node, index, link)
- html = yield
-
- replace_text_with_html(node, index, html) unless html == link
- end
-
- def text_node?(node)
- node.is_a?(Nokogiri::XML::Text)
- end
-
- def element_node?(node)
- node.is_a?(Nokogiri::XML::Element)
- end
-
- private
-
- def query
- @query ||= %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})]
- | descendant-or-self::a[
- not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "")
- ]}
- end
-
- def replace_text_with_html(node, index, html)
- replace_and_update_new_nodes(node, index, html)
- end
-
- def replace_and_update_new_nodes(node, index, html)
- previous_node = node.previous
- next_node = node.next
- parent_node = node.parent
- # Unfortunately node.replace(html) returns re-parented nodes, not the actual replaced nodes in the doc
- # We need to find the actual nodes in the doc that were replaced
- node.replace(html)
- @new_nodes[index] = []
-
- # We replaced node with new nodes, so we find first new node. If previous_node is nil, we take first parent child
- new_node = previous_node ? previous_node.next : parent_node&.children&.first
-
- # We iterate from first to last replaced node and store replaced nodes in @new_nodes
- while new_node && new_node != next_node
- @new_nodes[index] << new_node.xpath(query)
- new_node = new_node.next
- end
-
- @new_nodes[index].flatten!
- end
-
- def only_path?
- context[:only_path]
- end
-
- def with_update_nodes
- @new_nodes = {}
- yield.tap { update_nodes! }
- end
-
- # Once Filter completes replacing nodes, we update nodes with @new_nodes
- def update_nodes!
- @new_nodes.sort_by { |index, _new_nodes| -index }.each do |index, new_nodes|
- nodes[index, 1] = new_nodes
- end
- result[:reference_filter_nodes] = nodes
- end
- end
- end
-end
diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb
new file mode 100644
index 00000000000..7109373dbce
--- /dev/null
+++ b/lib/banzai/filter/references/abstract_reference_filter.rb
@@ -0,0 +1,448 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # Issues, merge requests, Snippets, Commits and Commit Ranges share
+ # similar functionality in reference filtering.
+ class AbstractReferenceFilter < ReferenceFilter
+ include CrossProjectReference
+
+ # REFERENCE_PLACEHOLDER is used for re-escaping HTML text except found
+ # reference (which we replace with placeholder during re-scaping). The
+ # random number helps ensure it's pretty close to unique. Since it's a
+ # transitory value (it never gets saved) we can initialize once, and it
+ # doesn't matter if it changes on a restart.
+ REFERENCE_PLACEHOLDER = "_reference_#{SecureRandom.hex(16)}_"
+ REFERENCE_PLACEHOLDER_PATTERN = %r{#{REFERENCE_PLACEHOLDER}(\d+)}.freeze
+
+ def self.object_class
+ # Implement in child class
+ # Example: MergeRequest
+ end
+
+ def self.object_name
+ @object_name ||= object_class.name.underscore
+ end
+
+ def self.object_sym
+ @object_sym ||= object_name.to_sym
+ end
+
+ # Public: Find references in text (like `!123` for merge requests)
+ #
+ # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches|
+ # object = find_object(project_ref, id)
+ # "<a href=...>#{object.to_reference}</a>"
+ # end
+ #
+ # text - String text to search.
+ #
+ # Yields the String match, the Integer referenced object ID, an optional String
+ # of the external project reference, and all of the matchdata.
+ #
+ # Returns a String replaced with the return of the block.
+ def self.references_in(text, pattern = object_class.reference_pattern)
+ text.gsub(pattern) do |match|
+ if ident = identifier($~)
+ yield match, ident, $~[:project], $~[:namespace], $~
+ else
+ match
+ end
+ end
+ end
+
+ def self.identifier(match_data)
+ symbol = symbol_from_match(match_data)
+
+ parse_symbol(symbol, match_data) if object_class.reference_valid?(symbol)
+ end
+
+ def identifier(match_data)
+ self.class.identifier(match_data)
+ end
+
+ def self.symbol_from_match(match)
+ key = object_sym
+ match[key] if match.names.include?(key.to_s)
+ end
+
+ # Transform a symbol extracted from the text to a meaningful value
+ # In most cases these will be integers, so we call #to_i by default
+ #
+ # This method has the contract that if a string `ref` refers to a
+ # record `record`, then `parse_symbol(ref) == record_identifier(record)`.
+ def self.parse_symbol(symbol, match_data)
+ symbol.to_i
+ end
+
+ # We assume that most classes are identifying records by ID.
+ #
+ # This method has the contract that if a string `ref` refers to a
+ # record `record`, then `class.parse_symbol(ref) == record_identifier(record)`.
+ def record_identifier(record)
+ record.id
+ end
+
+ def object_class
+ self.class.object_class
+ end
+
+ def object_sym
+ self.class.object_sym
+ end
+
+ def references_in(*args, &block)
+ self.class.references_in(*args, &block)
+ end
+
+ # Implement in child class
+ # Example: project.merge_requests.find
+ def find_object(parent_object, id)
+ end
+
+ # Override if the link reference pattern produces a different ID (global
+ # ID vs internal ID, for instance) to the regular reference pattern.
+ def find_object_from_link(parent_object, id)
+ find_object(parent_object, id)
+ end
+
+ # Implement in child class
+ # Example: project_merge_request_url
+ def url_for_object(object, parent_object)
+ end
+
+ def find_object_cached(parent_object, id)
+ cached_call(:banzai_find_object, id, path: [object_class, parent_object.id]) do
+ find_object(parent_object, id)
+ end
+ end
+
+ def find_object_from_link_cached(parent_object, id)
+ cached_call(:banzai_find_object_from_link, id, path: [object_class, parent_object.id]) do
+ find_object_from_link(parent_object, id)
+ end
+ end
+
+ def from_ref_cached(ref)
+ cached_call("banzai_#{parent_type}_refs".to_sym, ref) do
+ parent_from_ref(ref)
+ end
+ end
+
+ def url_for_object_cached(object, parent_object)
+ cached_call(:banzai_url_for_object, object, path: [object_class, parent_object.id]) do
+ url_for_object(object, parent_object)
+ end
+ end
+
+ def call
+ return doc unless project || group || user
+
+ ref_pattern = object_class.reference_pattern
+ link_pattern = object_class.link_reference_pattern
+
+ # Compile often used regexps only once outside of the loop
+ ref_pattern_anchor = /\A#{ref_pattern}\z/
+ link_pattern_start = /\A#{link_pattern}/
+ link_pattern_anchor = /\A#{link_pattern}\z/
+
+ nodes.each_with_index do |node, index|
+ if text_node?(node) && ref_pattern
+ replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
+ object_link_filter(content, ref_pattern)
+ end
+
+ elsif element_node?(node)
+ yield_valid_link(node) do |link, inner_html|
+ if ref_pattern && link =~ ref_pattern_anchor
+ replace_link_node_with_href(node, index, link) do
+ object_link_filter(link, ref_pattern, link_content: inner_html)
+ end
+
+ next
+ end
+
+ next unless link_pattern
+
+ if link == inner_html && inner_html =~ link_pattern_start
+ replace_link_node_with_text(node, index) do
+ object_link_filter(inner_html, link_pattern, link_reference: true)
+ end
+
+ next
+ end
+
+ if link =~ link_pattern_anchor
+ replace_link_node_with_href(node, index, link) do
+ object_link_filter(link, link_pattern, link_content: inner_html, link_reference: true)
+ end
+
+ next
+ end
+ end
+ end
+ end
+
+ doc
+ end
+
+ # Replace references (like `!123` for merge requests) in text with links
+ # to the referenced object's details page.
+ #
+ # text - String text to replace references in.
+ # pattern - Reference pattern to match against.
+ # link_content - Original content of the link being replaced.
+ # link_reference - True if this was using the link reference pattern,
+ # false otherwise.
+ #
+ # Returns a String with references replaced with links. All links
+ # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling.
+ def object_link_filter(text, pattern, link_content: nil, link_reference: false)
+ references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches|
+ parent_path = if parent_type == :group
+ full_group_path(namespace_ref)
+ else
+ full_project_path(namespace_ref, project_ref)
+ end
+
+ parent = from_ref_cached(parent_path)
+
+ if parent
+ object =
+ if link_reference
+ find_object_from_link_cached(parent, id)
+ else
+ find_object_cached(parent, id)
+ end
+ end
+
+ if object
+ title = object_link_title(object, matches)
+ klass = reference_class(object_sym)
+
+ data_attributes = data_attributes_for(link_content || match, parent, object,
+ link_content: !!link_content,
+ link_reference: link_reference)
+ data = data_attribute(data_attributes)
+
+ url =
+ if matches.names.include?("url") && matches[:url]
+ matches[:url]
+ else
+ url_for_object_cached(object, parent)
+ end
+
+ content = link_content || object_link_text(object, matches)
+
+ link = %(<a href="#{url}" #{data}
+ title="#{escape_once(title)}"
+ class="#{klass}">#{content}</a>)
+
+ wrap_link(link, object)
+ else
+ match
+ end
+ end
+ end
+
+ def wrap_link(link, object)
+ link
+ end
+
+ def data_attributes_for(text, parent, object, link_content: false, link_reference: false)
+ object_parent_type = parent.is_a?(Group) ? :group : :project
+
+ {
+ original: escape_html_entities(text),
+ link: link_content,
+ link_reference: link_reference,
+ object_parent_type => parent.id,
+ object_sym => object.id
+ }
+ end
+
+ def object_link_text_extras(object, matches)
+ extras = []
+
+ if matches.names.include?("anchor") && matches[:anchor] && matches[:anchor] =~ /\A\#note_(\d+)\z/
+ extras << "comment #{Regexp.last_match(1)}"
+ end
+
+ extension = matches[:extension] if matches.names.include?("extension")
+
+ extras << extension if extension
+
+ extras
+ end
+
+ def object_link_title(object, matches)
+ object.title
+ end
+
+ def object_link_text(object, matches)
+ parent = project || group || user
+ text = object.reference_link_text(parent)
+
+ extras = object_link_text_extras(object, matches)
+ text += " (#{extras.join(", ")})" if extras.any?
+
+ text
+ end
+
+ # Returns a Hash containing all object references (e.g. issue IDs) per the
+ # project they belong to.
+ def references_per_parent
+ @references_per ||= {}
+
+ @references_per[parent_type] ||= begin
+ refs = Hash.new { |hash, key| hash[key] = Set.new }
+ regex = [
+ object_class.link_reference_pattern,
+ object_class.reference_pattern
+ ].compact.reduce { |a, b| Regexp.union(a, b) }
+
+ nodes.each do |node|
+ node.to_html.scan(regex) do
+ path = if parent_type == :project
+ full_project_path($~[:namespace], $~[:project])
+ else
+ full_group_path($~[:group])
+ end
+
+ if ident = identifier($~)
+ refs[path] << ident
+ end
+ end
+ end
+
+ refs
+ end
+ end
+
+ # Returns a Hash containing referenced projects grouped per their full
+ # path.
+ def parent_per_reference
+ @per_reference ||= {}
+
+ @per_reference[parent_type] ||= begin
+ refs = Set.new
+
+ references_per_parent.each do |ref, _|
+ refs << ref
+ end
+
+ find_for_paths(refs.to_a).index_by(&:full_path)
+ end
+ end
+
+ def relation_for_paths(paths)
+ klass = parent_type.to_s.camelize.constantize
+ result = klass.where_full_path_in(paths)
+ return result if parent_type == :group
+
+ result.includes(:namespace) if parent_type == :project
+ end
+
+ # Returns projects for the given paths.
+ def find_for_paths(paths)
+ if Gitlab::SafeRequestStore.active?
+ cache = refs_cache
+ to_query = paths - cache.keys
+
+ unless to_query.empty?
+ records = relation_for_paths(to_query)
+
+ found = []
+ records.each do |record|
+ ref = record.full_path
+ get_or_set_cache(cache, ref) { record }
+ found << ref
+ end
+
+ not_found = to_query - found
+ not_found.each do |ref|
+ get_or_set_cache(cache, ref) { nil }
+ end
+ end
+
+ cache.slice(*paths).values.compact
+ else
+ relation_for_paths(paths)
+ end
+ end
+
+ def current_parent_path
+ @current_parent_path ||= parent&.full_path
+ end
+
+ def current_project_namespace_path
+ @current_project_namespace_path ||= project&.namespace&.full_path
+ end
+
+ def records_per_parent
+ @_records_per_project ||= {}
+
+ @_records_per_project[object_class.to_s.underscore] ||= begin
+ hash = Hash.new { |h, k| h[k] = {} }
+
+ parent_per_reference.each do |path, parent|
+ record_ids = references_per_parent[path]
+
+ parent_records(parent, record_ids).each do |record|
+ hash[parent][record_identifier(record)] = record
+ end
+ end
+
+ hash
+ end
+ end
+
+ private
+
+ def full_project_path(namespace, project_ref)
+ return current_parent_path unless project_ref
+
+ namespace_ref = namespace || current_project_namespace_path
+ "#{namespace_ref}/#{project_ref}"
+ end
+
+ def refs_cache
+ Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {}
+ end
+
+ def parent_type
+ :project
+ end
+
+ def parent
+ parent_type == :project ? project : group
+ end
+
+ def full_group_path(group_ref)
+ return current_parent_path unless group_ref
+
+ group_ref
+ end
+
+ def unescape_html_entities(text)
+ CGI.unescapeHTML(text.to_s)
+ end
+
+ def escape_html_entities(text)
+ CGI.escapeHTML(text.to_s)
+ end
+
+ def escape_with_placeholders(text, placeholder_data)
+ escaped = escape_html_entities(text)
+
+ escaped.gsub(REFERENCE_PLACEHOLDER_PATTERN) do |match|
+ placeholder_data[Regexp.last_match(1).to_i]
+ end
+ end
+ end
+ end
+ end
+end
+
+Banzai::Filter::References::AbstractReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::AbstractReferenceFilter')
diff --git a/lib/banzai/filter/references/alert_reference_filter.rb b/lib/banzai/filter/references/alert_reference_filter.rb
new file mode 100644
index 00000000000..90fef536605
--- /dev/null
+++ b/lib/banzai/filter/references/alert_reference_filter.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ class AlertReferenceFilter < IssuableReferenceFilter
+ self.reference_type = :alert
+
+ def self.object_class
+ AlertManagement::Alert
+ end
+
+ def self.object_sym
+ :alert
+ end
+
+ def parent_records(parent, ids)
+ parent.alert_management_alerts.where(iid: ids.to_a)
+ end
+
+ def url_for_object(alert, project)
+ ::Gitlab::Routing.url_helpers.details_project_alert_management_url(
+ project,
+ alert.iid,
+ only_path: context[:only_path]
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/commit_range_reference_filter.rb b/lib/banzai/filter/references/commit_range_reference_filter.rb
new file mode 100644
index 00000000000..ad79f8a173c
--- /dev/null
+++ b/lib/banzai/filter/references/commit_range_reference_filter.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces commit range references with links.
+ #
+ # This filter supports cross-project references.
+ class CommitRangeReferenceFilter < AbstractReferenceFilter
+ self.reference_type = :commit_range
+
+ def self.object_class
+ CommitRange
+ end
+
+ def self.references_in(text, pattern = CommitRange.reference_pattern)
+ text.gsub(pattern) do |match|
+ yield match, $~[:commit_range], $~[:project], $~[:namespace], $~
+ end
+ end
+
+ def initialize(*args)
+ super
+
+ @commit_map = {}
+ end
+
+ def find_object(project, id)
+ return unless project.is_a?(Project)
+
+ range = CommitRange.new(id, project)
+
+ range.valid_commits? ? range : nil
+ end
+
+ def url_for_object(range, project)
+ h = Gitlab::Routing.url_helpers
+ h.project_compare_url(project,
+ range.to_param.merge(only_path: context[:only_path]))
+ end
+
+ def object_link_title(range, matches)
+ nil
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/commit_reference_filter.rb b/lib/banzai/filter/references/commit_reference_filter.rb
new file mode 100644
index 00000000000..457921bd07d
--- /dev/null
+++ b/lib/banzai/filter/references/commit_reference_filter.rb
@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces commit references with links.
+ #
+ # This filter supports cross-project references.
+ class CommitReferenceFilter < AbstractReferenceFilter
+ self.reference_type = :commit
+
+ def self.object_class
+ Commit
+ end
+
+ def self.references_in(text, pattern = Commit.reference_pattern)
+ text.gsub(pattern) do |match|
+ yield match, $~[:commit], $~[:project], $~[:namespace], $~
+ end
+ end
+
+ def find_object(project, id)
+ return unless project.is_a?(Project) && project.valid_repo?
+
+ _, record = records_per_parent[project].detect { |k, _v| Gitlab::Git.shas_eql?(k, id) }
+
+ record
+ end
+
+ def referenced_merge_request_commit_shas
+ return [] unless noteable.is_a?(MergeRequest)
+
+ @referenced_merge_request_commit_shas ||= begin
+ referenced_shas = references_per_parent.values.reduce(:|).to_a
+ noteable.all_commit_shas.select do |sha|
+ referenced_shas.any? { |ref| Gitlab::Git.shas_eql?(sha, ref) }
+ end
+ end
+ end
+
+ # The default behaviour is `#to_i` - we just pass the hash through.
+ def self.parse_symbol(sha_hash, _match)
+ sha_hash
+ end
+
+ def url_for_object(commit, project)
+ h = Gitlab::Routing.url_helpers
+
+ if referenced_merge_request_commit_shas.include?(commit.id)
+ h.diffs_project_merge_request_url(project,
+ noteable,
+ commit_id: commit.id,
+ only_path: only_path?)
+ else
+ h.project_commit_url(project,
+ commit,
+ only_path: only_path?)
+ end
+ end
+
+ def object_link_text_extras(object, matches)
+ extras = super
+
+ path = matches[:path] if matches.names.include?("path")
+ if path == '/builds'
+ extras.unshift "builds"
+ end
+
+ extras
+ end
+
+ private
+
+ def parent_records(parent, ids)
+ parent.commits_by(oids: ids.to_a)
+ end
+
+ def noteable
+ context[:noteable]
+ end
+
+ def only_path?
+ context[:only_path]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/design_reference_filter.rb b/lib/banzai/filter/references/design_reference_filter.rb
new file mode 100644
index 00000000000..61234e61c15
--- /dev/null
+++ b/lib/banzai/filter/references/design_reference_filter.rb
@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ class DesignReferenceFilter < AbstractReferenceFilter
+ class Identifier
+ include Comparable
+ attr_reader :issue_iid, :filename
+
+ def initialize(issue_iid:, filename:)
+ @issue_iid = issue_iid
+ @filename = filename
+ end
+
+ def as_composite_id(id_for_iid)
+ id = id_for_iid[issue_iid]
+ return unless id
+
+ { issue_id: id, filename: filename }
+ end
+
+ def <=>(other)
+ return unless other.is_a?(Identifier)
+
+ [issue_iid, filename] <=> [other.issue_iid, other.filename]
+ end
+ alias_method :eql?, :==
+
+ def hash
+ [issue_iid, filename].hash
+ end
+ end
+
+ self.reference_type = :design
+
+ def find_object(project, identifier)
+ records_per_parent[project][identifier]
+ end
+
+ def parent_records(project, identifiers)
+ return [] unless project.design_management_enabled?
+
+ iids = identifiers.map(&:issue_iid).to_set
+ issues = project.issues.where(iid: iids)
+ id_for_iid = issues.index_by(&:iid).transform_values(&:id)
+ issue_by_id = issues.index_by(&:id)
+
+ designs(identifiers, id_for_iid).each do |d|
+ issue = issue_by_id[d.issue_id]
+ # optimisation: assign values we have already fetched
+ d.project = project
+ d.issue = issue
+ end
+ end
+
+ def relation_for_paths(paths)
+ super.includes(:route, :namespace, :group)
+ end
+
+ def parent_type
+ :project
+ end
+
+ # optimisation to reuse the parent_per_reference query information
+ def parent_from_ref(ref)
+ parent_per_reference[ref || current_parent_path]
+ end
+
+ def url_for_object(design, project)
+ path_options = { vueroute: design.filename }
+ Gitlab::Routing.url_helpers.designs_project_issue_path(project, design.issue, path_options)
+ end
+
+ def data_attributes_for(_text, _project, design, **_kwargs)
+ super.merge(issue: design.issue_id)
+ end
+
+ def self.object_class
+ ::DesignManagement::Design
+ end
+
+ def self.object_sym
+ :design
+ end
+
+ def self.parse_symbol(raw, match_data)
+ filename = match_data[:url_filename]
+ iid = match_data[:issue].to_i
+ Identifier.new(filename: CGI.unescape(filename), issue_iid: iid)
+ end
+
+ def record_identifier(design)
+ Identifier.new(filename: design.filename, issue_iid: design.issue.iid)
+ end
+
+ private
+
+ def designs(identifiers, id_for_iid)
+ identifiers
+ .map { |identifier| identifier.as_composite_id(id_for_iid) }
+ .compact
+ .in_groups_of(100, false) # limitation of by_issue_id_and_filename, so we batch
+ .flat_map { |ids| DesignManagement::Design.by_issue_id_and_filename(ids) }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/epic_reference_filter.rb b/lib/banzai/filter/references/epic_reference_filter.rb
new file mode 100644
index 00000000000..4ee446e5317
--- /dev/null
+++ b/lib/banzai/filter/references/epic_reference_filter.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # The actual filter is implemented in the EE mixin
+ class EpicReferenceFilter < IssuableReferenceFilter
+ self.reference_type = :epic
+
+ def self.object_class
+ Epic
+ end
+
+ private
+
+ def group
+ context[:group] || context[:project]&.group
+ end
+ end
+ end
+ end
+end
+
+Banzai::Filter::References::EpicReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::EpicReferenceFilter')
diff --git a/lib/banzai/filter/references/external_issue_reference_filter.rb b/lib/banzai/filter/references/external_issue_reference_filter.rb
new file mode 100644
index 00000000000..247e20967df
--- /dev/null
+++ b/lib/banzai/filter/references/external_issue_reference_filter.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces external issue tracker references with links.
+ # References are ignored if the project doesn't use an external issue
+ # tracker.
+ #
+ # This filter does not support cross-project references.
+ class ExternalIssueReferenceFilter < ReferenceFilter
+ self.reference_type = :external_issue
+
+ # Public: Find `JIRA-123` issue references in text
+ #
+ # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue|
+ # "<a href=...>##{issue}</a>"
+ # end
+ #
+ # text - String text to search.
+ #
+ # Yields the String match and the String issue reference.
+ #
+ # Returns a String replaced with the return of the block.
+ def self.references_in(text, pattern)
+ text.gsub(pattern) do |match|
+ yield match, $~[:issue]
+ end
+ end
+
+ def call
+ # Early return if the project isn't using an external tracker
+ return doc if project.nil? || default_issues_tracker?
+
+ ref_pattern = issue_reference_pattern
+ ref_start_pattern = /\A#{ref_pattern}\z/
+
+ nodes.each_with_index do |node, index|
+ if text_node?(node)
+ replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
+ issue_link_filter(content)
+ end
+
+ elsif element_node?(node)
+ yield_valid_link(node) do |link, inner_html|
+ if link =~ ref_start_pattern
+ replace_link_node_with_href(node, index, link) do
+ issue_link_filter(link, link_content: inner_html)
+ end
+ end
+ end
+ end
+ end
+
+ doc
+ end
+
+ private
+
+ # Replace `JIRA-123` issue references in text with links to the referenced
+ # issue's details page.
+ #
+ # text - String text to replace references in.
+ # link_content - Original content of the link being replaced.
+ #
+ # Returns a String with `JIRA-123` references replaced with links. All
+ # links have `gfm` and `gfm-issue` class names attached for styling.
+ def issue_link_filter(text, link_content: nil)
+ self.class.references_in(text, issue_reference_pattern) do |match, id|
+ url = url_for_issue(id)
+ klass = reference_class(:issue)
+ data = data_attribute(project: project.id, external_issue: id)
+ content = link_content || match
+
+ %(<a href="#{url}" #{data}
+ title="#{escape_once(issue_title)}"
+ class="#{klass}">#{content}</a>)
+ end
+ end
+
+ def url_for_issue(issue_id)
+ return '' if project.nil?
+
+ url = if only_path?
+ project.external_issue_tracker.issue_path(issue_id)
+ else
+ project.external_issue_tracker.issue_url(issue_id)
+ end
+
+ # Ensure we return a valid URL to prevent possible XSS.
+ URI.parse(url).to_s
+ rescue URI::InvalidURIError
+ ''
+ end
+
+ def default_issues_tracker?
+ external_issues_cached(:default_issues_tracker?)
+ end
+
+ def issue_reference_pattern
+ external_issues_cached(:external_issue_reference_pattern)
+ end
+
+ def project
+ context[:project]
+ end
+
+ def issue_title
+ "Issue in #{project.external_issue_tracker.title}"
+ end
+
+ def external_issues_cached(attribute)
+ cached_attributes = Gitlab::SafeRequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} }
+ cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? # rubocop:disable GitlabSecurity/PublicSend
+ cached_attributes[project.id][attribute]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/feature_flag_reference_filter.rb b/lib/banzai/filter/references/feature_flag_reference_filter.rb
new file mode 100644
index 00000000000..be9ded1ff43
--- /dev/null
+++ b/lib/banzai/filter/references/feature_flag_reference_filter.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ class FeatureFlagReferenceFilter < IssuableReferenceFilter
+ self.reference_type = :feature_flag
+
+ def self.object_class
+ Operations::FeatureFlag
+ end
+
+ def self.object_sym
+ :feature_flag
+ end
+
+ def parent_records(parent, ids)
+ parent.operations_feature_flags.where(iid: ids.to_a)
+ end
+
+ def url_for_object(feature_flag, project)
+ ::Gitlab::Routing.url_helpers.edit_project_feature_flag_url(
+ project,
+ feature_flag.iid,
+ only_path: context[:only_path]
+ )
+ end
+
+ def object_link_title(object, matches)
+ object.name
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/issuable_reference_filter.rb b/lib/banzai/filter/references/issuable_reference_filter.rb
new file mode 100644
index 00000000000..b8ccb926ae9
--- /dev/null
+++ b/lib/banzai/filter/references/issuable_reference_filter.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ class IssuableReferenceFilter < AbstractReferenceFilter
+ def record_identifier(record)
+ record.iid.to_i
+ end
+
+ def find_object(parent, iid)
+ records_per_parent[parent][iid]
+ end
+
+ def parent_from_ref(ref)
+ parent_per_reference[ref || current_parent_path]
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/issue_reference_filter.rb b/lib/banzai/filter/references/issue_reference_filter.rb
new file mode 100644
index 00000000000..eacf261b15f
--- /dev/null
+++ b/lib/banzai/filter/references/issue_reference_filter.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces issue references with links. References to
+ # issues that do not exist are ignored.
+ #
+ # This filter supports cross-project references.
+ #
+ # When external issues tracker like Jira is activated we should not
+ # use issue reference pattern, but we should still be able
+ # to reference issues from other GitLab projects.
+ class IssueReferenceFilter < IssuableReferenceFilter
+ self.reference_type = :issue
+
+ def self.object_class
+ Issue
+ end
+
+ def url_for_object(issue, project)
+ return issue_path(issue, project) if only_path?
+
+ issue_url(issue, project)
+ end
+
+ def parent_records(parent, ids)
+ parent.issues.where(iid: ids.to_a)
+ end
+
+ def object_link_text_extras(issue, matches)
+ super + design_link_extras(issue, matches.named_captures['path'])
+ end
+
+ private
+
+ def issue_path(issue, project)
+ Gitlab::Routing.url_helpers.namespace_project_issue_path(namespace_id: project.namespace, project_id: project, id: issue.iid)
+ end
+
+ def issue_url(issue, project)
+ Gitlab::Routing.url_helpers.namespace_project_issue_url(namespace_id: project.namespace, project_id: project, id: issue.iid)
+ end
+
+ def design_link_extras(issue, path)
+ if path == '/designs' && read_designs?(issue)
+ ['designs']
+ else
+ []
+ end
+ end
+
+ def read_designs?(issue)
+ issue.project.design_management_enabled?
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/iteration_reference_filter.rb b/lib/banzai/filter/references/iteration_reference_filter.rb
new file mode 100644
index 00000000000..cf3d446147f
--- /dev/null
+++ b/lib/banzai/filter/references/iteration_reference_filter.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # The actual filter is implemented in the EE mixin
+ class IterationReferenceFilter < AbstractReferenceFilter
+ self.reference_type = :iteration
+
+ def self.object_class
+ Iteration
+ end
+ end
+ end
+ end
+end
+
+Banzai::Filter::References::IterationReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::IterationReferenceFilter')
diff --git a/lib/banzai/filter/references/label_reference_filter.rb b/lib/banzai/filter/references/label_reference_filter.rb
new file mode 100644
index 00000000000..a6a5eec5d9a
--- /dev/null
+++ b/lib/banzai/filter/references/label_reference_filter.rb
@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces label references with links.
+ class LabelReferenceFilter < AbstractReferenceFilter
+ self.reference_type = :label
+
+ def self.object_class
+ Label
+ end
+
+ def find_object(parent_object, id)
+ find_labels(parent_object).find(id)
+ end
+
+ def references_in(text, pattern = Label.reference_pattern)
+ labels = {}
+ unescaped_html = unescape_html_entities(text).gsub(pattern) do |match|
+ namespace = $~[:namespace]
+ project = $~[:project]
+ project_path = full_project_path(namespace, project)
+ label = find_label_cached(project_path, $~[:label_id], $~[:label_name])
+
+ if label
+ labels[label.id] = yield match, label.id, project, namespace, $~
+ "#{REFERENCE_PLACEHOLDER}#{label.id}"
+ else
+ match
+ end
+ end
+
+ return text if labels.empty?
+
+ escape_with_placeholders(unescaped_html, labels)
+ end
+
+ def find_label_cached(parent_ref, label_id, label_name)
+ cached_call(:banzai_find_label_cached, label_name&.tr('"', '') || label_id, path: [object_class, parent_ref]) do
+ find_label(parent_ref, label_id, label_name)
+ end
+ end
+
+ def find_label(parent_ref, label_id, label_name)
+ parent = parent_from_ref(parent_ref)
+ return unless parent
+
+ label_params = label_params(label_id, label_name)
+ find_labels(parent).find_by(label_params)
+ end
+
+ def find_labels(parent)
+ params = if parent.is_a?(Group)
+ { group_id: parent.id,
+ include_ancestor_groups: true,
+ only_group_labels: true }
+ else
+ { project: parent,
+ include_ancestor_groups: true }
+ end
+
+ LabelsFinder.new(nil, params).execute(skip_authorization: true)
+ end
+
+ # Parameters to pass to `Label.find_by` based on the given arguments
+ #
+ # id - Integer ID to pass. If present, returns {id: id}
+ # name - String name to pass. If `id` is absent, finds by name without
+ # surrounding quotes.
+ #
+ # Returns a Hash.
+ def label_params(id, name)
+ if name
+ { name: name.tr('"', '') }
+ else
+ { id: id.to_i }
+ end
+ end
+
+ def url_for_object(label, parent)
+ label_url_method =
+ if context[:label_url_method]
+ context[:label_url_method]
+ elsif parent.is_a?(Project)
+ :project_issues_url
+ end
+
+ return unless label_url_method
+
+ Gitlab::Routing.url_helpers.public_send(label_url_method, parent, label_name: label.name, only_path: context[:only_path]) # rubocop:disable GitlabSecurity/PublicSend
+ end
+
+ def object_link_text(object, matches)
+ label_suffix = ''
+ parent = project || group
+
+ if project || full_path_ref?(matches)
+ project_path = full_project_path(matches[:namespace], matches[:project])
+ parent_from_ref = from_ref_cached(project_path)
+ reference = parent_from_ref.to_human_reference(parent)
+
+ label_suffix = " <i>in #{ERB::Util.html_escape(reference)}</i>" if reference.present?
+ end
+
+ presenter = object.present(issuable_subject: parent)
+ LabelsHelper.render_colored_label(presenter, suffix: label_suffix)
+ end
+
+ def wrap_link(link, label)
+ presenter = label.present(issuable_subject: project || group)
+ LabelsHelper.wrap_label_html(link, small: true, label: presenter)
+ end
+
+ def full_path_ref?(matches)
+ matches[:namespace] && matches[:project]
+ end
+
+ def reference_class(type, tooltip: true)
+ super + ' gl-link gl-label-link'
+ end
+
+ def object_link_title(object, matches)
+ presenter = object.present(issuable_subject: project || group)
+ LabelsHelper.label_tooltip_title(presenter)
+ end
+ end
+ end
+ end
+end
+
+Banzai::Filter::References::LabelReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::LabelReferenceFilter')
diff --git a/lib/banzai/filter/references/merge_request_reference_filter.rb b/lib/banzai/filter/references/merge_request_reference_filter.rb
new file mode 100644
index 00000000000..872c33f6873
--- /dev/null
+++ b/lib/banzai/filter/references/merge_request_reference_filter.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces merge request references with links. References
+ # to merge requests that do not exist are ignored.
+ #
+ # This filter supports cross-project references.
+ class MergeRequestReferenceFilter < IssuableReferenceFilter
+ self.reference_type = :merge_request
+
+ def self.object_class
+ MergeRequest
+ end
+
+ def url_for_object(mr, project)
+ h = Gitlab::Routing.url_helpers
+ h.project_merge_request_url(project, mr,
+ only_path: context[:only_path])
+ end
+
+ def object_link_title(object, matches)
+ # The method will return `nil` if object is not a commit
+ # allowing for properly handling the extended MR Tooltip
+ object_link_commit_title(object, matches)
+ end
+
+ def object_link_text_extras(object, matches)
+ extras = super
+
+ if commit_ref = object_link_commit_ref(object, matches)
+ klass = reference_class(:commit, tooltip: false)
+ commit_ref_tag = %(<span class="#{klass}">#{commit_ref}</span>)
+
+ return extras.unshift(commit_ref_tag)
+ end
+
+ path = matches[:path] if matches.names.include?("path")
+
+ case path
+ when '/diffs'
+ extras.unshift "diffs"
+ when '/commits'
+ extras.unshift "commits"
+ when '/builds'
+ extras.unshift "builds"
+ end
+
+ extras
+ end
+
+ def parent_records(parent, ids)
+ parent.merge_requests
+ .where(iid: ids.to_a)
+ .includes(target_project: :namespace)
+ end
+
+ def reference_class(object_sym, options = {})
+ super(object_sym, tooltip: false)
+ end
+
+ def data_attributes_for(text, parent, object, **data)
+ super.merge(project_path: parent.full_path, iid: object.iid, mr_title: object.title)
+ end
+
+ private
+
+ def object_link_commit_title(object, matches)
+ object_link_commit(object, matches)&.title
+ end
+
+ def object_link_commit_ref(object, matches)
+ object_link_commit(object, matches)&.short_id
+ end
+
+ def object_link_commit(object, matches)
+ return unless matches.names.include?('query') && query = matches[:query]
+
+ # Removes leading "?". CGI.parse expects "arg1&arg2&arg3"
+ params = CGI.parse(query.sub(/^\?/, ''))
+
+ return unless commit_sha = params['commit_id']&.first
+
+ if commit = find_commit_by_sha(object, commit_sha)
+ Commit.from_hash(commit.to_hash, object.project)
+ end
+ end
+
+ def find_commit_by_sha(object, commit_sha)
+ @all_commits ||= {}
+ @all_commits[object.id] ||= object.all_commits
+
+ @all_commits[object.id].find { |commit| commit.sha == commit_sha }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/milestone_reference_filter.rb b/lib/banzai/filter/references/milestone_reference_filter.rb
new file mode 100644
index 00000000000..49110194ddc
--- /dev/null
+++ b/lib/banzai/filter/references/milestone_reference_filter.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces milestone references with links.
+ class MilestoneReferenceFilter < AbstractReferenceFilter
+ include Gitlab::Utils::StrongMemoize
+
+ self.reference_type = :milestone
+
+ def self.object_class
+ Milestone
+ end
+
+ # Links to project milestones contain the IID, but when we're handling
+ # 'regular' references, we need to use the global ID to disambiguate
+ # between group and project milestones.
+ def find_object(parent, id)
+ return unless valid_context?(parent)
+
+ find_milestone_with_finder(parent, id: id)
+ end
+
+ def find_object_from_link(parent, iid)
+ return unless valid_context?(parent)
+
+ find_milestone_with_finder(parent, iid: iid)
+ end
+
+ def valid_context?(parent)
+ strong_memoize(:valid_context) do
+ group_context?(parent) || project_context?(parent)
+ end
+ end
+
+ def group_context?(parent)
+ strong_memoize(:group_context) do
+ parent.is_a?(Group)
+ end
+ end
+
+ def project_context?(parent)
+ strong_memoize(:project_context) do
+ parent.is_a?(Project)
+ end
+ end
+
+ def references_in(text, pattern = Milestone.reference_pattern)
+ # We'll handle here the references that follow the `reference_pattern`.
+ # Other patterns (for example, the link pattern) are handled by the
+ # default implementation.
+ return super(text, pattern) if pattern != Milestone.reference_pattern
+
+ milestones = {}
+ unescaped_html = unescape_html_entities(text).gsub(pattern) do |match|
+ milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name])
+
+ if milestone
+ milestones[milestone.id] = yield match, milestone.id, $~[:project], $~[:namespace], $~
+ "#{REFERENCE_PLACEHOLDER}#{milestone.id}"
+ else
+ match
+ end
+ end
+
+ return text if milestones.empty?
+
+ escape_with_placeholders(unescaped_html, milestones)
+ end
+
+ def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name)
+ project_path = full_project_path(namespace_ref, project_ref)
+
+ # Returns group if project is not found by path
+ parent = parent_from_ref(project_path)
+
+ return unless parent
+
+ milestone_params = milestone_params(milestone_id, milestone_name)
+
+ find_milestone_with_finder(parent, milestone_params)
+ end
+
+ def milestone_params(iid, name)
+ if name
+ { name: name.tr('"', '') }
+ else
+ { iid: iid.to_i }
+ end
+ end
+
+ def find_milestone_with_finder(parent, params)
+ finder_params = milestone_finder_params(parent, params[:iid].present?)
+
+ MilestonesFinder.new(finder_params).find_by(params)
+ end
+
+ def milestone_finder_params(parent, find_by_iid)
+ { order: nil, state: 'all' }.tap do |params|
+ params[:project_ids] = parent.id if project_context?(parent)
+
+ # We don't support IID lookups because IIDs can clash between
+ # group/project milestones and group/subgroup milestones.
+ params[:group_ids] = self_and_ancestors_ids(parent) unless find_by_iid
+ end
+ end
+
+ def self_and_ancestors_ids(parent)
+ if group_context?(parent)
+ parent.self_and_ancestors.select(:id)
+ elsif project_context?(parent)
+ parent.group&.self_and_ancestors&.select(:id)
+ end
+ end
+
+ def url_for_object(milestone, project)
+ Gitlab::Routing
+ .url_helpers
+ .milestone_url(milestone, only_path: context[:only_path])
+ end
+
+ def object_link_text(object, matches)
+ milestone_link = escape_once(super)
+ reference = object.project&.to_reference_base(project)
+
+ if reference.present?
+ "#{milestone_link} <i>in #{reference}</i>".html_safe
+ else
+ milestone_link
+ end
+ end
+
+ def object_link_title(object, matches)
+ nil
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/project_reference_filter.rb b/lib/banzai/filter/references/project_reference_filter.rb
new file mode 100644
index 00000000000..522c6e0f5f3
--- /dev/null
+++ b/lib/banzai/filter/references/project_reference_filter.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces project references with links.
+ class ProjectReferenceFilter < ReferenceFilter
+ self.reference_type = :project
+
+ # Public: Find `namespace/project>` project references in text
+ #
+ # ProjectReferenceFilter.references_in(text) do |match, project|
+ # "<a href=...>#{project}></a>"
+ # end
+ #
+ # text - String text to search.
+ #
+ # Yields the String match, and the String project name.
+ #
+ # Returns a String replaced with the return of the block.
+ def self.references_in(text)
+ text.gsub(Project.markdown_reference_pattern) do |match|
+ yield match, "#{$~[:namespace]}/#{$~[:project]}"
+ end
+ end
+
+ def call
+ ref_pattern = Project.markdown_reference_pattern
+ ref_pattern_start = /\A#{ref_pattern}\z/
+
+ nodes.each_with_index do |node, index|
+ if text_node?(node)
+ replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
+ project_link_filter(content)
+ end
+ elsif element_node?(node)
+ yield_valid_link(node) do |link, inner_html|
+ if link =~ ref_pattern_start
+ replace_link_node_with_href(node, index, link) do
+ project_link_filter(link, link_content: inner_html)
+ end
+ end
+ end
+ end
+ end
+
+ doc
+ end
+
+ # Replace `namespace/project>` project references in text with links to the referenced
+ # project page.
+ #
+ # text - String text to replace references in.
+ # link_content - Original content of the link being replaced.
+ #
+ # Returns a String with `namespace/project>` references replaced with links. All links
+ # have `gfm` and `gfm-project` class names attached for styling.
+ def project_link_filter(text, link_content: nil)
+ self.class.references_in(text) do |match, project_path|
+ cached_call(:banzai_url_for_object, match, path: [Project, project_path.downcase]) do
+ if project = projects_hash[project_path.downcase]
+ link_to_project(project, link_content: link_content) || match
+ else
+ match
+ end
+ end
+ end
+ end
+
+ # Returns a Hash containing all Project objects for the project
+ # references in the current document.
+ #
+ # The keys of this Hash are the project paths, the values the
+ # corresponding Project objects.
+ def projects_hash
+ @projects ||= Project.eager_load(:route, namespace: [:route])
+ .where_full_path_in(projects)
+ .index_by(&:full_path)
+ .transform_keys(&:downcase)
+ end
+
+ # Returns all projects referenced in the current document.
+ def projects
+ refs = Set.new
+
+ nodes.each do |node|
+ node.to_html.scan(Project.markdown_reference_pattern) do
+ refs << "#{$~[:namespace]}/#{$~[:project]}"
+ end
+ end
+
+ refs.to_a
+ end
+
+ private
+
+ def urls
+ Gitlab::Routing.url_helpers
+ end
+
+ def link_class
+ reference_class(:project)
+ end
+
+ def link_to_project(project, link_content: nil)
+ url = urls.project_url(project, only_path: context[:only_path])
+ data = data_attribute(project: project.id)
+ content = link_content || project.to_reference
+
+ link_tag(url, data, content, project.name)
+ end
+
+ def link_tag(url, data, link_content, title)
+ %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb
new file mode 100644
index 00000000000..dd15c43f5d8
--- /dev/null
+++ b/lib/banzai/filter/references/reference_filter.rb
@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+# Generated HTML is transformed back to GFM by app/assets/javascripts/behaviors/markdown/nodes/reference.js
+module Banzai
+ module Filter
+ module References
+ # Base class for GitLab Flavored Markdown reference filters.
+ #
+ # References within <pre>, <code>, <a>, and <style> elements are ignored.
+ #
+ # Context options:
+ # :project (required) - Current project, ignored if reference is cross-project.
+ # :only_path - Generate path-only links.
+ class ReferenceFilter < HTML::Pipeline::Filter
+ include RequestStoreReferenceCache
+ include OutputSafety
+
+ class << self
+ attr_accessor :reference_type
+
+ def call(doc, context = nil, result = nil)
+ new(doc, context, result).call_and_update_nodes
+ end
+ end
+
+ def initialize(doc, context = nil, result = nil)
+ super
+
+ @new_nodes = {}
+ @nodes = self.result[:reference_filter_nodes]
+ end
+
+ def call_and_update_nodes
+ with_update_nodes { call }
+ end
+
+ # Returns a data attribute String to attach to a reference link
+ #
+ # attributes - Hash, where the key becomes the data attribute name and the
+ # value is the data attribute value
+ #
+ # Examples:
+ #
+ # data_attribute(project: 1, issue: 2)
+ # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\""
+ #
+ # data_attribute(project: 3, merge_request: 4)
+ # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\""
+ #
+ # Returns a String
+ def data_attribute(attributes = {})
+ attributes = attributes.reject { |_, v| v.nil? }
+
+ attributes[:reference_type] ||= self.class.reference_type
+ attributes[:container] ||= 'body'
+ attributes[:placement] ||= 'top'
+ attributes.delete(:original) if context[:no_original_data]
+ attributes.map do |key, value|
+ %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}")
+ end.join(' ')
+ end
+
+ def ignore_ancestor_query
+ @ignore_ancestor_query ||= begin
+ parents = %w(pre code a style)
+ parents << 'blockquote' if context[:ignore_blockquotes]
+
+ parents.map { |n| "ancestor::#{n}" }.join(' or ')
+ end
+ end
+
+ def project
+ context[:project]
+ end
+
+ def group
+ context[:group]
+ end
+
+ def user
+ context[:user]
+ end
+
+ def skip_project_check?
+ context[:skip_project_check]
+ end
+
+ def reference_class(type, tooltip: true)
+ gfm_klass = "gfm gfm-#{type}"
+
+ return gfm_klass unless tooltip
+
+ "#{gfm_klass} has-tooltip"
+ end
+
+ # Ensure that a :project key exists in context
+ #
+ # Note that while the key might exist, its value could be nil!
+ def validate
+ needs :project unless skip_project_check?
+ end
+
+ # Iterates over all <a> and text() nodes in a document.
+ #
+ # Nodes are skipped whenever their ancestor is one of the nodes returned
+ # by `ignore_ancestor_query`. Link tags are not processed if they have a
+ # "gfm" class or the "href" attribute is empty.
+ def each_node
+ return to_enum(__method__) unless block_given?
+
+ doc.xpath(query).each do |node|
+ yield node
+ end
+ end
+
+ # Returns an Array containing all HTML nodes.
+ def nodes
+ @nodes ||= each_node.to_a
+ end
+
+ # Yields the link's URL and inner HTML whenever the node is a valid <a> tag.
+ def yield_valid_link(node)
+ link = unescape_link(node.attr('href').to_s)
+ inner_html = node.inner_html
+
+ return unless link.force_encoding('UTF-8').valid_encoding?
+
+ yield link, inner_html
+ end
+
+ def unescape_link(href)
+ CGI.unescape(href)
+ end
+
+ def replace_text_when_pattern_matches(node, index, pattern)
+ return unless node.text =~ pattern
+
+ content = node.to_html
+ html = yield content
+
+ replace_text_with_html(node, index, html) unless html == content
+ end
+
+ def replace_link_node_with_text(node, index)
+ html = yield
+
+ replace_text_with_html(node, index, html) unless html == node.text
+ end
+
+ def replace_link_node_with_href(node, index, link)
+ html = yield
+
+ replace_text_with_html(node, index, html) unless html == link
+ end
+
+ def text_node?(node)
+ node.is_a?(Nokogiri::XML::Text)
+ end
+
+ def element_node?(node)
+ node.is_a?(Nokogiri::XML::Element)
+ end
+
+ private
+
+ def query
+ @query ||= %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})]
+ | descendant-or-self::a[
+ not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "")
+ ]}
+ end
+
+ def replace_text_with_html(node, index, html)
+ replace_and_update_new_nodes(node, index, html)
+ end
+
+ def replace_and_update_new_nodes(node, index, html)
+ previous_node = node.previous
+ next_node = node.next
+ parent_node = node.parent
+ # Unfortunately node.replace(html) returns re-parented nodes, not the actual replaced nodes in the doc
+ # We need to find the actual nodes in the doc that were replaced
+ node.replace(html)
+ @new_nodes[index] = []
+
+ # We replaced node with new nodes, so we find first new node. If previous_node is nil, we take first parent child
+ new_node = previous_node ? previous_node.next : parent_node&.children&.first
+
+ # We iterate from first to last replaced node and store replaced nodes in @new_nodes
+ while new_node && new_node != next_node
+ @new_nodes[index] << new_node.xpath(query)
+ new_node = new_node.next
+ end
+
+ @new_nodes[index].flatten!
+ end
+
+ def only_path?
+ context[:only_path]
+ end
+
+ def with_update_nodes
+ @new_nodes = {}
+ yield.tap { update_nodes! }
+ end
+
+ # Once Filter completes replacing nodes, we update nodes with @new_nodes
+ def update_nodes!
+ @new_nodes.sort_by { |index, _new_nodes| -index }.each do |index, new_nodes|
+ nodes[index, 1] = new_nodes
+ end
+ result[:reference_filter_nodes] = nodes
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/snippet_reference_filter.rb b/lib/banzai/filter/references/snippet_reference_filter.rb
new file mode 100644
index 00000000000..bf7e0f78609
--- /dev/null
+++ b/lib/banzai/filter/references/snippet_reference_filter.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces snippet references with links. References to
+ # snippets that do not exist are ignored.
+ #
+ # This filter supports cross-project references.
+ class SnippetReferenceFilter < AbstractReferenceFilter
+ self.reference_type = :snippet
+
+ def self.object_class
+ Snippet
+ end
+
+ def find_object(project, id)
+ return unless project.is_a?(Project)
+
+ project.snippets.find_by(id: id)
+ end
+
+ def url_for_object(snippet, project)
+ h = Gitlab::Routing.url_helpers
+ h.project_snippet_url(project, snippet,
+ only_path: context[:only_path])
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/user_reference_filter.rb b/lib/banzai/filter/references/user_reference_filter.rb
new file mode 100644
index 00000000000..04665973f51
--- /dev/null
+++ b/lib/banzai/filter/references/user_reference_filter.rb
@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # HTML filter that replaces user or group references with links.
+ #
+ # A special `@all` reference is also supported.
+ class UserReferenceFilter < ReferenceFilter
+ self.reference_type = :user
+
+ # Public: Find `@user` user references in text
+ #
+ # UserReferenceFilter.references_in(text) do |match, username|
+ # "<a href=...>@#{user}</a>"
+ # end
+ #
+ # text - String text to search.
+ #
+ # Yields the String match, and the String user name.
+ #
+ # Returns a String replaced with the return of the block.
+ def self.references_in(text)
+ text.gsub(User.reference_pattern) do |match|
+ yield match, $~[:user]
+ end
+ end
+
+ def call
+ return doc if project.nil? && group.nil? && !skip_project_check?
+
+ ref_pattern = User.reference_pattern
+ ref_pattern_start = /\A#{ref_pattern}\z/
+
+ nodes.each_with_index do |node, index|
+ if text_node?(node)
+ replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
+ user_link_filter(content)
+ end
+ elsif element_node?(node)
+ yield_valid_link(node) do |link, inner_html|
+ if link =~ ref_pattern_start
+ replace_link_node_with_href(node, index, link) do
+ user_link_filter(link, link_content: inner_html)
+ end
+ end
+ end
+ end
+ end
+
+ doc
+ end
+
+ # Replace `@user` user references in text with links to the referenced
+ # user's profile page.
+ #
+ # text - String text to replace references in.
+ # link_content - Original content of the link being replaced.
+ #
+ # Returns a String with `@user` references replaced with links. All links
+ # have `gfm` and `gfm-project_member` class names attached for styling.
+ def user_link_filter(text, link_content: nil)
+ self.class.references_in(text) do |match, username|
+ if username == 'all' && !skip_project_check?
+ link_to_all(link_content: link_content)
+ else
+ cached_call(:banzai_url_for_object, match, path: [User, username.downcase]) do
+ if namespace = namespaces[username.downcase]
+ link_to_namespace(namespace, link_content: link_content) || match
+ else
+ match
+ end
+ end
+ end
+ end
+ end
+
+ # Returns a Hash containing all Namespace objects for the username
+ # references in the current document.
+ #
+ # The keys of this Hash are the namespace paths, the values the
+ # corresponding Namespace objects.
+ def namespaces
+ @namespaces ||= Namespace.eager_load(:owner, :route)
+ .where_full_path_in(usernames)
+ .index_by(&:full_path)
+ .transform_keys(&:downcase)
+ end
+
+ # Returns all usernames referenced in the current document.
+ def usernames
+ refs = Set.new
+
+ nodes.each do |node|
+ node.to_html.scan(User.reference_pattern) do
+ refs << $~[:user]
+ end
+ end
+
+ refs.to_a
+ end
+
+ private
+
+ def urls
+ Gitlab::Routing.url_helpers
+ end
+
+ def link_class
+ [reference_class(:project_member, tooltip: false), "js-user-link"].join(" ")
+ end
+
+ def link_to_all(link_content: nil)
+ author = context[:author]
+
+ if author && !team_member?(author)
+ link_content
+ else
+ parent_url(link_content, author)
+ end
+ end
+
+ def link_to_namespace(namespace, link_content: nil)
+ if namespace.is_a?(Group)
+ link_to_group(namespace.full_path, namespace, link_content: link_content)
+ else
+ link_to_user(namespace.path, namespace, link_content: link_content)
+ end
+ end
+
+ def link_to_group(group, namespace, link_content: nil)
+ url = urls.group_url(group, only_path: context[:only_path])
+ data = data_attribute(group: namespace.id)
+ content = link_content || Group.reference_prefix + group
+
+ link_tag(url, data, content, namespace.full_name)
+ end
+
+ def link_to_user(user, namespace, link_content: nil)
+ url = urls.user_url(user, only_path: context[:only_path])
+ data = data_attribute(user: namespace.owner_id)
+ content = link_content || User.reference_prefix + user
+
+ link_tag(url, data, content, namespace.owner_name)
+ end
+
+ def link_tag(url, data, link_content, title)
+ %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>)
+ end
+
+ def parent
+ context[:project] || context[:group]
+ end
+
+ def parent_group?
+ parent.is_a?(Group)
+ end
+
+ def team_member?(user)
+ if parent_group?
+ parent.member?(user)
+ else
+ parent.team.member?(user)
+ end
+ end
+
+ def parent_url(link_content, author)
+ if parent_group?
+ url = urls.group_url(parent, only_path: context[:only_path])
+ data = data_attribute(group: group.id, author: author.try(:id))
+ else
+ url = urls.project_url(parent, only_path: context[:only_path])
+ data = data_attribute(project: project.id, author: author.try(:id))
+ end
+
+ content = link_content || User.reference_prefix + 'all'
+ link_tag(url, data, content, 'All Project and Group Members')
+ end
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/references/vulnerability_reference_filter.rb b/lib/banzai/filter/references/vulnerability_reference_filter.rb
new file mode 100644
index 00000000000..e5f2408eda4
--- /dev/null
+++ b/lib/banzai/filter/references/vulnerability_reference_filter.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ module References
+ # The actual filter is implemented in the EE mixin
+ class VulnerabilityReferenceFilter < IssuableReferenceFilter
+ self.reference_type = :vulnerability
+
+ def self.object_class
+ Vulnerability
+ end
+
+ private
+
+ def project
+ context[:project]
+ end
+ end
+ end
+ end
+end
+
+Banzai::Filter::References::VulnerabilityReferenceFilter.prepend_if_ee('EE::Banzai::Filter::References::VulnerabilityReferenceFilter')
diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb
index 66b9aac3e7e..04bbcabd93f 100644
--- a/lib/banzai/filter/repository_link_filter.rb
+++ b/lib/banzai/filter/repository_link_filter.rb
@@ -60,7 +60,7 @@ module Banzai
def get_uri_types(paths)
return {} if paths.empty?
- uri_types = Hash[paths.collect { |name| [name, nil] }]
+ uri_types = paths.to_h { |name| [name, nil] }
get_blob_types(paths).each do |name, type|
if type == :blob
diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb
deleted file mode 100644
index f4b6edb6174..00000000000
--- a/lib/banzai/filter/snippet_reference_filter.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces snippet references with links. References to
- # snippets that do not exist are ignored.
- #
- # This filter supports cross-project references.
- class SnippetReferenceFilter < AbstractReferenceFilter
- self.reference_type = :snippet
-
- def self.object_class
- Snippet
- end
-
- def find_object(project, id)
- return unless project.is_a?(Project)
-
- project.snippets.find_by(id: id)
- end
-
- def url_for_object(snippet, project)
- h = Gitlab::Routing.url_helpers
- h.project_snippet_url(project, snippet,
- only_path: context[:only_path])
- end
- end
- end
-end
diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb
index f52ffe117d9..ca26e6d1581 100644
--- a/lib/banzai/filter/spaced_link_filter.rb
+++ b/lib/banzai/filter/spaced_link_filter.rb
@@ -42,7 +42,7 @@ module Banzai
TEXT_QUERY = %Q(descendant-or-self::text()[
not(#{IGNORE_PARENTS.map { |p| "ancestor::#{p}" }.join(' or ')})
and contains(., ']\(')
- ]).freeze
+ ])
def call
doc.xpath(TEXT_QUERY).each do |node|
diff --git a/lib/banzai/filter/suggestion_filter.rb b/lib/banzai/filter/suggestion_filter.rb
index ae093580001..56a14ec0737 100644
--- a/lib/banzai/filter/suggestion_filter.rb
+++ b/lib/banzai/filter/suggestion_filter.rb
@@ -10,7 +10,7 @@ module Banzai
def call
return doc unless suggestions_filter_enabled?
- doc.search('pre.suggestion > code').each do |node|
+ doc.search('pre.language-suggestion > code').each do |node|
node.add_class(TAG_CLASS)
end
diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb
index 1d3bbe43344..731a2bb4c77 100644
--- a/lib/banzai/filter/syntax_highlight_filter.rb
+++ b/lib/banzai/filter/syntax_highlight_filter.rb
@@ -37,7 +37,7 @@ module Banzai
begin
code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language)
- css_classes << " #{language}" if language
+ css_classes << " language-#{language}" if language
rescue
# Gracefully handle syntax highlighter bugs/errors to ensure users can
# still access an issue/comment/etc. First, retry with the plain text
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
deleted file mode 100644
index 262385524f4..00000000000
--- a/lib/banzai/filter/user_reference_filter.rb
+++ /dev/null
@@ -1,180 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # HTML filter that replaces user or group references with links.
- #
- # A special `@all` reference is also supported.
- class UserReferenceFilter < ReferenceFilter
- self.reference_type = :user
-
- # Public: Find `@user` user references in text
- #
- # UserReferenceFilter.references_in(text) do |match, username|
- # "<a href=...>@#{user}</a>"
- # end
- #
- # text - String text to search.
- #
- # Yields the String match, and the String user name.
- #
- # Returns a String replaced with the return of the block.
- def self.references_in(text)
- text.gsub(User.reference_pattern) do |match|
- yield match, $~[:user]
- end
- end
-
- def call
- return doc if project.nil? && group.nil? && !skip_project_check?
-
- ref_pattern = User.reference_pattern
- ref_pattern_start = /\A#{ref_pattern}\z/
-
- nodes.each_with_index do |node, index|
- if text_node?(node)
- replace_text_when_pattern_matches(node, index, ref_pattern) do |content|
- user_link_filter(content)
- end
- elsif element_node?(node)
- yield_valid_link(node) do |link, inner_html|
- if link =~ ref_pattern_start
- replace_link_node_with_href(node, index, link) do
- user_link_filter(link, link_content: inner_html)
- end
- end
- end
- end
- end
-
- doc
- end
-
- # Replace `@user` user references in text with links to the referenced
- # user's profile page.
- #
- # text - String text to replace references in.
- # link_content - Original content of the link being replaced.
- #
- # Returns a String with `@user` references replaced with links. All links
- # have `gfm` and `gfm-project_member` class names attached for styling.
- def user_link_filter(text, link_content: nil)
- self.class.references_in(text) do |match, username|
- if username == 'all' && !skip_project_check?
- link_to_all(link_content: link_content)
- else
- cached_call(:banzai_url_for_object, match, path: [User, username.downcase]) do
- if namespace = namespaces[username.downcase]
- link_to_namespace(namespace, link_content: link_content) || match
- else
- match
- end
- end
- end
- end
- end
-
- # Returns a Hash containing all Namespace objects for the username
- # references in the current document.
- #
- # The keys of this Hash are the namespace paths, the values the
- # corresponding Namespace objects.
- def namespaces
- @namespaces ||= Namespace.eager_load(:owner, :route)
- .where_full_path_in(usernames)
- .index_by(&:full_path)
- .transform_keys(&:downcase)
- end
-
- # Returns all usernames referenced in the current document.
- def usernames
- refs = Set.new
-
- nodes.each do |node|
- node.to_html.scan(User.reference_pattern) do
- refs << $~[:user]
- end
- end
-
- refs.to_a
- end
-
- private
-
- def urls
- Gitlab::Routing.url_helpers
- end
-
- def link_class
- [reference_class(:project_member, tooltip: false), "js-user-link"].join(" ")
- end
-
- def link_to_all(link_content: nil)
- author = context[:author]
-
- if author && !team_member?(author)
- link_content
- else
- parent_url(link_content, author)
- end
- end
-
- def link_to_namespace(namespace, link_content: nil)
- if namespace.is_a?(Group)
- link_to_group(namespace.full_path, namespace, link_content: link_content)
- else
- link_to_user(namespace.path, namespace, link_content: link_content)
- end
- end
-
- def link_to_group(group, namespace, link_content: nil)
- url = urls.group_url(group, only_path: context[:only_path])
- data = data_attribute(group: namespace.id)
- content = link_content || Group.reference_prefix + group
-
- link_tag(url, data, content, namespace.full_name)
- end
-
- def link_to_user(user, namespace, link_content: nil)
- url = urls.user_url(user, only_path: context[:only_path])
- data = data_attribute(user: namespace.owner_id)
- content = link_content || User.reference_prefix + user
-
- link_tag(url, data, content, namespace.owner_name)
- end
-
- def link_tag(url, data, link_content, title)
- %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>)
- end
-
- def parent
- context[:project] || context[:group]
- end
-
- def parent_group?
- parent.is_a?(Group)
- end
-
- def team_member?(user)
- if parent_group?
- parent.member?(user)
- else
- parent.team.member?(user)
- end
- end
-
- def parent_url(link_content, author)
- if parent_group?
- url = urls.group_url(parent, only_path: context[:only_path])
- data = data_attribute(group: group.id, author: author.try(:id))
- else
- url = urls.project_url(parent, only_path: context[:only_path])
- data = data_attribute(project: project.id, author: author.try(:id))
- end
-
- content = link_content || User.reference_prefix + 'all'
- link_tag(url, data, content, 'All Project and Group Members')
- end
- end
- end
-end
diff --git a/lib/banzai/filter/vulnerability_reference_filter.rb b/lib/banzai/filter/vulnerability_reference_filter.rb
deleted file mode 100644
index a59e9836d69..00000000000
--- a/lib/banzai/filter/vulnerability_reference_filter.rb
+++ /dev/null
@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-module Banzai
- module Filter
- # The actual filter is implemented in the EE mixin
- class VulnerabilityReferenceFilter < IssuableReferenceFilter
- self.reference_type = :vulnerability
-
- def self.object_class
- Vulnerability
- end
-
- private
-
- def project
- context[:project]
- end
- end
- end
-end
-
-Banzai::Filter::VulnerabilityReferenceFilter.prepend_if_ee('EE::Banzai::Filter::VulnerabilityReferenceFilter')
diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb
index f4cc8beeb52..b4c2e7efae3 100644
--- a/lib/banzai/filter/wiki_link_filter/rewriter.rb
+++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb
@@ -6,7 +6,7 @@ module Banzai
class Rewriter
def initialize(link_string, wiki:, slug:)
@uri = Addressable::URI.parse(link_string)
- @wiki_base_path = wiki && wiki.wiki_base_path
+ @wiki_base_path = wiki&.wiki_base_path
@slug = slug
end
@@ -41,7 +41,8 @@ module Banzai
# Any link _not_ of the form `http://example.com/`
def apply_relative_link_rules!
if @uri.relative? && @uri.path.present?
- link = ::File.join(@wiki_base_path, @uri.path)
+ link = @uri.path
+ link = ::File.join(@wiki_base_path, link) unless link.starts_with?(@wiki_base_path)
link = "#{link}##{@uri.fragment}" if @uri.fragment
@uri = Addressable::URI.parse(link)
end
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index e5ec0a0a006..028e3c44dc3 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -51,19 +51,19 @@ module Banzai
def self.reference_filters
[
- Filter::UserReferenceFilter,
- Filter::ProjectReferenceFilter,
- Filter::DesignReferenceFilter,
- Filter::IssueReferenceFilter,
- Filter::ExternalIssueReferenceFilter,
- Filter::MergeRequestReferenceFilter,
- Filter::SnippetReferenceFilter,
- Filter::CommitRangeReferenceFilter,
- Filter::CommitReferenceFilter,
- Filter::LabelReferenceFilter,
- Filter::MilestoneReferenceFilter,
- Filter::AlertReferenceFilter,
- Filter::FeatureFlagReferenceFilter
+ Filter::References::UserReferenceFilter,
+ Filter::References::ProjectReferenceFilter,
+ Filter::References::DesignReferenceFilter,
+ Filter::References::IssueReferenceFilter,
+ Filter::References::ExternalIssueReferenceFilter,
+ Filter::References::MergeRequestReferenceFilter,
+ Filter::References::SnippetReferenceFilter,
+ Filter::References::CommitRangeReferenceFilter,
+ Filter::References::CommitReferenceFilter,
+ Filter::References::LabelReferenceFilter,
+ Filter::References::MilestoneReferenceFilter,
+ Filter::References::AlertReferenceFilter,
+ Filter::References::FeatureFlagReferenceFilter
]
end
diff --git a/lib/banzai/pipeline/label_pipeline.rb b/lib/banzai/pipeline/label_pipeline.rb
index 725cccc4b2b..ccfda2052e6 100644
--- a/lib/banzai/pipeline/label_pipeline.rb
+++ b/lib/banzai/pipeline/label_pipeline.rb
@@ -6,7 +6,7 @@ module Banzai
def self.filters
@filters ||= FilterArray[
Filter::SanitizationFilter,
- Filter::LabelReferenceFilter
+ Filter::References::LabelReferenceFilter
]
end
end
diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb
index 4bf98099662..65a5e28b704 100644
--- a/lib/banzai/pipeline/single_line_pipeline.rb
+++ b/lib/banzai/pipeline/single_line_pipeline.rb
@@ -17,15 +17,15 @@ module Banzai
def self.reference_filters
[
- Filter::UserReferenceFilter,
- Filter::IssueReferenceFilter,
- Filter::ExternalIssueReferenceFilter,
- Filter::MergeRequestReferenceFilter,
- Filter::SnippetReferenceFilter,
- Filter::CommitRangeReferenceFilter,
- Filter::CommitReferenceFilter,
- Filter::AlertReferenceFilter,
- Filter::FeatureFlagReferenceFilter
+ Filter::References::UserReferenceFilter,
+ Filter::References::IssueReferenceFilter,
+ Filter::References::ExternalIssueReferenceFilter,
+ Filter::References::MergeRequestReferenceFilter,
+ Filter::References::SnippetReferenceFilter,
+ Filter::References::CommitRangeReferenceFilter,
+ Filter::References::CommitReferenceFilter,
+ Filter::References::AlertReferenceFilter,
+ Filter::References::FeatureFlagReferenceFilter
]
end
diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb
index 97a03895ff3..caba9570ab9 100644
--- a/lib/banzai/pipeline/wiki_pipeline.rb
+++ b/lib/banzai/pipeline/wiki_pipeline.rb
@@ -5,7 +5,7 @@ module Banzai
class WikiPipeline < FullPipeline
def self.filters
@filters ||= begin
- super.insert_after(Filter::TableOfContentsFilter, Filter::GollumTagsFilter)
+ super.insert_before(Filter::ImageLazyLoadFilter, Filter::GollumTagsFilter)
.insert_before(Filter::TaskListFilter, Filter::WikiLinkFilter)
end
end
diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb
index 2e81863e53a..ef99122cdfd 100644
--- a/lib/bulk_imports/clients/http.rb
+++ b/lib/bulk_imports/clients/http.rb
@@ -3,9 +3,9 @@
module BulkImports
module Clients
class Http
- API_VERSION = 'v4'.freeze
- DEFAULT_PAGE = 1.freeze
- DEFAULT_PER_PAGE = 30.freeze
+ API_VERSION = 'v4'
+ DEFAULT_PAGE = 1
+ DEFAULT_PER_PAGE = 30
ConnectionError = Class.new(StandardError)
@@ -23,7 +23,7 @@ module BulkImports
resource_url(resource),
headers: request_headers,
follow_redirects: false,
- query: query.merge(request_query)
+ query: query.reverse_merge(request_query)
)
end
end
diff --git a/lib/bulk_imports/common/extractors/rest_extractor.rb b/lib/bulk_imports/common/extractors/rest_extractor.rb
new file mode 100644
index 00000000000..b18e27fd475
--- /dev/null
+++ b/lib/bulk_imports/common/extractors/rest_extractor.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Common
+ module Extractors
+ class RestExtractor
+ def initialize(options = {})
+ @query = options[:query]
+ end
+
+ def extract(context)
+ client = http_client(context.configuration)
+ params = query.to_h(context)
+ response = client.get(params[:resource], params[:query])
+
+ BulkImports::Pipeline::ExtractedData.new(
+ data: response.parsed_response,
+ page_info: page_info(response.headers)
+ )
+ end
+
+ private
+
+ attr_reader :query
+
+ def http_client(configuration)
+ @http_client ||= BulkImports::Clients::Http.new(
+ uri: configuration.url,
+ token: configuration.access_token,
+ per_page: 100
+ )
+ end
+
+ def page_info(headers)
+ next_page = headers['x-next-page']
+
+ {
+ 'has_next_page' => next_page.present?,
+ 'next_page' => next_page
+ }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/common/transformers/user_reference_transformer.rb b/lib/bulk_imports/common/transformers/user_reference_transformer.rb
index ca077b4ef43..c330ea59113 100644
--- a/lib/bulk_imports/common/transformers/user_reference_transformer.rb
+++ b/lib/bulk_imports/common/transformers/user_reference_transformer.rb
@@ -12,7 +12,7 @@ module BulkImports
DEFAULT_REFERENCE = 'user'
def initialize(options = {})
- @reference = options[:reference] || DEFAULT_REFERENCE
+ @reference = options[:reference].to_s.presence || DEFAULT_REFERENCE
@suffixed_reference = "#{@reference}_id"
end
diff --git a/lib/bulk_imports/groups/graphql/get_labels_query.rb b/lib/bulk_imports/groups/graphql/get_labels_query.rb
index 23efbc33581..f957cf0be52 100644
--- a/lib/bulk_imports/groups/graphql/get_labels_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_labels_query.rb
@@ -8,11 +8,11 @@ module BulkImports
def to_s
<<-'GRAPHQL'
- query ($full_path: ID!, $cursor: String) {
+ query ($full_path: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
- labels(first: 100, after: $cursor, onlyGroupLabels: true) {
+ labels(first: $per_page, after: $cursor, onlyGroupLabels: true) {
page_info: pageInfo {
- end_cursor: endCursor
+ next_page: endCursor
has_next_page: hasNextPage
}
nodes {
@@ -31,7 +31,8 @@ module BulkImports
def variables(context)
{
full_path: context.entity.source_full_path,
- cursor: context.entity.next_page_for(:labels)
+ cursor: context.tracker.next_page,
+ per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
diff --git a/lib/bulk_imports/groups/graphql/get_members_query.rb b/lib/bulk_imports/groups/graphql/get_members_query.rb
index e3a78124a47..e44d3c5aa9b 100644
--- a/lib/bulk_imports/groups/graphql/get_members_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_members_query.rb
@@ -7,11 +7,11 @@ module BulkImports
extend self
def to_s
<<-'GRAPHQL'
- query($full_path: ID!, $cursor: String) {
+ query($full_path: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
- group_members: groupMembers(relations: DIRECT, first: 100, after: $cursor) {
+ group_members: groupMembers(relations: DIRECT, first: $per_page, after: $cursor) {
page_info: pageInfo {
- end_cursor: endCursor
+ next_page: endCursor
has_next_page: hasNextPage
}
nodes {
@@ -34,7 +34,8 @@ module BulkImports
def variables(context)
{
full_path: context.entity.source_full_path,
- cursor: context.entity.next_page_for(:group_members)
+ cursor: context.tracker.next_page,
+ per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
diff --git a/lib/bulk_imports/groups/graphql/get_milestones_query.rb b/lib/bulk_imports/groups/graphql/get_milestones_query.rb
index 2ade87e6fa0..5dd5b31cf0e 100644
--- a/lib/bulk_imports/groups/graphql/get_milestones_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_milestones_query.rb
@@ -8,14 +8,15 @@ module BulkImports
def to_s
<<-'GRAPHQL'
- query ($full_path: ID!, $cursor: String) {
+ query ($full_path: ID!, $cursor: String, $per_page: Int) {
group(fullPath: $full_path) {
- milestones(first: 100, after: $cursor, includeDescendants: false) {
+ milestones(first: $per_page, after: $cursor, includeDescendants: false) {
page_info: pageInfo {
- end_cursor: endCursor
+ next_page: endCursor
has_next_page: hasNextPage
}
nodes {
+ iid
title
description
state
@@ -33,7 +34,8 @@ module BulkImports
def variables(context)
{
full_path: context.entity.source_full_path,
- cursor: context.entity.next_page_for(:milestones)
+ cursor: context.tracker.next_page,
+ per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
}
end
diff --git a/lib/bulk_imports/groups/pipelines/badges_pipeline.rb b/lib/bulk_imports/groups/pipelines/badges_pipeline.rb
new file mode 100644
index 00000000000..8569ff3f77a
--- /dev/null
+++ b/lib/bulk_imports/groups/pipelines/badges_pipeline.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Pipelines
+ class BadgesPipeline
+ include Pipeline
+
+ extractor BulkImports::Common::Extractors::RestExtractor,
+ query: BulkImports::Groups::Rest::GetBadgesQuery
+
+ transformer Common::Transformers::ProhibitedAttributesTransformer
+
+ def transform(_, data)
+ return if data.blank?
+
+ {
+ name: data['name'],
+ link_url: data['link_url'],
+ image_url: data['image_url']
+ }
+ end
+
+ def load(context, data)
+ return if data.blank?
+
+ context.group.badges.create!(data)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/groups/pipelines/entity_finisher.rb b/lib/bulk_imports/groups/pipelines/entity_finisher.rb
new file mode 100644
index 00000000000..1d237bc0f7f
--- /dev/null
+++ b/lib/bulk_imports/groups/pipelines/entity_finisher.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Pipelines
+ class EntityFinisher
+ def initialize(context)
+ @context = context
+ end
+
+ def run
+ return if context.entity.finished?
+
+ context.entity.finish!
+
+ logger.info(
+ bulk_import_id: context.bulk_import.id,
+ bulk_import_entity_id: context.entity.id,
+ bulk_import_entity_type: context.entity.source_type,
+ pipeline_class: self.class.name,
+ message: 'Entity finished'
+ )
+ end
+
+ private
+
+ attr_reader :context
+
+ def logger
+ @logger ||= Gitlab::Import::Logger.build
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb
index 9f8b8682751..0dc4a968b84 100644
--- a/lib/bulk_imports/groups/pipelines/labels_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/labels_pipeline.rb
@@ -14,18 +14,6 @@ module BulkImports
def load(context, data)
Labels::CreateService.new(data).execute(group: context.group)
end
-
- def after_run(extracted_data)
- context.entity.update_tracker_for(
- relation: :labels,
- has_next_page: extracted_data.has_next_page?,
- next_page: extracted_data.next_page
- )
-
- if extracted_data.has_next_page?
- run
- end
- end
end
end
end
diff --git a/lib/bulk_imports/groups/pipelines/members_pipeline.rb b/lib/bulk_imports/groups/pipelines/members_pipeline.rb
index 32fc931e8c3..5e4293d2c06 100644
--- a/lib/bulk_imports/groups/pipelines/members_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/members_pipeline.rb
@@ -17,18 +17,6 @@ module BulkImports
context.group.members.create!(data)
end
-
- def after_run(extracted_data)
- context.entity.update_tracker_for(
- relation: :group_members,
- has_next_page: extracted_data.has_next_page?,
- next_page: extracted_data.next_page
- )
-
- if extracted_data.has_next_page?
- run
- end
- end
end
end
end
diff --git a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb
index 8497162e0e7..9b2be30735c 100644
--- a/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/milestones_pipeline.rb
@@ -19,18 +19,6 @@ module BulkImports
context.group.milestones.create!(data)
end
- def after_run(extracted_data)
- context.entity.update_tracker_for(
- relation: :milestones,
- has_next_page: extracted_data.has_next_page?,
- next_page: extracted_data.next_page
- )
-
- if extracted_data.has_next_page?
- run
- end
- end
-
private
def authorized?
diff --git a/lib/bulk_imports/groups/rest/get_badges_query.rb b/lib/bulk_imports/groups/rest/get_badges_query.rb
new file mode 100644
index 00000000000..79ffdd9a1f6
--- /dev/null
+++ b/lib/bulk_imports/groups/rest/get_badges_query.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module BulkImports
+ module Groups
+ module Rest
+ module GetBadgesQuery
+ extend self
+
+ def to_h(context)
+ encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path)
+
+ {
+ resource: ['groups', encoded_full_path, 'badges'].join('/'),
+ query: {
+ page: context.tracker.next_page
+ }
+ }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb
deleted file mode 100644
index f016b552fd4..00000000000
--- a/lib/bulk_imports/importers/group_importer.rb
+++ /dev/null
@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-module BulkImports
- module Importers
- class GroupImporter
- def initialize(entity)
- @entity = entity
- end
-
- def execute
- context = BulkImports::Pipeline::Context.new(entity)
-
- pipelines.each { |pipeline| pipeline.new(context).run }
-
- entity.finish!
- end
-
- private
-
- attr_reader :entity
-
- def pipelines
- [
- BulkImports::Groups::Pipelines::GroupPipeline,
- BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline,
- BulkImports::Groups::Pipelines::MembersPipeline,
- BulkImports::Groups::Pipelines::LabelsPipeline,
- BulkImports::Groups::Pipelines::MilestonesPipeline
- ]
- end
- end
- end
-end
-
-BulkImports::Importers::GroupImporter.prepend_if_ee('EE::BulkImports::Importers::GroupImporter')
diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb
index 14445162737..df4f020d6b2 100644
--- a/lib/bulk_imports/pipeline.rb
+++ b/lib/bulk_imports/pipeline.rb
@@ -15,6 +15,10 @@ module BulkImports
@context = context
end
+ def tracker
+ @tracker ||= context.tracker
+ end
+
included do
private
diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb
index dd121b2dbed..3c69c729f36 100644
--- a/lib/bulk_imports/pipeline/context.rb
+++ b/lib/bulk_imports/pipeline/context.rb
@@ -3,25 +3,33 @@
module BulkImports
module Pipeline
class Context
- attr_reader :entity, :bulk_import
attr_accessor :extra
- def initialize(entity, extra = {})
- @entity = entity
- @bulk_import = entity.bulk_import
+ attr_reader :tracker
+
+ def initialize(tracker, extra = {})
+ @tracker = tracker
@extra = extra
end
+ def entity
+ @entity ||= tracker.entity
+ end
+
def group
- entity.group
+ @group ||= entity.group
+ end
+
+ def bulk_import
+ @bulk_import ||= entity.bulk_import
end
def current_user
- bulk_import.user
+ @current_user ||= bulk_import.user
end
def configuration
- bulk_import.configuration
+ @configuration ||= bulk_import.configuration
end
end
end
diff --git a/lib/bulk_imports/pipeline/extracted_data.rb b/lib/bulk_imports/pipeline/extracted_data.rb
index 685a91a4afe..c9e54b61dd3 100644
--- a/lib/bulk_imports/pipeline/extracted_data.rb
+++ b/lib/bulk_imports/pipeline/extracted_data.rb
@@ -11,11 +11,14 @@ module BulkImports
end
def has_next_page?
- @page_info['has_next_page']
+ Gitlab::Utils.to_boolean(
+ @page_info&.dig('has_next_page'),
+ default: false
+ )
end
def next_page
- @page_info['end_cursor']
+ @page_info&.dig('next_page')
end
def each(&block)
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb
index e3535e585cc..b756fba3bee 100644
--- a/lib/bulk_imports/pipeline/runner.rb
+++ b/lib/bulk_imports/pipeline/runner.rb
@@ -14,19 +14,24 @@ module BulkImports
extracted_data = extracted_data_from
- extracted_data&.each do |entry|
- transformers.each do |transformer|
- entry = run_pipeline_step(:transformer, transformer.class.name) do
- transformer.transform(context, entry)
+ if extracted_data
+ extracted_data.each do |entry|
+ transformers.each do |transformer|
+ entry = run_pipeline_step(:transformer, transformer.class.name) do
+ transformer.transform(context, entry)
+ end
end
- end
- run_pipeline_step(:loader, loader.class.name) do
- loader.load(context, entry)
+ run_pipeline_step(:loader, loader.class.name) do
+ loader.load(context, entry)
+ end
end
- end
- if respond_to?(:after_run)
+ tracker.update!(
+ has_next_page: extracted_data.has_next_page?,
+ next_page: extracted_data.next_page
+ )
+
run_pipeline_step(:after_run) do
after_run(extracted_data)
end
@@ -34,7 +39,7 @@ module BulkImports
info(message: 'Pipeline finished')
rescue MarkedAsFailedError
- log_skip
+ skip!('Skipping pipeline due to failed entity')
end
private # rubocop:disable Lint/UselessAccessModifier
@@ -46,7 +51,11 @@ module BulkImports
yield
rescue MarkedAsFailedError
- log_skip(step => class_name)
+ skip!(
+ 'Skipping pipeline due to failed entity',
+ pipeline_step: step,
+ step_class: class_name
+ )
rescue => e
log_import_failure(e, step)
@@ -61,14 +70,21 @@ module BulkImports
end
end
+ def after_run(extracted_data)
+ run if extracted_data.has_next_page?
+ end
+
def mark_as_failed
warn(message: 'Pipeline failed')
context.entity.fail_op!
+ tracker.fail_op!
end
- def log_skip(extra = {})
- info({ message: 'Skipping due to failed pipeline status' }.merge(extra))
+ def skip!(message, extra = {})
+ warn({ message: message }.merge(extra))
+
+ tracker.skip!
end
def log_import_failure(exception, step)
diff --git a/lib/constraints/admin_constrainer.rb b/lib/constraints/admin_constrainer.rb
index 59c855a1b73..2f32cc7ad91 100644
--- a/lib/constraints/admin_constrainer.rb
+++ b/lib/constraints/admin_constrainer.rb
@@ -3,7 +3,7 @@
module Constraints
class AdminConstrainer
def matches?(request)
- if Feature.enabled?(:user_mode_in_session)
+ if Gitlab::CurrentSettings.admin_mode
admin_mode_enabled?(request)
else
user_is_admin?(request)
diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb
index 40dd92befd2..aafa9b1c182 100644
--- a/lib/container_registry/config.rb
+++ b/lib/container_registry/config.rb
@@ -5,7 +5,8 @@ module ContainerRegistry
attr_reader :tag, :blob, :data
def initialize(tag, blob)
- @tag, @blob = tag, blob
+ @tag = tag
+ @blob = blob
@data = Gitlab::Json.parse(blob.data)
end
diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb
index 09c0aa66a0d..614b1b5e6c6 100644
--- a/lib/container_registry/tag.rb
+++ b/lib/container_registry/tag.rb
@@ -10,7 +10,8 @@ module ContainerRegistry
delegate :revision, :short_revision, to: :config_blob, allow_nil: true
def initialize(repository, name)
- @repository, @name = repository, name
+ @repository = repository
+ @name = name
end
def valid?
diff --git a/lib/csv_builder.rb b/lib/csv_builder.rb
index 6116009f171..43ceed9519b 100644
--- a/lib/csv_builder.rb
+++ b/lib/csv_builder.rb
@@ -14,7 +14,7 @@
# CsvBuilder.new(@posts, columns).render
#
class CsvBuilder
- DEFAULT_ORDER_BY = 'id'.freeze
+ DEFAULT_ORDER_BY = 'id'
DEFAULT_BATCH_SIZE = 1000
PREFIX_REGEX = /^[=\+\-@;]/.freeze
diff --git a/lib/declarative_policy/preferred_scope.rb b/lib/declarative_policy/preferred_scope.rb
index d653a0ec1e1..9e512086593 100644
--- a/lib/declarative_policy/preferred_scope.rb
+++ b/lib/declarative_policy/preferred_scope.rb
@@ -5,7 +5,8 @@ module DeclarativePolicy
PREFERRED_SCOPE_KEY = :"DeclarativePolicy.preferred_scope"
def with_preferred_scope(scope)
- Thread.current[PREFERRED_SCOPE_KEY], old_scope = scope, Thread.current[PREFERRED_SCOPE_KEY]
+ old_scope = Thread.current[PREFERRED_SCOPE_KEY]
+ Thread.current[PREFERRED_SCOPE_KEY] = scope
yield
ensure
Thread.current[PREFERRED_SCOPE_KEY] = old_scope
diff --git a/lib/error_tracking/sentry_client/issue.rb b/lib/error_tracking/sentry_client/issue.rb
index 513fb3daabe..bdc567bd859 100644
--- a/lib/error_tracking/sentry_client/issue.rb
+++ b/lib/error_tracking/sentry_client/issue.rb
@@ -113,9 +113,7 @@ module ErrorTracking
uri = URI(url)
uri.path.squeeze!('/')
# Remove trailing slash
- uri = uri.to_s.delete_suffix('/')
-
- uri
+ uri.to_s.delete_suffix('/')
end
def map_to_errors(issues)
diff --git a/lib/feature.rb b/lib/feature.rb
index 7c926b25587..709610b91be 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -18,10 +18,6 @@ class Feature
superclass.table_name = 'feature_gates'
end
- class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore
- # overrides methods in EE
- end
-
InvalidFeatureFlagError = Class.new(Exception) # rubocop:disable Lint/InheritException
class << self
diff --git a/lib/feature/active_support_cache_store_adapter.rb b/lib/feature/active_support_cache_store_adapter.rb
new file mode 100644
index 00000000000..ae2d623abe1
--- /dev/null
+++ b/lib/feature/active_support_cache_store_adapter.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# rubocop:disable Gitlab/NamespacedClass
+# This class was already nested this way before moving to a separate file
+class Feature
+ class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore
+ def enable(feature, gate, thing)
+ result = @adapter.enable(feature, gate, thing)
+ @cache.write(key_for(feature.key), @adapter.get(feature), @write_options)
+ result
+ end
+
+ def disable(feature, gate, thing)
+ result = @adapter.disable(feature, gate, thing)
+ @cache.write(key_for(feature.key), @adapter.get(feature), @write_options)
+ result
+ end
+
+ def remove(feature)
+ result = @adapter.remove(feature)
+ @cache.delete(FeaturesKey)
+ @cache.write(key_for(feature.key), {}, @write_options)
+ result
+ end
+ end
+end
+# rubocop:disable Gitlab/NamespacedClass
diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb
index 71908d2130f..e9868732172 100644
--- a/lib/file_size_validator.rb
+++ b/lib/file_size_validator.rb
@@ -11,7 +11,8 @@ class FileSizeValidator < ActiveModel::EachValidator
if range = (options.delete(:in) || options.delete(:within))
raise ArgumentError, ":in and :within must be a Range" unless range.is_a?(Range)
- options[:minimum], options[:maximum] = range.begin, range.end
+ options[:minimum] = range.begin
+ options[:maximum] = range.end
options[:maximum] -= 1 if range.exclude_end?
end
diff --git a/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb
new file mode 100644
index 00000000000..d826c51a73d
--- /dev/null
+++ b/lib/generators/gitlab/usage_metric_definition/redis_hll_generator.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+require_relative '../usage_metric_definition_generator'
+
+module Gitlab
+ module UsageMetricDefinition
+ class RedisHllGenerator < Rails::Generators::Base
+ desc 'Generates a metric definition .yml file with defaults for Redis HLL.'
+
+ argument :category, type: :string, desc: "Category name"
+ argument :event, type: :string, desc: "Event name"
+
+ def create_metrics
+ Gitlab::UsageMetricDefinitionGenerator.start(["#{key_path}_weekly", '--dir', '7d'])
+ Gitlab::UsageMetricDefinitionGenerator.start(["#{key_path}_monthly", '--dir', '28d'])
+ end
+
+ private
+
+ def key_path
+ "redis_hll_counters.#{category}.#{event}"
+ end
+ end
+ end
+end
diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb
index 7a01050ed0c..cadc319a212 100644
--- a/lib/generators/gitlab/usage_metric_definition_generator.rb
+++ b/lib/generators/gitlab/usage_metric_definition_generator.rb
@@ -5,6 +5,11 @@ require 'rails/generators'
module Gitlab
class UsageMetricDefinitionGenerator < Rails::Generators::Base
Directory = Struct.new(:name, :time_frame, :value_type) do
+ def initialize(...)
+ super
+ freeze
+ end
+
def match?(str)
(name == str || time_frame == str) && str != 'none'
end
@@ -60,9 +65,7 @@ module Gitlab
private
def metric_name_suggestion
- return unless Feature.enabled?(:product_intelligence_metrics_names_suggestions, default_enabled: :yaml)
-
- "\nname: #{Usage::Metrics::NamesSuggestions::Generator.generate(key_path)}"
+ "\nname: \"#{Usage::Metrics::NamesSuggestions::Generator.generate(key_path)}\""
end
def file_path
@@ -101,7 +104,7 @@ module Gitlab
end
def metric_definitions
- @definitions ||= Gitlab::Usage::MetricDefinition.definitions
+ @definitions ||= Gitlab::Usage::MetricDefinition.definitions(skip_validation: true)
end
def metric_definition_exists?
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index 11ecfb951aa..ddf08c8dc20 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -108,10 +108,21 @@ module Gitlab
!%w[true 1].include?(ENV['FOSS_ONLY'].to_s)
end
+ def self.jh?
+ @is_jh ||=
+ ee? &&
+ root.join('jh').exist? &&
+ !%w[true 1].include?(ENV['EE_ONLY'].to_s)
+ end
+
def self.ee
yield if ee?
end
+ def self.jh
+ yield if jh?
+ end
+
def self.http_proxy_env?
HTTP_PROXY_ENV_VARS.any? { |name| ENV[name] }
end
diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb
index c8b8d6c259d..786c5bf675b 100644
--- a/lib/gitlab/alert_management/payload/base.rb
+++ b/lib/gitlab/alert_management/payload/base.rb
@@ -132,7 +132,7 @@ module Gitlab
EnvironmentsFinder
.new(project, nil, { name: environment_name })
- .find
+ .execute
.first
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
index 178ebe0d4d4..b4752ed9e5b 100644
--- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -31,14 +31,34 @@ module Gitlab
@params = params
@sort = params[:sort] || :end_event
@direction = params[:direction] || :desc
+ @page = params[:page] || 1
+ @per_page = MAX_RECORDS
end
+ # rubocop: disable CodeReuse/ActiveRecord
def serialized_records
strong_memoize(:serialized_records) do
# special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records
if default_test_stage? || default_staging_stage?
+ ci_build_join = mr_metrics_table
+ .join(build_table)
+ .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id]))
+ .join_sources
+
+ records = ordered_and_limited_query
+ .joins(ci_build_join)
+ .select(build_table[:id], *time_columns)
+
+ yield records if block_given?
+ ci_build_records = preload_ci_build_associations(records)
+
AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] })
else
+ records = ordered_and_limited_query.select(*columns, *time_columns)
+
+ yield records if block_given?
+ records = preload_associations(records)
+
records.map do |record|
project = record.project
attributes = record.attributes.merge({
@@ -51,10 +71,11 @@ module Gitlab
end
end
end
+ # rubocop: enable CodeReuse/ActiveRecord
private
- attr_reader :stage, :query, :params, :sort, :direction
+ attr_reader :stage, :query, :params, :sort, :direction, :page, :per_page
def columns
MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name|
@@ -74,41 +95,32 @@ module Gitlab
MAPPINGS.fetch(subject_class).fetch(:serializer_class).new
end
- # Loading Ci::Build records instead of MergeRequest records
# rubocop: disable CodeReuse/ActiveRecord
- def ci_build_records
- ci_build_join = mr_metrics_table
- .join(build_table)
- .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id]))
- .join_sources
-
- q = ordered_and_limited_query
- .joins(ci_build_join)
- .select(build_table[:id], *time_columns)
-
- results = execute_query(q).to_a
+ def preload_ci_build_associations(records)
+ results = records.map(&:attributes)
Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] }))
end
+ # rubocop: enable CodeReuse/ActiveRecord
def ordered_and_limited_query
- order_by(query, sort, direction, columns).limit(MAX_RECORDS)
+ strong_memoize(:ordered_and_limited_query) do
+ order_by(query, sort, direction, columns).page(page).per(per_page).without_count
+ end
end
- def records
- results = ordered_and_limited_query
- .select(*columns, *time_columns)
-
+ # rubocop: disable CodeReuse/ActiveRecord
+ def preload_associations(records)
# using preloader instead of includes to avoid AR generating a large column list
ActiveRecord::Associations::Preloader.new.preload(
- results,
+ records,
MAPPINGS.fetch(subject_class).fetch(:includes_for_query)
)
- results
+ records
end
- # rubocop: enable CodeReuse/ActiveRecord
+ # rubocop: enable CodeReuse/ActiveRecord
def time_columns
[
stage.start_event.timestamp_projection.as('start_event_timestamp'),
diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb
index e367d33d743..723486231b1 100644
--- a/lib/gitlab/analytics/unique_visits.rb
+++ b/lib/gitlab/analytics/unique_visits.rb
@@ -3,8 +3,8 @@
module Gitlab
module Analytics
class UniqueVisits
- def track_visit(visitor_id, target_id, time = Time.zone.now)
- Gitlab::UsageDataCounters::HLLRedisCounter.track_event(target_id, values: visitor_id, time: time)
+ def track_visit(*args, **kwargs)
+ Gitlab::UsageDataCounters::HLLRedisCounter.track_event(*args, **kwargs)
end
# Returns number of unique visitors for given targets in given time frame
diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb
index a75da3a682b..ceda82cb6f6 100644
--- a/lib/gitlab/application_context.rb
+++ b/lib/gitlab/application_context.rb
@@ -8,6 +8,9 @@ module Gitlab
Attribute = Struct.new(:name, :type)
+ LOG_KEY = Labkit::Context::LOG_KEY
+ KNOWN_KEYS = Labkit::Context::KNOWN_KEYS
+
APPLICATION_ATTRIBUTES = [
Attribute.new(:project, Project),
Attribute.new(:namespace, Namespace),
@@ -24,6 +27,10 @@ module Gitlab
application_context.use(&block)
end
+ def self.with_raw_context(attributes = {}, &block)
+ Labkit::Context.with_context(attributes, &block)
+ end
+
def self.push(args)
application_context = new(**args)
Labkit::Context.push(application_context.to_lazy_hash)
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index 4c6254c9e69..6f6ac79c16b 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -24,9 +24,9 @@ module Gitlab
PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN'
PRIVATE_TOKEN_PARAM = :private_token
- JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze
+ JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'
JOB_TOKEN_PARAM = :job_token
- DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN'.freeze
+ DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN'
RUNNER_TOKEN_PARAM = :token
RUNNER_JOB_TOKEN_PARAM = :token
diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb
index b7bb61f0677..7f85d3b1cd3 100644
--- a/lib/gitlab/auth/ldap/adapter.rb
+++ b/lib/gitlab/auth/ldap/adapter.rb
@@ -5,7 +5,7 @@ module Gitlab
module Ldap
class Adapter
SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze
- MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze
+ MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size
attr_reader :provider, :ldap
diff --git a/lib/gitlab/auth/saml/origin_validator.rb b/lib/gitlab/auth/saml/origin_validator.rb
index 4ecc688888f..ff0d25314f7 100644
--- a/lib/gitlab/auth/saml/origin_validator.rb
+++ b/lib/gitlab/auth/saml/origin_validator.rb
@@ -4,7 +4,7 @@ module Gitlab
module Auth
module Saml
class OriginValidator
- AUTH_REQUEST_SESSION_KEY = "last_authn_request_id".freeze
+ AUTH_REQUEST_SESSION_KEY = "last_authn_request_id"
def initialize(session)
@session = session || {}
diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb
index 553571d5d00..6d1df95c66d 100644
--- a/lib/gitlab/background_migration/backfill_design_internal_ids.rb
+++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb
@@ -97,13 +97,13 @@ module Gitlab
ActiveRecord::Base.connection.execute <<~SQL
WITH
- starting_iids(project_id, iid) as (
+ starting_iids(project_id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}(
SELECT project_id, MAX(COALESCE(iid, 0))
FROM #{table}
WHERE project_id BETWEEN #{start_id} AND #{end_id}
GROUP BY project_id
),
- with_calculated_iid(id, iid) as (
+ with_calculated_iid(id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}(
SELECT design.id,
init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC)
FROM #{table} as design, starting_iids as init
diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb
index 7484027a0fa..030dfd2d99b 100644
--- a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb
+++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb
@@ -8,7 +8,7 @@ module Gitlab
updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id)
Project.connection.execute <<-SQL
- WITH repository_storage_cte as (
+ WITH repository_storage_cte as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
#{updated_repository_storages.to_sql}
)
UPDATE projects
diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb
index 60682bd2ec1..b89ea7dc250 100644
--- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb
+++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb
@@ -34,12 +34,18 @@ module Gitlab
parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
- sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}")
+ batch_metrics.time_operation(:update_all) do
+ sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}")
+ end
sleep(PAUSE_SECONDS)
end
end
+ def batch_metrics
+ @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
+ end
+
private
def connection
diff --git a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb
index 6014ccc12eb..691bdb457d7 100644
--- a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb
+++ b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb
@@ -8,7 +8,7 @@ module Gitlab
def perform(start_id, stop_id)
ActiveRecord::Base.connection.execute <<~SQL
- WITH merge_requests_batch AS (
+ WITH merge_requests_batch AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
SELECT id, target_project_id
FROM merge_requests WHERE id BETWEEN #{Integer(start_id)} AND #{Integer(stop_id)}
)
diff --git a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb
index 68665db522e..83c01afa432 100644
--- a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb
+++ b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb
@@ -22,7 +22,7 @@ module Gitlab
def sql(from_id, to_id)
<<~SQL
- WITH created_records AS (
+ WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
INSERT INTO project_features (
project_id,
merge_requests_access_level,
diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb
index e750b8ca374..b8e4562b3bf 100644
--- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb
+++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb
@@ -136,7 +136,7 @@ module Gitlab
# there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT
def create_sql(from_id, to_id)
<<~SQL
- WITH created_records AS (
+ WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
INSERT INTO services (project_id, #{DEFAULTS.keys.map { |key| %("#{key}")}.join(',')}, created_at, updated_at)
#{select_insert_values_sql(from_id, to_id)}
RETURNING *
@@ -149,7 +149,7 @@ module Gitlab
# there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT
def update_sql(from_id, to_id)
<<~SQL
- WITH updated_records AS (
+ WITH updated_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
UPDATE services SET active = TRUE
WHERE services.project_id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)} AND services.properties = '{}' AND services.type = '#{Migratable::PrometheusService.type}'
AND #{group_cluster_condition(from_id, to_id)} AND services.active = FALSE
diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb
index d767cbfd8f5..cd5b4ab103d 100644
--- a/lib/gitlab/background_migration/fix_user_namespace_names.rb
+++ b/lib/gitlab/background_migration/fix_user_namespace_names.rb
@@ -14,7 +14,7 @@ module Gitlab
def fix_namespace_names(from_id, to_id)
ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES
- WITH namespaces_to_update AS (
+ WITH namespaces_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
SELECT
namespaces.id,
users.name AS correct_name
@@ -39,7 +39,7 @@ module Gitlab
def fix_namespace_route_names(from_id, to_id)
ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE
- WITH routes_to_update AS (
+ WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
SELECT
routes.id,
users.name AS correct_name
diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb
index 6b99685fd68..e534f2449aa 100644
--- a/lib/gitlab/background_migration/fix_user_project_route_names.rb
+++ b/lib/gitlab/background_migration/fix_user_project_route_names.rb
@@ -8,7 +8,7 @@ module Gitlab
class FixUserProjectRouteNames
def perform(from_id, to_id)
ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE
- WITH routes_to_update AS (
+ WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
SELECT
routes.id,
users.name || ' / ' || projects.name AS correct_name
diff --git a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb
new file mode 100644
index 00000000000..b7a912da060
--- /dev/null
+++ b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # migrates pages from legacy storage to zip format
+ # we intentionally use application code here because
+ # it has a lot of dependencies including models, carrierwave uploaders and service objects
+ # and copying all or part of this code in the background migration doesn't add much value
+ # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578 for discussion
+ class MigratePagesToZipStorage
+ def perform(start_id, stop_id)
+ ::Pages::MigrateFromLegacyStorageService.new(Gitlab::AppLogger,
+ ignore_invalid_entries: false,
+ mark_projects_as_not_deployed: false)
+ .execute_for_batch(start_id..stop_id)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb
index 4eaef26c9c6..9ecf53317d0 100644
--- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb
+++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb
@@ -6,7 +6,7 @@ module Gitlab
# project_features.container_registry_access_level for the projects within
# the given range of ids.
class MoveContainerRegistryEnabledToProjectFeature
- MAX_BATCH_SIZE = 1_000
+ MAX_BATCH_SIZE = 300
module Migratable
# Migration model namespace isolated from application code.
diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb
index 78140b768fc..28ff2070209 100644
--- a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb
+++ b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb
@@ -8,21 +8,23 @@ module Gitlab
class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation
self.table_name = 'project_settings'
- UPSERT_SQL = <<~SQL
- WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS (
- SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids})
- )
- INSERT INTO project_settings
- (project_id, has_vulnerabilities, created_at, updated_at)
- (SELECT * FROM upsert_data)
- ON CONFLICT (project_id)
- DO UPDATE SET
- has_vulnerabilities = true,
- updated_at = EXCLUDED.updated_at
- SQL
-
def self.upsert_for(project_ids)
- connection.execute(UPSERT_SQL % { project_ids: project_ids.join(', ') })
+ connection.execute(upsert_sql % { project_ids: project_ids.join(', ') })
+ end
+
+ def self.upsert_sql
+ <<~SQL
+ WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
+ SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids})
+ )
+ INSERT INTO project_settings
+ (project_id, has_vulnerabilities, created_at, updated_at)
+ (SELECT * FROM upsert_data)
+ ON CONFLICT (project_id)
+ DO UPDATE SET
+ has_vulnerabilities = true,
+ updated_at = EXCLUDED.updated_at
+ SQL
end
end
diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb
index eb4bc0aaf28..28cc4a5e3fa 100644
--- a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb
+++ b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb
@@ -11,7 +11,7 @@ module Gitlab
MergeRequest
.where(merge_request_assignees_not_exists_clause)
.where(id: from_id..to_id)
- .where('assignee_id IS NOT NULL')
+ .where.not(assignee_id: nil)
.select(:id, :assignee_id)
.to_sql
diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
index 7b18e617c81..888a12f2330 100644
--- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
+++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb
@@ -32,7 +32,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
}.freeze
NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze
- PACK_PATTERN = "NnnnnN".freeze
+ PACK_PATTERN = "NnnnnN"
def self.call(value)
Digest::UUID.uuid_v5(namespace_id, value)
diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb
new file mode 100644
index 00000000000..5930d65bc2c
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Lib
+ module Banzai
+ module ReferenceParser
+ # isolated Banzai::ReferenceParser::MentionedGroupParser
+ class IsolatedMentionedProjectParser < ::Banzai::ReferenceParser::MentionedProjectParser
+ extend ::Gitlab::Utils::Override
+
+ self.reference_type = :user
+
+ override :references_relation
+ def references_relation
+ ::Gitlab::BackgroundMigration::UserMentions::Models::Project
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb
new file mode 100644
index 00000000000..f5f98517433
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Lib
+ module Banzai
+ module ReferenceParser
+ # isolated Banzai::ReferenceParser::MentionedGroupParser
+ class IsolatedMentionedUserParser < ::Banzai::ReferenceParser::MentionedUserParser
+ extend ::Gitlab::Utils::Override
+
+ self.reference_type = :user
+
+ override :references_relation
+ def references_relation
+ ::Gitlab::BackgroundMigration::UserMentions::Models::User
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
index 1d3a3af81a1..8610129533d 100644
--- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
+++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb
@@ -7,7 +7,7 @@ module Gitlab
module Gitlab
# Extract possible GFM references from an arbitrary String for further processing.
class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor
- REFERABLES = %i(isolated_mentioned_group).freeze
+ REFERABLES = %i(isolated_mentioned_group isolated_mentioned_user isolated_mentioned_project).freeze
REFERABLES.each do |type|
define_method("#{type}s") do
diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb
new file mode 100644
index 00000000000..0334ea1dd08
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Lib
+ module Gitlab
+ # Gitlab::IsolatedVisibilityLevel module
+ #
+ # Define allowed public modes that can be used for
+ # GitLab projects to determine project public mode
+ #
+ module IsolatedVisibilityLevel
+ extend ::ActiveSupport::Concern
+
+ included do
+ scope :public_to_user, -> (user = nil) do
+ where(visibility_level: IsolatedVisibilityLevel.levels_for_user(user))
+ end
+ end
+
+ PRIVATE = 0 unless const_defined?(:PRIVATE)
+ INTERNAL = 10 unless const_defined?(:INTERNAL)
+ PUBLIC = 20 unless const_defined?(:PUBLIC)
+
+ class << self
+ def levels_for_user(user = nil)
+ return [PUBLIC] unless user
+
+ if user.can_read_all_resources?
+ [PRIVATE, INTERNAL, PUBLIC]
+ elsif user.external?
+ [PUBLIC]
+ else
+ [INTERNAL, PUBLIC]
+ end
+ end
+ end
+
+ def private?
+ visibility_level_value == PRIVATE
+ end
+
+ def internal?
+ visibility_level_value == INTERNAL
+ end
+
+ def public?
+ visibility_level_value == PUBLIC
+ end
+
+ def visibility_level_value
+ self[visibility_level_field]
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb
index bdb4d6c7d48..f4cc96c8bc0 100644
--- a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb
@@ -7,6 +7,7 @@ module Gitlab
module Models
class CommitUserMention < ActiveRecord::Base
self.table_name = 'commit_user_mentions'
+ self.inheritance_column = :_type_disabled
def self.resource_foreign_key
:commit_id
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb
new file mode 100644
index 00000000000..ba6b783f9f1
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Models
+ module Concerns
+ # isolated FeatureGate module
+ module IsolatedFeatureGate
+ def flipper_id
+ return if new_record?
+
+ "#{self.class.name}:#{id}"
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
index be9c0ad2b3a..f684f789ea9 100644
--- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb
@@ -70,8 +70,8 @@ module Gitlab
def build_mention_values(resource_foreign_key)
refs = all_references(author)
- mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id))
- mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id))
+ mentioned_users_ids = array_to_sql(refs.isolated_mentioned_users.pluck(:id))
+ mentioned_projects_ids = array_to_sql(refs.isolated_mentioned_projects.pluck(:id))
mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id))
return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank?
diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
index 5cadfa45b5b..75759ed0111 100644
--- a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb
@@ -6,7 +6,7 @@ module Gitlab
module Models
module Concerns
module Namespace
- # extracted methods for recursive traversing of namespace hierarchy
+ # isolate recursive traversal code for namespace hierarchy
module RecursiveTraversal
extend ActiveSupport::Concern
diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb
index bdb90b5d2b9..d010d68600d 100644
--- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb
@@ -10,6 +10,9 @@ module Gitlab
include EachBatch
include Concerns::MentionableMigrationMethods
+ self.table_name = 'design_management_designs'
+ self.inheritance_column = :_type_disabled
+
def self.user_mention_model
Gitlab::BackgroundMigration::UserMentions::Models::DesignUserMention
end
diff --git a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb
index 68205ecd3c2..eb00f6cfa3f 100644
--- a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb
@@ -7,6 +7,7 @@ module Gitlab
module Models
class DesignUserMention < ActiveRecord::Base
self.table_name = 'design_user_mentions'
+ self.inheritance_column = :_type_disabled
def self.resource_foreign_key
:design_id
diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb
index 61d9244a4c9..cfd9a4faa9b 100644
--- a/lib/gitlab/background_migration/user_mentions/models/epic.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb
@@ -17,10 +17,10 @@ module Gitlab
cache_markdown_field :description, issuable_state_filter_enabled: true
self.table_name = 'epics'
+ self.inheritance_column = :_type_disabled
- belongs_to :author, class_name: "User"
- belongs_to :project
- belongs_to :group
+ belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User"
+ belongs_to :group, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group"
def self.user_mention_model
Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention
diff --git a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb
index 4e3ce9bf3a7..579e4d99612 100644
--- a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb
@@ -7,6 +7,7 @@ module Gitlab
module Models
class EpicUserMention < ActiveRecord::Base
self.table_name = 'epic_user_mentions'
+ self.inheritance_column = :_type_disabled
def self.resource_foreign_key
:epic_id
diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb
index bc04172b9a2..a8b4b59b06c 100644
--- a/lib/gitlab/background_migration/user_mentions/models/group.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/group.rb
@@ -7,6 +7,8 @@ module Gitlab
# isolated Group model
class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace
self.store_full_sti_class = false
+ self.inheritance_column = :_type_disabled
+
has_one :saml_provider
def self.declarative_policy_class
diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb
index 6b52afea17c..13addcc3c55 100644
--- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb
@@ -17,10 +17,11 @@ module Gitlab
cache_markdown_field :description, issuable_state_filter_enabled: true
self.table_name = 'merge_requests'
+ self.inheritance_column = :_type_disabled
- belongs_to :author, class_name: "User"
- belongs_to :target_project, class_name: "Project"
- belongs_to :source_project, class_name: "Project"
+ belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User"
+ belongs_to :target_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project"
+ belongs_to :source_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project"
alias_attribute :project, :target_project
diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb
index e9b85e9cb8c..4a85892d7b8 100644
--- a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb
@@ -7,6 +7,7 @@ module Gitlab
module Models
class MergeRequestUserMention < ActiveRecord::Base
self.table_name = 'merge_request_user_mentions'
+ self.inheritance_column = :_type_disabled
def self.resource_foreign_key
:merge_request_id
diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
index 8fa0db5fd4b..a2b50c41f4a 100644
--- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb
@@ -5,9 +5,11 @@ module Gitlab
module UserMentions
module Models
# isolated Namespace model
- class Namespace < ApplicationRecord
- include FeatureGate
- include ::Gitlab::VisibilityLevel
+ class Namespace < ActiveRecord::Base
+ self.inheritance_column = :_type_disabled
+
+ include Concerns::IsolatedFeatureGate
+ include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel
include ::Gitlab::Utils::StrongMemoize
include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal
@@ -21,8 +23,13 @@ module Gitlab
parent_id.present? || parent.present?
end
+ # Deprecated, use #licensed_feature_available? instead. Remove once Namespace#feature_available? isn't used anymore.
+ def feature_available?(feature)
+ licensed_feature_available?(feature)
+ end
+
# Overridden in EE::Namespace
- def feature_available?(_feature)
+ def licensed_feature_available?(_feature)
false
end
end
diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb
index a3224c8c456..7da933c7b11 100644
--- a/lib/gitlab/background_migration/user_mentions/models/note.rb
+++ b/lib/gitlab/background_migration/user_mentions/models/note.rb
@@ -16,9 +16,9 @@ module Gitlab
attr_mentionable :note, pipeline: :note
cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true
- belongs_to :author, class_name: "User"
+ belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User"
belongs_to :noteable, polymorphic: true
- belongs_to :project
+ belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project"
def for_personal_snippet?
noteable && noteable.class.name == 'PersonalSnippet'
diff --git a/lib/gitlab/background_migration/user_mentions/models/project.rb b/lib/gitlab/background_migration/user_mentions/models/project.rb
new file mode 100644
index 00000000000..4e02bf97d12
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/project.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Models
+ # isolated Namespace model
+ class Project < ActiveRecord::Base
+ include Concerns::IsolatedFeatureGate
+ include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel
+
+ self.table_name = 'projects'
+ self.inheritance_column = :_type_disabled
+
+ belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id', class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group"
+ belongs_to :namespace, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace"
+ alias_method :parent, :namespace
+
+ # Returns a collection of projects that is either public or visible to the
+ # logged in user.
+ def self.public_or_visible_to_user(user = nil, min_access_level = nil)
+ min_access_level = nil if user&.can_read_all_resources?
+
+ return public_to_user unless user
+
+ if user.is_a?(::Gitlab::BackgroundMigration::UserMentions::Models::User)
+ where('EXISTS (?) OR projects.visibility_level IN (?)',
+ user.authorizations_for_projects(min_access_level: min_access_level),
+ levels_for_user(user))
+ end
+ end
+
+ def grafana_integration
+ nil
+ end
+
+ def default_issues_tracker?
+ true # we do not care of the issue tracker type(internal or external) when parsing mentions
+ end
+
+ def visibility_level_field
+ :visibility_level
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/user_mentions/models/user.rb b/lib/gitlab/background_migration/user_mentions/models/user.rb
new file mode 100644
index 00000000000..a30220b6934
--- /dev/null
+++ b/lib/gitlab/background_migration/user_mentions/models/user.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ module UserMentions
+ module Models
+ # isolated Namespace model
+ class User < ActiveRecord::Base
+ include Concerns::IsolatedFeatureGate
+
+ self.table_name = 'users'
+ self.inheritance_column = :_type_disabled
+
+ has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
+
+ def authorizations_for_projects(min_access_level: nil, related_project_column: 'projects.id')
+ authorizations = project_authorizations
+ .select(1)
+ .where("project_authorizations.project_id = #{related_project_column}")
+
+ return authorizations unless min_access_level.present?
+
+ authorizations.where('project_authorizations.access_level >= ?', min_access_level)
+ end
+
+ def can_read_all_resources?
+ can?(:read_all_resources)
+ end
+
+ def can?(action, subject = :global)
+ Ability.allowed?(self, action, subject)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb
index baacc912df3..665ad7abcbb 100644
--- a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb
+++ b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb
@@ -27,7 +27,7 @@ module Gitlab
joins(:user)
.merge(UserModel.active)
.where(id: (start_id..stop_id))
- .where('emails.confirmed_at IS NOT NULL')
+ .where.not('emails.confirmed_at' => nil)
.where('emails.confirmed_at = users.confirmed_at')
.where('emails.email <> users.email')
.where('NOT EXISTS (SELECT 1 FROM user_synced_attributes_metadata WHERE user_id=users.id AND email_synced IS true)')
@@ -57,7 +57,7 @@ module Gitlab
def update_email_records(start_id, stop_id)
EmailModel.connection.execute <<-SQL
- WITH md5_strings as (
+ WITH md5_strings as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
#{email_query_for_update(start_id, stop_id).to_sql}
)
UPDATE #{EmailModel.connection.quote_table_name(EmailModel.table_name)}
diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb
index e18f1320ea4..62fc8cd048e 100644
--- a/lib/gitlab/batch_pop_queueing.rb
+++ b/lib/gitlab/batch_pop_queueing.rb
@@ -46,7 +46,8 @@ module Gitlab
def initialize(namespace, queue_id)
raise ArgumentError if namespace.empty? || queue_id.empty?
- @namespace, @queue_id = namespace, queue_id
+ @namespace = namespace
+ @queue_id = queue_id
end
##
diff --git a/lib/gitlab/bullet.rb b/lib/gitlab/bullet.rb
new file mode 100644
index 00000000000..f5f8a316855
--- /dev/null
+++ b/lib/gitlab/bullet.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Bullet
+ extend self
+
+ def enabled?
+ Gitlab::Utils.to_boolean(ENV['ENABLE_BULLET'], default: false)
+ end
+ alias_method :extra_logging_enabled?, :enabled?
+
+ def configure_bullet?
+ defined?(::Bullet) && (enabled? || Rails.env.development?)
+ end
+ end
+end
diff --git a/lib/gitlab/bullet/exclusions.rb b/lib/gitlab/bullet/exclusions.rb
new file mode 100644
index 00000000000..f897ff492d9
--- /dev/null
+++ b/lib/gitlab/bullet/exclusions.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Bullet
+ class Exclusions
+ def initialize(config_file = Gitlab.root.join('config/bullet.yml'))
+ @config_file = config_file
+ end
+
+ def execute
+ exclusions.map { |v| v['exclude'] }
+ end
+
+ def validate_paths!
+ exclusions.each do |properties|
+ next unless properties['path_with_method']
+
+ file = properties['exclude'].first
+
+ raise "Bullet: File used by #{config_file} doesn't exist, validate the #{file} exclusion!" unless File.exist?(file)
+ end
+ end
+
+ private
+
+ attr_reader :config_file
+
+ def exclusions
+ @exclusions ||= if File.exist?(config_file)
+ YAML.load_file(config_file)['exclusions']&.values || []
+ else
+ []
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb
index d981f263c5e..9e958eb52fb 100644
--- a/lib/gitlab/cache/ci/project_pipeline_status.rb
+++ b/lib/gitlab/cache/ci/project_pipeline_status.rb
@@ -69,7 +69,9 @@ module Gitlab
def load_from_project
return unless commit
- self.sha, self.status, self.ref = commit.sha, commit.status, project.default_branch
+ self.sha = commit.sha
+ self.status = commit.status
+ self.ref = project.default_branch
end
# We only cache the status for the HEAD commit of a project
diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb
index 105050936ce..be8009750da 100644
--- a/lib/gitlab/changelog/config.rb
+++ b/lib/gitlab/changelog/config.rb
@@ -17,7 +17,24 @@ module Gitlab
# The default template to use for generating release sections.
DEFAULT_TEMPLATE = File.read(File.join(__dir__, 'template.tpl'))
- attr_accessor :date_format, :categories, :template
+ # The regex to use for extracting the version from a Git tag.
+ #
+ # This regex is based on the official semantic versioning regex (as found
+ # on https://semver.org/), with the addition of allowing a "v" at the
+ # start of a tag name.
+ #
+ # We default to a strict regex as we simply don't know what kind of data
+ # users put in their tags. As such, using simpler patterns (e.g. just
+ # `\d+` for the major version) could lead to unexpected results.
+ #
+ # We use a String here as `Gitlab::UntrustedRegexp` is a mutable object.
+ DEFAULT_TAG_REGEX = '^v?(?P<major>0|[1-9]\d*)' \
+ '\.(?P<minor>0|[1-9]\d*)' \
+ '\.(?P<patch>0|[1-9]\d*)' \
+ '(?:-(?P<pre>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))' \
+ '?(?:\+(?P<meta>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$'
+
+ attr_accessor :date_format, :categories, :template, :tag_regex
def self.from_git(project)
if (yaml = project.repository.changelog_config)
@@ -46,6 +63,10 @@ module Gitlab
end
end
+ if (regex = hash['tag_regex'])
+ config.tag_regex = regex
+ end
+
config
end
@@ -54,6 +75,7 @@ module Gitlab
@date_format = DEFAULT_DATE_FORMAT
@template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE)
@categories = {}
+ @tag_regex = DEFAULT_TAG_REGEX
end
def contributor?(user)
diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb
index 029a9210dc9..495f12882e5 100644
--- a/lib/gitlab/chaos.rb
+++ b/lib/gitlab/chaos.rb
@@ -43,9 +43,9 @@ module Gitlab
Kernel.sleep(duration_s)
end
- # Kill will send a SIGKILL signal to the current process
- def self.kill
- Process.kill("KILL", Process.pid)
+ # Kill will send the given signal to the current process.
+ def self.kill(signal)
+ Process.kill(signal, Process.pid)
end
def self.run_gc
diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb
index c5afb16ab1a..88d624503df 100644
--- a/lib/gitlab/ci/build/artifacts/metadata.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata.rb
@@ -17,7 +17,9 @@ module Gitlab
attr_reader :stream, :path, :full_version
def initialize(stream, path, **opts)
- @stream, @path, @opts = stream, path, opts
+ @stream = stream
+ @path = path
+ @opts = opts
@full_version = read_version
end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index d3f030c3b36..23b0c93a3ee 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -17,12 +17,14 @@ module Gitlab
Config::Yaml::Tags::TagError
].freeze
- attr_reader :root
+ attr_reader :root, :context, :ref
- def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil)
+ def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil)
@context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline)
@context.set_deadline(TIMEOUT_SECONDS)
+ @ref = ref
+
@config = expand_config(config)
@root = Entry::Root.new(@config)
@@ -94,9 +96,7 @@ module Gitlab
initial_config = Config::External::Processor.new(initial_config, @context).perform
initial_config = Config::Extendable.new(initial_config).to_hash
initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash
- initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash
-
- initial_config
+ Config::EdgeStagesInjector.new(initial_config).to_hash
end
def find_sha(project)
diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb
index cf599ce5294..f9688c500d2 100644
--- a/lib/gitlab/ci/config/entry/cache.rb
+++ b/lib/gitlab/ci/config/entry/cache.rb
@@ -8,8 +8,8 @@ module Gitlab
# Entry that represents a cache configuration
#
class Cache < ::Gitlab::Config::Entry::Simplifiable
- strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job) }
- strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job) }
+ strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) }
+ strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job, default_enabled: :yaml) }
class Caches < ::Gitlab::Config::Entry::ComposableArray
include ::Gitlab::Config::Entry::Validatable
@@ -17,8 +17,6 @@ module Gitlab
MULTIPLE_CACHE_LIMIT = 4
validations do
- validates :config, presence: true
-
validate do
unless config.is_a?(Hash) || config.is_a?(Array)
errors.add(:config, 'can only be a Hash or an Array')
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index 9584d19bdec..947b6787aa0 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -124,7 +124,9 @@ module Gitlab
stage: stage_value,
extends: extends,
rules: rules_value,
- variables: root_and_job_variables_value,
+ variables: root_and_job_variables_value, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+ job_variables: job_variables,
+ root_variables_inheritance: root_variables_inheritance,
only: only_value,
except: except_value,
resource_group: resource_group }.compact
@@ -139,6 +141,14 @@ module Gitlab
root_variables.merge(variables_value.to_h)
end
+ def job_variables
+ variables_value.to_h
+ end
+
+ def root_variables_inheritance
+ inherit_entry&.variables_entry&.value
+ end
+
def manual_action?
self.when == 'manual'
end
diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb
index aa34cfb3acc..e869e0bbb31 100644
--- a/lib/gitlab/ci/config/entry/product/variables.rb
+++ b/lib/gitlab/ci/config/entry/product/variables.rb
@@ -25,8 +25,7 @@ module Gitlab
def value
@config
- .map { |key, value| [key.to_s, Array(value).map(&:to_s)] }
- .to_h
+ .to_h { |key, value| [key.to_s, Array(value).map(&:to_s)] }
end
end
end
diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb
index dc164d752be..efb469ee32a 100644
--- a/lib/gitlab/ci/config/entry/variables.rb
+++ b/lib/gitlab/ci/config/entry/variables.rb
@@ -18,7 +18,7 @@ module Gitlab
end
def value
- Hash[@config.map { |key, value| [key.to_s, expand_value(value)[:value]] }]
+ @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] }
end
def self.default(**)
@@ -26,7 +26,7 @@ module Gitlab
end
def value_with_data
- Hash[@config.map { |key, value| [key.to_s, expand_value(value)] }]
+ @config.to_h { |key, value| [key.to_s, expand_value(value)] }
end
def use_value_data?
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index b85b7a9edeb..3216d4eaac4 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -34,6 +34,7 @@ module Gitlab
.compact
.map(&method(:normalize_location))
.flat_map(&method(:expand_project_files))
+ .flat_map(&method(:expand_wildcard_paths))
.map(&method(:expand_variables))
.each(&method(:verify_duplicates!))
.map(&method(:select_first_matching))
@@ -63,6 +64,17 @@ module Gitlab
end
end
+ def expand_wildcard_paths(location)
+ return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml)
+
+ # We only support local files for wildcard paths
+ return location unless location[:local] && location[:local].include?('*')
+
+ context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path|
+ { local: path }
+ end
+ end
+
def normalize_location_string(location)
if ::Gitlab::UrlSanitizer.valid?(location)
{ remote: location }
diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
index 5a23836d8a0..5cabbc86d3e 100644
--- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
+++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
@@ -43,9 +43,10 @@ module Gitlab
{
name: name,
instance: instance,
- variables: variables,
+ variables: variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+ job_variables: variables,
parallel: { total: total }
- }
+ }.compact
end
def name
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index c811ef211d6..12e182b38fc 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -10,10 +10,6 @@ module Gitlab
::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true)
end
- def self.instance_variables_ui_enabled?
- ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true)
- end
-
def self.pipeline_latest?
::Feature.enabled?(:ci_pipeline_latest, default_enabled: true)
end
@@ -60,16 +56,12 @@ module Gitlab
::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false)
end
- def self.display_codequality_backend_comparison?(project)
- ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml)
- end
-
def self.multiple_cache_per_job?
::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml)
end
- def self.ci_commit_pipeline_mini_graph_vue_enabled?(project)
- ::Feature.enabled?(:ci_commit_pipeline_mini_graph_vue, project, default_enabled: :yaml)
+ def self.gldropdown_tags_enabled?
+ ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml)
end
end
end
diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb
index af06e124736..a6ae249fa58 100644
--- a/lib/gitlab/ci/jwt.rb
+++ b/lib/gitlab/ci/jwt.rb
@@ -72,16 +72,16 @@ module Gitlab
def key
@key ||= begin
- key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true)
- Gitlab::CurrentSettings.ci_jwt_signing_key
- else
- Rails.application.secrets.openid_connect_signing_key
- end
+ key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true)
+ Gitlab::CurrentSettings.ci_jwt_signing_key
+ else
+ Rails.application.secrets.openid_connect_signing_key
+ end
- raise NoSigningKeyError unless key_data
+ raise NoSigningKeyError unless key_data
- OpenSSL::PKey::RSA.new(key_data)
- end
+ OpenSSL::PKey::RSA.new(key_data)
+ end
end
def public_key
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 815fe6bac6d..c3c1728602c 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -12,7 +12,7 @@ module Gitlab
:seeds_block, :variables_attributes, :push_options,
:chat_data, :allow_mirror_update, :bridge, :content, :dry_run,
# These attributes are set by Chains during processing:
- :config_content, :yaml_processor_result, :pipeline_seed
+ :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed
) do
include Gitlab::Utils::StrongMemoize
@@ -84,7 +84,7 @@ module Gitlab
end
def metrics
- @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new
+ @metrics ||= ::Gitlab::Ci::Pipeline::Metrics
end
def observe_creation_duration(duration)
@@ -97,6 +97,11 @@ module Gitlab
.observe({ source: pipeline.source.to_s }, pipeline.total_size)
end
+ def increment_pipeline_failure_reason_counter(reason)
+ metrics.pipeline_failure_reason_counter
+ .increment(reason: (reason || :unknown_failure).to_s)
+ end
+
def dangling_build?
%i[ondemand_dast_scan webide].include?(source)
end
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
index c3fbd0c9e24..8f1c49563f2 100644
--- a/lib/gitlab/ci/pipeline/chain/config/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -14,6 +14,7 @@ module Gitlab
result = ::Gitlab::Ci::YamlProcessor.new(
@command.config_content, {
project: project,
+ ref: @pipeline.ref,
sha: @pipeline.sha,
user: current_user,
parent_pipeline: parent_pipeline
diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
index 3c910963a2a..cceaa52de16 100644
--- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
+++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb
@@ -9,6 +9,8 @@ module Gitlab
include Chain::Helpers
def perform!
+ @command.workflow_rules_result = workflow_rules_result
+
error('Pipeline filtered out by workflow rules.') unless workflow_passed?
end
@@ -19,27 +21,33 @@ module Gitlab
private
def workflow_passed?
- strong_memoize(:workflow_passed) do
- workflow_rules.evaluate(@pipeline, global_context).pass?
+ workflow_rules_result.pass?
+ end
+
+ def workflow_rules_result
+ strong_memoize(:workflow_rules_result) do
+ workflow_rules.evaluate(@pipeline, global_context)
end
end
def workflow_rules
Gitlab::Ci::Build::Rules.new(
- workflow_config[:rules], default_when: 'always')
+ workflow_rules_config, default_when: 'always')
end
def global_context
Gitlab::Ci::Build::Context::Global.new(
- @pipeline, yaml_variables: workflow_config[:yaml_variables])
+ @pipeline, yaml_variables: @command.yaml_processor_result.root_variables)
end
def has_workflow_rules?
- workflow_config[:rules].present?
+ workflow_rules_config.present?
end
- def workflow_config
- @command.yaml_processor_result.workflow_attributes || {}
+ def workflow_rules_config
+ strong_memoize(:workflow_rules_config) do
+ @command.yaml_processor_result.workflow_rules
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb
index d7271df1694..9988b6f18ed 100644
--- a/lib/gitlab/ci/pipeline/chain/helpers.rb
+++ b/lib/gitlab/ci/pipeline/chain/helpers.rb
@@ -12,7 +12,8 @@ module Gitlab
end
pipeline.add_error_message(message)
- pipeline.drop!(drop_reason) if drop_reason && persist_pipeline?
+
+ drop_pipeline!(drop_reason)
# TODO: consider not to rely on AR errors directly as they can be
# polluted with other unrelated errors (e.g. state machine)
@@ -24,8 +25,21 @@ module Gitlab
pipeline.add_warning_message(message)
end
- def persist_pipeline?
- command.save_incompleted && !pipeline.readonly?
+ private
+
+ def drop_pipeline!(drop_reason)
+ return if pipeline.readonly?
+
+ if drop_reason && command.save_incompleted
+ # Project iid must be called outside a transaction, so we ensure it is set here
+ # otherwise it may be set within the state transition transaction of the drop! call
+ # which it will lock the InternalId row for the whole transaction
+ pipeline.ensure_project_iid!
+
+ pipeline.drop!(drop_reason)
+ else
+ command.increment_pipeline_failure_reason_counter(drop_reason)
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb
index 0d7449813b4..b17ae77d445 100644
--- a/lib/gitlab/ci/pipeline/chain/metrics.rb
+++ b/lib/gitlab/ci/pipeline/chain/metrics.rb
@@ -14,7 +14,7 @@ module Gitlab
end
def counter
- ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter
+ ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
index 1eb7474e915..c1b6dfb7e36 100644
--- a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb
@@ -8,9 +8,7 @@ module Gitlab
# After pipeline has been successfully created we can start processing it.
class Process < Chain::Base
def perform!
- ::Ci::ProcessPipelineService
- .new(@pipeline)
- .execute
+ ::Ci::InitialPipelineProcessWorker.perform_async(pipeline.id)
end
def break?
diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb
index 7b537125b9b..66fc6741252 100644
--- a/lib/gitlab/ci/pipeline/chain/seed.rb
+++ b/lib/gitlab/ci/pipeline/chain/seed.rb
@@ -11,6 +11,10 @@ module Gitlab
def perform!
raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result
+ if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml)
+ raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result
+ end
+
# Allocate next IID. This operation must be outside of transactions of pipeline creations.
pipeline.ensure_project_iid!
pipeline.ensure_ci_ref!
@@ -38,7 +42,21 @@ module Gitlab
def pipeline_seed
strong_memoize(:pipeline_seed) do
stages_attributes = @command.yaml_processor_result.stages_attributes
- Gitlab::Ci::Pipeline::Seed::Pipeline.new(pipeline, stages_attributes)
+ Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes)
+ end
+ end
+
+ def context
+ Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables)
+ end
+
+ def root_variables
+ if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml)
+ ::Gitlab::Ci::Variables::Helpers.merge_variables(
+ @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables
+ )
+ else
+ @command.yaml_processor_result.root_variables
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb
index d056501a6d3..6149d2f04d7 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/external.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb
@@ -10,77 +10,116 @@ module Gitlab
InvalidResponseCode = Class.new(StandardError)
- VALIDATION_REQUEST_TIMEOUT = 5
+ DEFAULT_VALIDATION_REQUEST_TIMEOUT = 5
+ ACCEPTED_STATUS = 200
+ DOT_COM_REJECTED_STATUS = 406
+ GENERAL_REJECTED_STATUS = (400..499).freeze
def perform!
+ return unless enabled?
+
pipeline_authorized = validate_external
log_message = pipeline_authorized ? 'authorized' : 'not authorized'
- Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: @pipeline.project.id, user_id: @pipeline.user.id)
+ Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: project.id, user_id: current_user.id)
error('External validation failed', drop_reason: :external_validation_failure) unless pipeline_authorized
end
def break?
- @pipeline.errors.any?
+ pipeline.errors.any?
end
private
+ def enabled?
+ return true unless Gitlab.com?
+
+ ::Feature.enabled?(:ci_external_validation_service, project, default_enabled: :yaml)
+ end
+
def validate_external
return true unless validation_service_url
# 200 - accepted
- # 4xx - not accepted
+ # 406 - not accepted on GitLab.com
+ # 4XX - not accepted for other installations
# everything else - accepted and logged
response_code = validate_service_request.code
case response_code
- when 200
+ when ACCEPTED_STATUS
true
- when 400..499
+ when rejected_status
false
else
raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}"
end
rescue => ex
- Gitlab::ErrorTracking.track_exception(ex)
+ Gitlab::ErrorTracking.track_exception(ex, project_id: project.id)
true
end
+ def rejected_status
+ if Gitlab.com?
+ DOT_COM_REJECTED_STATUS
+ else
+ GENERAL_REJECTED_STATUS
+ end
+ end
+
def validate_service_request
+ headers = {
+ 'X-Gitlab-Correlation-id' => Labkit::Correlation::CorrelationId.current_id,
+ 'X-Gitlab-Token' => validation_service_token
+ }.compact
+
Gitlab::HTTP.post(
- validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT,
- body: validation_service_payload(@pipeline, @command.yaml_processor_result.stages_attributes)
+ validation_service_url, timeout: validation_service_timeout,
+ headers: headers,
+ body: validation_service_payload.to_json
)
end
+ def validation_service_timeout
+ timeout = Gitlab::CurrentSettings.external_pipeline_validation_service_timeout || ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i
+ return timeout if timeout > 0
+
+ DEFAULT_VALIDATION_REQUEST_TIMEOUT
+ end
+
def validation_service_url
- ENV['EXTERNAL_VALIDATION_SERVICE_URL']
+ Gitlab::CurrentSettings.external_pipeline_validation_service_url || ENV['EXTERNAL_VALIDATION_SERVICE_URL']
+ end
+
+ def validation_service_token
+ Gitlab::CurrentSettings.external_pipeline_validation_service_token || ENV['EXTERNAL_VALIDATION_SERVICE_TOKEN']
end
- def validation_service_payload(pipeline, stages_attributes)
+ def validation_service_payload
{
project: {
- id: pipeline.project.id,
- path: pipeline.project.full_path
+ id: project.id,
+ path: project.full_path,
+ created_at: project.created_at&.iso8601
},
user: {
- id: pipeline.user.id,
- username: pipeline.user.username,
- email: pipeline.user.email
+ id: current_user.id,
+ username: current_user.username,
+ email: current_user.email,
+ created_at: current_user.created_at&.iso8601
},
pipeline: {
sha: pipeline.sha,
ref: pipeline.ref,
type: pipeline.source
},
- builds: builds_validation_payload(stages_attributes)
- }.to_json
+ builds: builds_validation_payload
+ }
end
- def builds_validation_payload(stages_attributes)
- stages_attributes.map { |stage| stage[:builds] }.flatten
+ def builds_validation_payload
+ stages_attributes.flat_map { |stage| stage[:builds] }
.map(&method(:build_validation_payload))
end
@@ -97,9 +136,15 @@ module Gitlab
].flatten.compact
}
end
+
+ def stages_attributes
+ command.yaml_processor_result.stages_attributes
+ end
end
end
end
end
end
end
+
+Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::External')
diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb
index c77f4dcca5a..6cb6fd3920d 100644
--- a/lib/gitlab/ci/pipeline/metrics.rb
+++ b/lib/gitlab/ci/pipeline/metrics.rb
@@ -4,55 +4,57 @@ module Gitlab
module Ci
module Pipeline
class Metrics
- include Gitlab::Utils::StrongMemoize
+ def self.pipeline_creation_duration_histogram
+ name = :gitlab_ci_pipeline_creation_duration_seconds
+ comment = 'Pipeline creation duration'
+ labels = {}
+ buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0]
- def pipeline_creation_duration_histogram
- strong_memoize(:pipeline_creation_duration_histogram) do
- name = :gitlab_ci_pipeline_creation_duration_seconds
- comment = 'Pipeline creation duration'
- labels = {}
- buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0]
+ ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+ end
+
+ def self.pipeline_size_histogram
+ name = :gitlab_ci_pipeline_size_builds
+ comment = 'Pipeline size'
+ labels = { source: nil }
+ buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000]
+
+ ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+ end
+
+ def self.pipeline_processing_events_counter
+ name = :gitlab_ci_pipeline_processing_events_total
+ comment = 'Total amount of pipeline processing events'
- ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
- end
+ Gitlab::Metrics.counter(name, comment)
end
- def pipeline_size_histogram
- strong_memoize(:pipeline_size_histogram) do
- name = :gitlab_ci_pipeline_size_builds
- comment = 'Pipeline size'
- labels = { source: nil }
- buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000]
+ def self.pipelines_created_counter
+ name = :pipelines_created_total
+ comment = 'Counter of pipelines created'
- ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
- end
+ Gitlab::Metrics.counter(name, comment)
end
- def pipeline_processing_events_counter
- strong_memoize(:pipeline_processing_events_counter) do
- name = :gitlab_ci_pipeline_processing_events_total
- comment = 'Total amount of pipeline processing events'
+ def self.legacy_update_jobs_counter
+ name = :ci_legacy_update_jobs_as_retried_total
+ comment = 'Counter of occurrences when jobs were not being set as retried before update_retried'
- Gitlab::Metrics.counter(name, comment)
- end
+ Gitlab::Metrics.counter(name, comment)
end
- def pipelines_created_counter
- strong_memoize(:pipelines_created_count) do
- name = :pipelines_created_total
- comment = 'Counter of pipelines created'
+ def self.pipeline_failure_reason_counter
+ name = :gitlab_ci_pipeline_failure_reasons
+ comment = 'Counter of pipeline failure reasons'
- Gitlab::Metrics.counter(name, comment)
- end
+ Gitlab::Metrics.counter(name, comment)
end
- def legacy_update_jobs_counter
- strong_memoize(:legacy_update_jobs_counter) do
- name = :ci_legacy_update_jobs_as_retried_total
- comment = 'Counter of occurrences when jobs were not being set as retried before update_retried'
+ def self.job_failure_reason_counter
+ name = :gitlab_ci_job_failure_reasons
+ comment = 'Counter of job failure reasons'
- Gitlab::Metrics.counter(name, comment)
- end
+ Gitlab::Metrics.counter(name, comment)
end
end
end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 11b01822e4b..39dee7750d6 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -11,12 +11,15 @@ module Gitlab
delegate :dig, to: :@seed_attributes
- def initialize(pipeline, attributes, previous_stages)
- @pipeline = pipeline
+ def initialize(context, attributes, previous_stages)
+ @context = context
+ @pipeline = context.pipeline
@seed_attributes = attributes
@previous_stages = previous_stages
@needs_attributes = dig(:needs_attributes)
@resource_group_key = attributes.delete(:resource_group_key)
+ @job_variables = @seed_attributes.delete(:job_variables)
+ @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true }
@using_rules = attributes.key?(:rules)
@using_only = attributes.key?(:only)
@@ -29,7 +32,9 @@ module Gitlab
@rules = Gitlab::Ci::Build::Rules
.new(attributes.delete(:rules), default_when: 'on_success')
@cache = Gitlab::Ci::Build::Cache
- .new(attributes.delete(:cache), pipeline)
+ .new(attributes.delete(:cache), @pipeline)
+
+ recalculate_yaml_variables!
end
def name
@@ -206,6 +211,14 @@ module Gitlab
{ options: { allow_failure_criteria: nil } }
end
+
+ def recalculate_yaml_variables!
+ return unless ::Feature.enabled?(:ci_workflow_rules_variables, @pipeline.project, default_enabled: :yaml)
+
+ @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables(
+ from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance
+ )
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/seed/context.rb b/lib/gitlab/ci/pipeline/seed/context.rb
new file mode 100644
index 00000000000..6194a78f682
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/seed/context.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Seed
+ class Context
+ attr_reader :pipeline, :root_variables
+
+ def initialize(pipeline, root_variables: [])
+ @pipeline = pipeline
+ @root_variables = root_variables
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb
index da9d853cf68..e1a15fb8d5b 100644
--- a/lib/gitlab/ci/pipeline/seed/pipeline.rb
+++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb
@@ -7,8 +7,8 @@ module Gitlab
class Pipeline
include Gitlab::Utils::StrongMemoize
- def initialize(pipeline, stages_attributes)
- @pipeline = pipeline
+ def initialize(context, stages_attributes)
+ @context = context
@stages_attributes = stages_attributes
end
@@ -37,7 +37,7 @@ module Gitlab
def stage_seeds
strong_memoize(:stage_seeds) do
seeds = @stages_attributes.inject([]) do |previous_stages, attributes|
- seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@pipeline, attributes, previous_stages)
+ seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@context, attributes, previous_stages)
previous_stages + [seed]
end
diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb
index b600df2f656..c988ea10e41 100644
--- a/lib/gitlab/ci/pipeline/seed/stage.rb
+++ b/lib/gitlab/ci/pipeline/seed/stage.rb
@@ -10,13 +10,14 @@ module Gitlab
delegate :size, to: :seeds
delegate :dig, to: :seeds
- def initialize(pipeline, attributes, previous_stages)
- @pipeline = pipeline
+ def initialize(context, attributes, previous_stages)
+ @context = context
+ @pipeline = context.pipeline
@attributes = attributes
@previous_stages = previous_stages
@builds = attributes.fetch(:builds).map do |attributes|
- Seed::Build.new(@pipeline, attributes, previous_stages)
+ Seed::Build.new(context, attributes, previous_stages)
end
end
diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb
index 5398c19e536..7ecb9a1db16 100644
--- a/lib/gitlab/ci/queue/metrics.rb
+++ b/lib/gitlab/ci/queue/metrics.rb
@@ -9,12 +9,12 @@ module Gitlab
QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
- QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze
- QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
+ QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze
+ QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
DEFAULT_METRICS_SHARD = 'default'
- JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze
+ JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5
OPERATION_COUNTERS = [
:build_can_pick,
@@ -94,13 +94,13 @@ module Gitlab
self.class.queue_depth_total.observe({ queue: queue }, size.to_f)
end
- def observe_queue_size(size_proc)
+ def observe_queue_size(size_proc, runner_type)
return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
- self.class.queue_size_total.observe({}, size_proc.call.to_f)
+ self.class.queue_size_total.observe({ runner_type: runner_type }, size_proc.call.to_f)
end
- def observe_queue_time
+ def observe_queue_time(metric, runner_type)
start_time = ::Gitlab::Metrics::System.monotonic_time
result = yield
@@ -108,7 +108,15 @@ module Gitlab
return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
seconds = ::Gitlab::Metrics::System.monotonic_time - start_time
- self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f)
+
+ case metric
+ when :process
+ self.class.queue_iteration_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f)
+ when :retrieve
+ self.class.queue_retrieval_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f)
+ else
+ raise ArgumentError unless Rails.env.production?
+ end
result
end
@@ -187,7 +195,18 @@ module Gitlab
strong_memoize(:queue_iteration_duration_seconds) do
name = :gitlab_ci_queue_iteration_duration_seconds
comment = 'Time it takes to find a build in CI/CD queue'
- buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS
+ buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
+ labels = {}
+
+ Gitlab::Metrics.histogram(name, comment, labels, buckets)
+ end
+ end
+
+ def self.queue_retrieval_duration_seconds
+ strong_memoize(:queue_retrieval_duration_seconds) do
+ name = :gitlab_ci_queue_retrieval_duration_seconds
+ comment = 'Time it takes to execute a SQL query to retrieve builds queue'
+ buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
labels = {}
Gitlab::Metrics.histogram(name, comment, labels, buckets)
diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb
index 060a1e2399b..27c41c384b8 100644
--- a/lib/gitlab/ci/reports/codequality_reports.rb
+++ b/lib/gitlab/ci/reports/codequality_reports.rb
@@ -6,6 +6,7 @@ module Gitlab
class CodequalityReports
attr_reader :degradations, :error_message
+ SEVERITY_PRIORITIES = %w(blocker critical major minor info).map.with_index.to_h.freeze # { "blocker" => 0, "critical" => 1 ... }
CODECLIMATE_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'codeclimate.json').to_s
def initialize
@@ -29,12 +30,17 @@ module Gitlab
@degradations.values
end
+ def sort_degradations!
+ @degradations = @degradations.sort_by do |_fingerprint, degradation|
+ SEVERITY_PRIORITIES[degradation.dig(:severity)]
+ end.to_h
+ end
+
private
def valid_degradation?(degradation)
- JSON::Validator.validate!(CODECLIMATE_SCHEMA_PATH, degradation)
- rescue JSON::Schema::ValidationError => e
- set_error_message("Invalid degradation format: #{e.message}")
+ JSONSchemer.schema(Pathname.new(CODECLIMATE_SCHEMA_PATH)).valid?(degradation)
+ rescue StandardError => _
false
end
end
diff --git a/lib/gitlab/ci/reports/codequality_reports_comparer.rb b/lib/gitlab/ci/reports/codequality_reports_comparer.rb
index 10748b8ca02..e34d9675c10 100644
--- a/lib/gitlab/ci/reports/codequality_reports_comparer.rb
+++ b/lib/gitlab/ci/reports/codequality_reports_comparer.rb
@@ -7,6 +7,11 @@ module Gitlab
def initialize(base_report, head_report)
@base_report = base_report
@head_report = head_report
+
+ unless not_found?
+ @base_report.sort_degradations!
+ @head_report.sort_degradations!
+ end
end
def success?
diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb
index c024e794ad5..37d0da38065 100644
--- a/lib/gitlab/ci/reports/test_failure_history.rb
+++ b/lib/gitlab/ci/reports/test_failure_history.rb
@@ -6,32 +6,32 @@ module Gitlab
class TestFailureHistory
include Gitlab::Utils::StrongMemoize
- def initialize(failed_test_cases, project)
- @failed_test_cases = build_map(failed_test_cases)
+ def initialize(failed_junit_tests, project)
+ @failed_junit_tests = build_map(failed_junit_tests)
@project = project
end
def load!
recent_failures_count.each do |key_hash, count|
- failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master)
+ failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_master)
end
end
private
- attr_reader :report, :project, :failed_test_cases
+ attr_reader :report, :project, :failed_junit_tests
def recent_failures_count
- ::Ci::TestCaseFailure.recent_failures_count(
+ ::Ci::UnitTestFailure.recent_failures_count(
project: project,
- test_case_keys: failed_test_cases.keys
+ unit_test_keys: failed_junit_tests.keys
)
end
- def build_map(test_cases)
+ def build_map(junit_tests)
{}.tap do |hash|
- test_cases.each do |test_case|
- hash[test_case.key] = test_case
+ junit_tests.each do |test|
+ hash[test.key] = test
end
end
end
diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb
index dd0bfa768a8..365864d3317 100644
--- a/lib/gitlab/ci/runner_instructions.rb
+++ b/lib/gitlab/ci/runner_instructions.rb
@@ -51,10 +51,7 @@ module Gitlab
attr_reader :errors
- def initialize(current_user:, group: nil, project: nil, os:, arch:)
- @current_user = current_user
- @group = group
- @project = project
+ def initialize(os:, arch:)
@os = os
@arch = arch
@errors = []
@@ -77,7 +74,7 @@ module Gitlab
server_url = Gitlab::Routing.url_helpers.root_url(only_path: false)
runner_executable = environment[:runner_executable]
- "#{runner_executable} register --url #{server_url} --registration-token #{registration_token}"
+ "#{runner_executable} register --url #{server_url} --registration-token $REGISTRATION_TOKEN"
end
end
@@ -108,30 +105,6 @@ module Gitlab
def get_file(path)
File.read(Rails.root.join(path).to_s)
end
-
- def registration_token
- project_token || group_token || instance_token
- end
-
- def project_token
- return unless @project
- raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_pipeline, @project)
-
- @project.runners_token
- end
-
- def group_token
- return unless @group
- raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_group, @group)
-
- @group.runners_token
- end
-
- def instance_token
- raise Gitlab::Access::AccessDeniedError unless @current_user&.admin?
-
- Gitlab::CurrentSettings.runners_registration_token
- end
end
end
end
diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb
index f6562737838..787dee3b267 100644
--- a/lib/gitlab/ci/status/build/failed.rb
+++ b/lib/gitlab/ci/status/build/failed.rb
@@ -26,7 +26,9 @@ module Gitlab
bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline',
downstream_pipeline_creation_failed: 'downstream pipeline can not be created',
secrets_provider_not_found: 'secrets provider can not be found',
- reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines'
+ reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines',
+ project_deleted: 'pipeline project was deleted',
+ user_blocked: 'pipeline user was blocked'
}.freeze
private_constant :REASONS
diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml
index 5ebbbf15682..2ff36bcc657 100644
--- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml
@@ -113,9 +113,10 @@ promoteBeta:
promoteProduction:
extends: .promote_job
stage: production
- # We only allow production promotion on `master` because
- # it has its own production scoped secret variables
+ # We only allow production promotion on the default branch because
+ # it has its own production scoped secret variables.
only:
- - master
+ variables:
+ - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
script:
- bundle exec fastlane promote_beta_to_production
diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
index 15cdbf63cb1..d0c63ab6edf 100644
--- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml
@@ -1,27 +1,31 @@
-docker-build-master:
- # Official docker image.
- image: docker:latest
- stage: build
- services:
- - docker:dind
- before_script:
- - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
- script:
- - docker build --pull -t "$CI_REGISTRY_IMAGE" .
- - docker push "$CI_REGISTRY_IMAGE"
- only:
- - master
-
+# Build a Docker image with CI/CD and push to the GitLab registry.
+# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html
+#
+# This template uses one generic job with conditional builds
+# for the default branch and all other (MR) branches.
docker-build:
- # Official docker image.
+ # Use the official docker image.
image: docker:latest
stage: build
services:
- docker:dind
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+ # Default branch leaves tag empty (= latest tag)
+ # All other branches are tagged with the escaped branch name (commit ref slug)
script:
- - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" .
- - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG"
- except:
- - master
+ - |
+ if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then
+ tag=""
+ echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'"
+ else
+ tag=":$CI_COMMIT_REF_SLUG"
+ echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag"
+ fi
+ - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" .
+ - docker push "$CI_REGISTRY_IMAGE${tag}"
+ # Run this job in a branch where a Dockerfile exists
+ rules:
+ - if: $CI_COMMIT_BRANCH
+ exists:
+ - Dockerfile
diff --git a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml
new file mode 100644
index 00000000000..90812083917
--- /dev/null
+++ b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml
@@ -0,0 +1,9 @@
+# This file is a template demonstrating the `script` keyword.
+# Learn more about this keyword here: https://docs.gitlab.com/ee/ci/yaml/README.html#script
+
+# After committing this template, visit CI/CD > Jobs to see the script output.
+
+job:
+ script:
+ # provide a shell script as argument for this keyword.
+ - echo "Hello World"
diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml
new file mode 100644
index 00000000000..c7fb1321055
--- /dev/null
+++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml
@@ -0,0 +1,91 @@
+# This template is provided and maintained by Indeni, an official Technology Partner with GitLab.
+# See https://about.gitlab.com/partners/technology-partners/#security for more information.
+
+# For more information about Indeni Cloudrail: https://indeni.com/cloudrail/
+#
+# This file shows an example of using Indeni Cloudrail with GitLab CI/CD.
+# It is not designed to be included in an existing CI/CD configuration with the "include:" keyword.
+# Documentation about this integration: https://indeni.com/doc-indeni-cloudrail/integrate-with-ci-cd/gitlab-instructions
+#
+# For an example of this used in a GitLab repository, see: https://gitlab.com/indeni/cloudrail-demo/-/blob/master/.gitlab-ci.yml
+
+# The sast-report output complies with GitLab's format. This report displays Cloudrail's
+# results in the Security tab in the pipeline view, if you have that feature enabled
+# (GitLab Ultimate only). Otherwise, Cloudrail generates a JUnit report, which displays
+# in the "Test summary" in merge requests.
+
+# Note that Cloudrail's input is the Terraform plan. That is why we've included in this
+# template an example of doing that. You are welcome to replace it with your own way
+# of generating a Terraform plan.
+
+# Before you can use this template, get a Cloudrail API key from the Cloudrail web
+# user interface. Save it as a CI/CD variable named CLOUDRAIL_API_KEY in your project
+# settings.
+
+variables:
+ TEST_ROOT: ${CI_PROJECT_DIR}/my_folder_with_terraform_content
+
+default:
+ before_script:
+ - cd ${CI_PROJECT_DIR}/my_folder_with_terraform_content
+
+stages:
+ - init_and_plan
+ - cloudrail
+
+init_and_plan:
+ stage: init_and_plan
+ image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13
+ rules:
+ - if: $SAST_DISABLED
+ when: never
+ - if: $CI_COMMIT_BRANCH
+ exists:
+ - '**/*.tf'
+ script:
+ - terraform init
+ - terraform plan -out=plan.out
+ artifacts:
+ name: "$CI_COMMIT_BRANCH-terraform_plan"
+ paths:
+ - ./**/plan.out
+ - ./**/.terraform
+
+cloudrail_scan:
+ stage: cloudrail
+ image: indeni/cloudrail-cli:1.2.44
+ rules:
+ - if: $SAST_DISABLED
+ when: never
+ - if: $CI_COMMIT_BRANCH
+ exists:
+ - '**/*.tf'
+ script:
+ - |
+ if [[ "${GITLAB_FEATURES}" == *"security_dashboard"* ]]; then
+ echo "You are licensed for GitLab Security Dashboards. Your scan results will display in the Security Dashboard."
+ cloudrail run --tf-plan plan.out \
+ --directory . \
+ --api-key ${CLOUDRAIL_API_KEY} \
+ --origin ci \
+ --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \
+ --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \
+ --output-format json-gitlab-sast \
+ --output-file ${CI_PROJECT_DIR}/cloudrail-sast-report.json \
+ --auto-approve
+ else
+ echo "Your scan results will display in the GitLab Test results visualization panel."
+ cloudrail run --tf-plan plan.out \
+ --directory . \
+ --api-key ${CLOUDRAIL_API_KEY} \
+ --origin ci \
+ --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \
+ --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \
+ --output-format junit \
+ --output-file ${CI_PROJECT_DIR}/cloudrail-junit-report.xml \
+ --auto-approve
+ fi
+ artifacts:
+ reports:
+ sast: cloudrail-sast-report.json
+ junit: cloudrail-junit-report.xml
diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
index 5edb26a0b56..01907ef9e2e 100644
--- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
@@ -20,15 +20,48 @@ performance:
fi
- export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
- mkdir gitlab-exporter
+ # Busybox wget does not support proxied HTTPS, get the real thing.
+ # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611.
+ - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget
- wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js
- mkdir sitespeed-results
- |
+ function propagate_env_vars() {
+ CURRENT_ENV=$(printenv)
+
+ for VAR_NAME; do
+ echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
+ done
+ }
+ - |
if [ -f .gitlab-urls.txt ]
then
sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt
- docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS
+ docker run \
+ $(propagate_env_vars \
+ auto_proxy \
+ https_proxy \
+ http_proxy \
+ no_proxy \
+ AUTO_PROXY \
+ HTTPS_PROXY \
+ HTTP_PROXY \
+ NO_PROXY \
+ ) \
+ --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS
else
- docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS
+ docker run \
+ $(propagate_env_vars \
+ auto_proxy \
+ https_proxy \
+ http_proxy \
+ no_proxy \
+ AUTO_PROXY \
+ HTTPS_PROXY \
+ HTTP_PROXY \
+ NO_PROXY \
+ ) \
+ --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS
fi
- mv sitespeed-results/data/performance.json browser-performance.json
artifacts:
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index 1c25d9d583b..196d42f3e3a 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -1,10 +1,10 @@
build:
stage: build
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0"
variables:
DOCKER_TLS_CERTDIR: ""
services:
- - docker:19.03.12-dind
+ - docker:20.10.6-dind
script:
- |
if [[ -z "$CI_COMMIT_TAG" ]]; then
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index fd6c51ea350..b29342216fc 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -36,6 +36,7 @@ code_quality:
REPORT_STDOUT \
REPORT_FORMAT \
ENGINE_MEMORY_LIMIT_BYTES \
+ CODECLIMATE_PREFIX \
) \
--volume "$PWD":/code \
--volume /var/run/docker.sock:/var/run/docker.sock \
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
index 654a03ced5f..bf42cd52605 100644
--- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml
@@ -12,7 +12,7 @@ stages:
variables:
FUZZAPI_PROFILE: Quick
- FUZZAPI_VERSION: latest
+ FUZZAPI_VERSION: "1.6"
FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml
FUZZAPI_TIMEOUT: 30
FUZZAPI_REPORT: gl-api-fuzzing-report.json
@@ -45,7 +45,7 @@ apifuzzer_fuzz:
entrypoint: ["/bin/bash", "-l", "-c"]
variables:
FUZZAPI_PROJECT: $CI_PROJECT_PATH
- FUZZAPI_API: http://localhost:80
+ FUZZAPI_API: http://localhost:5000
FUZZAPI_NEW_REPORT: 1
FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log
TZ: America/Los_Angeles
@@ -107,7 +107,7 @@ apifuzzer_fuzz_dnd:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
FUZZAPI_PROJECT: $CI_PROJECT_PATH
- FUZZAPI_API: http://apifuzzer:80
+ FUZZAPI_API: http://apifuzzer:5000
allow_failure: true
rules:
- if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null
@@ -142,6 +142,7 @@ apifuzzer_fuzz_dnd:
-e TZ=America/Los_Angeles \
-e GITLAB_FEATURES \
-p 80:80 \
+ -p 5000:5000 \
-p 8000:8000 \
-p 514:514 \
--restart=no \
@@ -168,7 +169,7 @@ apifuzzer_fuzz_dnd:
docker run \
--name worker \
--network $FUZZAPI_D_NETWORK \
- -e FUZZAPI_API=http://apifuzzer:80 \
+ -e FUZZAPI_API=http://apifuzzer:5000 \
-e FUZZAPI_PROJECT \
-e FUZZAPI_PROFILE \
-e FUZZAPI_CONFIG \
@@ -211,7 +212,7 @@ apifuzzer_fuzz_dnd:
--name worker \
--network $FUZZAPI_D_NETWORK \
-e TZ=America/Los_Angeles \
- -e FUZZAPI_API=http://apifuzzer:80 \
+ -e FUZZAPI_API=http://apifuzzer:5000 \
-e FUZZAPI_PROJECT \
-e FUZZAPI_PROFILE \
-e FUZZAPI_CONFIG \
@@ -237,6 +238,7 @@ apifuzzer_fuzz_dnd:
-v $CI_PROJECT_DIR:/app \
-v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \
-p 81:80 \
+ -p 5001:5000 \
-p 8001:8000 \
-p 515:514 \
--restart=no \
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..215029dc952
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
@@ -0,0 +1,270 @@
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/
+
+# Configure the scanning tool through the environment variables.
+# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables
+# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+
+variables:
+ FUZZAPI_PROFILE: Quick
+ FUZZAPI_VERSION: latest
+ FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml
+ FUZZAPI_TIMEOUT: 30
+ FUZZAPI_REPORT: gl-api-fuzzing-report.json
+ FUZZAPI_REPORT_ASSET_PATH: assets
+ #
+ FUZZAPI_D_NETWORK: testing-net
+ #
+ # Wait up to 5 minutes for API Fuzzer and target url to become
+ # available (non 500 response to HTTP(s))
+ FUZZAPI_SERVICE_START_TIMEOUT: "300"
+ #
+ FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine
+ #
+
+apifuzzer_fuzz_unlicensed:
+ stage: fuzz
+ allow_failure: true
+ rules:
+ - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null'
+ - when: never
+ script:
+ - |
+ echo "Error: Your GitLab project is not licensed for API Fuzzing."
+ - exit 1
+
+apifuzzer_fuzz:
+ stage: fuzz
+ image:
+ name: $FUZZAPI_IMAGE
+ entrypoint: ["/bin/bash", "-l", "-c"]
+ variables:
+ FUZZAPI_PROJECT: $CI_PROJECT_PATH
+ FUZZAPI_API: http://localhost:80
+ FUZZAPI_NEW_REPORT: 1
+ FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log
+ TZ: America/Los_Angeles
+ allow_failure: true
+ rules:
+ - if: $FUZZAPI_D_TARGET_IMAGE
+ when: never
+ - if: $FUZZAPI_D_WORKER_IMAGE
+ when: never
+ - if: $API_FUZZING_DISABLED
+ when: never
+ - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+ script:
+ #
+ # Validate options
+ - |
+ if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+ echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+ echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+ exit 1; \
+ fi
+ #
+ # Run user provided pre-script
+ - sh -c "$FUZZAPI_PRE_SCRIPT"
+ #
+ # Make sure asset path exists
+ - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
+ #
+ # Start API Security background process
+ - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER &
+ - APISEC_PID=$!
+ #
+ # Start scanning
+ - worker-entry
+ #
+ # Run user provided post-script
+ - sh -c "$FUZZAPI_POST_SCRIPT"
+ #
+ # Shutdown API Security
+ - kill $APISEC_PID
+ - wait $APISEC_PID
+ #
+ artifacts:
+ when: always
+ paths:
+ - $FUZZAPI_REPORT_ASSET_PATH
+ - $FUZZAPI_REPORT
+ - $FUZZAPI_LOG_SCANNER
+ reports:
+ api_fuzzing: $FUZZAPI_REPORT
+
+apifuzzer_fuzz_dnd:
+ stage: fuzz
+ image: docker:19.03.12
+ variables:
+ DOCKER_DRIVER: overlay2
+ DOCKER_TLS_CERTDIR: ""
+ FUZZAPI_PROJECT: $CI_PROJECT_PATH
+ FUZZAPI_API: http://apifuzzer:80
+ allow_failure: true
+ rules:
+ - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null
+ when: never
+ - if: $API_FUZZING_DISABLED
+ when: never
+ - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/
+ services:
+ - docker:19.03.12-dind
+ script:
+ #
+ #
+ - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+ #
+ - docker network create --driver bridge $FUZZAPI_D_NETWORK
+ #
+ # Run user provided pre-script
+ - sh -c "$FUZZAPI_PRE_SCRIPT"
+ #
+ # Make sure asset path exists
+ - mkdir -p $FUZZAPI_REPORT_ASSET_PATH
+ #
+ # Start peach testing engine container
+ - |
+ docker run -d \
+ --name apifuzzer \
+ --network $FUZZAPI_D_NETWORK \
+ -e Proxy:Port=8000 \
+ -e TZ=America/Los_Angeles \
+ -e GITLAB_FEATURES \
+ -p 80:80 \
+ -p 8000:8000 \
+ -p 514:514 \
+ --restart=no \
+ $FUZZAPI_IMAGE \
+ dotnet /peach/Peach.Web.dll
+ #
+ # Start target container
+ - |
+ if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \
+ docker run -d \
+ --name target \
+ --network $FUZZAPI_D_NETWORK \
+ $FUZZAPI_D_TARGET_ENV \
+ $FUZZAPI_D_TARGET_PORTS \
+ $FUZZAPI_D_TARGET_VOLUME \
+ --restart=no \
+ $FUZZAPI_D_TARGET_IMAGE \
+ ; fi
+ #
+ # Start worker container if provided
+ - |
+ if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \
+ echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \
+ docker run \
+ --name worker \
+ --network $FUZZAPI_D_NETWORK \
+ -e FUZZAPI_API=http://apifuzzer:80 \
+ -e FUZZAPI_PROJECT \
+ -e FUZZAPI_PROFILE \
+ -e FUZZAPI_CONFIG \
+ -e FUZZAPI_REPORT \
+ -e FUZZAPI_REPORT_ASSET_PATH \
+ -e FUZZAPI_NEW_REPORT=1 \
+ -e FUZZAPI_HAR \
+ -e FUZZAPI_OPENAPI \
+ -e FUZZAPI_POSTMAN_COLLECTION \
+ -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
+ -e FUZZAPI_TARGET_URL \
+ -e FUZZAPI_OVERRIDES_FILE \
+ -e FUZZAPI_OVERRIDES_ENV \
+ -e FUZZAPI_OVERRIDES_CMD \
+ -e FUZZAPI_OVERRIDES_INTERVAL \
+ -e FUZZAPI_TIMEOUT \
+ -e FUZZAPI_VERBOSE \
+ -e FUZZAPI_SERVICE_START_TIMEOUT \
+ -e FUZZAPI_HTTP_USERNAME \
+ -e FUZZAPI_HTTP_PASSWORD \
+ -e CI_PROJECT_URL \
+ -e CI_JOB_ID \
+ -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \
+ $FUZZAPI_D_WORKER_ENV \
+ $FUZZAPI_D_WORKER_PORTS \
+ $FUZZAPI_D_WORKER_VOLUME \
+ --restart=no \
+ $FUZZAPI_D_WORKER_IMAGE \
+ ; fi
+ #
+ # Start API Fuzzing provided worker if no other worker present
+ - |
+ if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \
+ if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \
+ echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \
+ echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \
+ exit 1; \
+ fi; \
+ docker run \
+ --name worker \
+ --network $FUZZAPI_D_NETWORK \
+ -e TZ=America/Los_Angeles \
+ -e FUZZAPI_API=http://apifuzzer:80 \
+ -e FUZZAPI_PROJECT \
+ -e FUZZAPI_PROFILE \
+ -e FUZZAPI_CONFIG \
+ -e FUZZAPI_REPORT \
+ -e FUZZAPI_REPORT_ASSET_PATH \
+ -e FUZZAPI_NEW_REPORT=1 \
+ -e FUZZAPI_HAR \
+ -e FUZZAPI_OPENAPI \
+ -e FUZZAPI_POSTMAN_COLLECTION \
+ -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \
+ -e FUZZAPI_TARGET_URL \
+ -e FUZZAPI_OVERRIDES_FILE \
+ -e FUZZAPI_OVERRIDES_ENV \
+ -e FUZZAPI_OVERRIDES_CMD \
+ -e FUZZAPI_OVERRIDES_INTERVAL \
+ -e FUZZAPI_TIMEOUT \
+ -e FUZZAPI_VERBOSE \
+ -e FUZZAPI_SERVICE_START_TIMEOUT \
+ -e FUZZAPI_HTTP_USERNAME \
+ -e FUZZAPI_HTTP_PASSWORD \
+ -e CI_PROJECT_URL \
+ -e CI_JOB_ID \
+ -v $CI_PROJECT_DIR:/app \
+ -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \
+ -p 81:80 \
+ -p 8001:8000 \
+ -p 515:514 \
+ --restart=no \
+ $FUZZAPI_IMAGE \
+ worker-entry \
+ ; fi
+ #
+ # Propagate exit code from api fuzzing scanner (if any)
+ - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi
+ #
+ # Run user provided post-script
+ - sh -c "$FUZZAPI_POST_SCRIPT"
+ #
+ after_script:
+ #
+ # Shutdown all containers
+ - echo "Stopping all containers"
+ - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi
+ - docker stop worker
+ - docker stop apifuzzer
+ #
+ # Save docker logs
+ - docker logs apifuzzer &> gl-api_fuzzing-logs.log
+ - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi
+ - docker logs worker &> gl-api_fuzzing-worker-logs.log
+ #
+ artifacts:
+ when: always
+ paths:
+ - ./gl-api_fuzzing*.log
+ - ./gl-api_fuzzing*.zip
+ - $FUZZAPI_REPORT_ASSET_PATH
+ - $FUZZAPI_REPORT
+ reports:
+ api_fuzzing: $FUZZAPI_REPORT
+
+# end
diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
index 64001c2828a..c628e30b2c7 100644
--- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
@@ -6,14 +6,10 @@ variables:
SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
CS_MAJOR_VERSION: 3
-container_scanning:
+.cs_common:
stage: test
image: "$CS_ANALYZER_IMAGE"
variables:
- # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image
- # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes
- CLAIR_DB_IMAGE_TAG: "latest"
- CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG"
# Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml`
# file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template
# for details
@@ -21,19 +17,44 @@ container_scanning:
# CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to
# override the analyzer image with a custom value. This may be subject to change or
# breakage across GitLab releases.
- CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION
+ CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/$CS_PROJECT:$CS_MAJOR_VERSION
allow_failure: true
+ artifacts:
+ reports:
+ container_scanning: gl-container-scanning-report.json
+ dependencies: []
+
+container_scanning:
+ extends: .cs_common
+ variables:
+ # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image
+ # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes
+ CLAIR_DB_IMAGE_TAG: "latest"
+ CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG"
+ CS_PROJECT: 'klar'
services:
- name: $CLAIR_DB_IMAGE
alias: clair-vulnerabilities-db
script:
- /analyzer run
+ rules:
+ - if: $CONTAINER_SCANNING_DISABLED
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ &&
+ $CS_MAJOR_VERSION =~ /^[0-3]$/
+
+container_scanning_new:
+ extends: .cs_common
+ variables:
+ CS_PROJECT: 'container-scanning'
+ script:
+ - gtcs scan
artifacts:
- reports:
- container_scanning: gl-container-scanning-report.json
- dependencies: []
+ paths: [gl-container-scanning-report.json]
rules:
- if: $CONTAINER_SCANNING_DISABLED
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
+ $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ &&
+ $CS_MAJOR_VERSION !~ /^[0-3]$/
diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
index fc1acd09714..533f8bb25f8 100644
--- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
@@ -1,3 +1,16 @@
+# To use this template, add the following to your .gitlab-ci.yml file:
+#
+# include:
+# template: DAST.latest.gitlab-ci.yml
+#
+# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST:
+#
+# stages:
+# - build
+# - test
+# - deploy
+# - dast
+
# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/
# Configure the scanning tool through the environment variables.
@@ -9,6 +22,19 @@ variables:
# Setting this variable will affect all Security templates
# (SAST, Dependency Scanning, ...)
SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+ #
+ DAST_API_PROFILE: Full
+ DAST_API_VERSION: latest
+ DAST_API_CONFIG: .gitlab-dast-api.yml
+ DAST_API_TIMEOUT: 30
+ DAST_API_REPORT: gl-dast-api-report.json
+ DAST_API_REPORT_ASSET_PATH: assets
+ #
+ # Wait up to 5 minutes for API Security and target url to become
+ # available (non 500 response to HTTP(s))
+ DAST_API_SERVICE_START_TIMEOUT: "300"
+ #
+ DAST_API_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${DAST_API_VERSION}-engine
dast:
stage: dast
@@ -25,6 +51,11 @@ dast:
reports:
dast: gl-dast-report.json
rules:
+ - if: $DAST_API_BETA && ( $DAST_API_SPECIFICATION ||
+ $DAST_API_OPENAPI ||
+ $DAST_API_POSTMAN_COLLECTION ||
+ $DAST_API_HAR )
+ when: never
- if: $DAST_DISABLED
when: never
- if: $DAST_DISABLED_FOR_DEFAULT_BRANCH &&
@@ -40,4 +71,72 @@ dast:
- if: $CI_COMMIT_BRANCH &&
$DAST_WEBSITE
- if: $CI_COMMIT_BRANCH &&
+ $DAST_API_BETA == null &&
$DAST_API_SPECIFICATION
+
+dast_api:
+ stage: dast
+ image:
+ name: $DAST_API_IMAGE
+ entrypoint: ["/bin/bash", "-l", "-c"]
+ variables:
+ API_SECURITY_MODE: DAST
+ DAST_API_NEW_REPORT: 1
+ DAST_API_PROJECT: $CI_PROJECT_PATH
+ DAST_API_API: http://127.0.0.1:5000
+ DAST_API_LOG_SCANNER: gl-dast-api-scanner.log
+ TZ: America/Los_Angeles
+ allow_failure: true
+ rules:
+ - if: $DAST_API_BETA == null
+ when: never
+ - if: $DAST_DISABLED
+ when: never
+ - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME &&
+ $REVIEW_DISABLED &&
+ $DAST_API_SPECIFICATION == null &&
+ $DAST_API_OPENAPI == null &&
+ $DAST_API_POSTMAN_COLLECTION == null &&
+ $DAST_API_HAR == null
+ when: never
+ - if: $DAST_API_SPECIFICATION == null &&
+ $DAST_API_OPENAPI == null &&
+ $DAST_API_POSTMAN_COLLECTION == null &&
+ $DAST_API_HAR == null
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdast\b/
+ script:
+ #
+ # Run user provided pre-script
+ - sh -c "$DAST_API_PRE_SCRIPT"
+ #
+ # Make sure asset path exists
+ - mkdir -p $DAST_API_REPORT_ASSET_PATH
+ #
+ # Start API Security background process
+ - dotnet /peach/Peach.Web.dll &> $DAST_API_LOG_SCANNER &
+ - APISEC_PID=$!
+ #
+ # Start scanning
+ - worker-entry
+ #
+ # Run user provided post-script
+ - sh -c "$DAST_API_POST_SCRIPT"
+ #
+ # Shutdown API Security
+ - kill $APISEC_PID
+ - wait $APISEC_PID
+ #
+ artifacts:
+ when: always
+ paths:
+ - $DAST_API_REPORT_ASSET_PATH
+ - $DAST_API_REPORT
+ - $DAST_API_LOG_SCANNER
+ - gl-*.log
+ reports:
+ dast: $DAST_API_REPORT
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 9693a4fbca2..3ebccfbba4a 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -160,7 +160,7 @@ mobsf-android-sast:
services:
# this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile
# Unfortunately, we need to keep track of mobsf version in 2 different places for now.
- - name: opensecurity/mobile-security-framework-mobsf:v3.3.3
+ - name: opensecurity/mobile-security-framework-mobsf:v3.4.0
alias: mobsf
image:
name: "$SAST_ANALYZER_IMAGE"
@@ -186,7 +186,7 @@ mobsf-ios-sast:
services:
# this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile
# Unfortunately, we need to keep track of mobsf version in 2 different places for now.
- - name: opensecurity/mobile-security-framework-mobsf:v3.3.3
+ - name: opensecurity/mobile-security-framework-mobsf:v3.4.0
alias: mobsf
image:
name: "$SAST_ANALYZER_IMAGE"
@@ -303,6 +303,10 @@ semgrep-sast:
$SAST_EXPERIMENTAL_FEATURES == 'true'
exists:
- '**/*.py'
+ - '**/*.js'
+ - '**/*.jsx'
+ - '**/*.ts'
+ - '**/*.tsx'
sobelow-sast:
extends: .sast-analyzer
@@ -348,3 +352,4 @@ spotbugs-sast:
- '**/*.groovy'
- '**/*.java'
- '**/*.scala'
+ - '**/*.kt'
diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
index e591e3cc1e2..404d4a4c6db 100644
--- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
@@ -18,9 +18,32 @@ performance:
- docker:stable-dind
script:
- mkdir gitlab-exporter
+ # Busybox wget does not support proxied HTTPS, get the real thing.
+ # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611.
+ - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget
- wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js
- mkdir sitespeed-results
- - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
+ - |
+ function propagate_env_vars() {
+ CURRENT_ENV=$(printenv)
+
+ for VAR_NAME; do
+ echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
+ done
+ }
+ - |
+ docker run \
+ $(propagate_env_vars \
+ auto_proxy \
+ https_proxy \
+ http_proxy \
+ no_proxy \
+ AUTO_PROXY \
+ HTTPS_PROXY \
+ HTTP_PROXY \
+ NO_PROXY \
+ ) \
+ --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
- mv sitespeed-results/data/performance.json browser-performance.json
artifacts:
paths:
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 3258d965c93..c25c4339c35 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -11,7 +11,7 @@ module Gitlab
LOCK_SLEEP = 0.001.seconds
WATCH_FLAG_TTL = 10.seconds
- UPDATE_FREQUENCY_DEFAULT = 30.seconds
+ UPDATE_FREQUENCY_DEFAULT = 60.seconds
UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds
ArchiveError = Class.new(StandardError)
@@ -93,6 +93,10 @@ module Gitlab
end
end
+ def erase_trace_chunks!
+ job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace
+ end
+
def erase!
##
# Erase the archived trace
@@ -100,7 +104,7 @@ module Gitlab
##
# Erase the live trace
- job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace
+ erase_trace_chunks!
FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace
job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace
ensure
@@ -114,7 +118,11 @@ module Gitlab
end
def update_interval
- being_watched? ? UPDATE_FREQUENCY_WHEN_BEING_WATCHED : UPDATE_FREQUENCY_DEFAULT
+ if being_watched?
+ UPDATE_FREQUENCY_WHEN_BEING_WATCHED
+ else
+ UPDATE_FREQUENCY_DEFAULT
+ end
end
def being_watched!
@@ -176,9 +184,14 @@ module Gitlab
end
def unsafe_archive!
- raise AlreadyArchivedError, 'Could not archive again' if trace_artifact
raise ArchiveError, 'Job is not finished yet' unless job.complete?
+ if trace_artifact
+ unsafe_trace_cleanup! if Feature.enabled?(:erase_traces_from_already_archived_jobs_when_archiving_again, job.project, default_enabled: :yaml)
+
+ raise AlreadyArchivedError, 'Could not archive again'
+ end
+
if job.trace_chunks.any?
Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
archive_stream!(stream)
@@ -197,6 +210,18 @@ module Gitlab
end
end
+ def unsafe_trace_cleanup!
+ return unless trace_artifact
+
+ if trace_artifact.archived_trace_exists?
+ # An archive already exists, so make sure to remove the trace chunks
+ erase_trace_chunks!
+ else
+ # An archive already exists, but its associated file does not, so remove it
+ trace_artifact.destroy!
+ end
+ end
+
def in_write_lock(&blk)
lock_key = "trace:write:lock:#{job.id}"
in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk)
diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb
index e2a54f90ecb..3a62f01e2e3 100644
--- a/lib/gitlab/ci/variables/helpers.rb
+++ b/lib/gitlab/ci/variables/helpers.rb
@@ -23,7 +23,21 @@ module Gitlab
def transform_from_yaml_variables(vars)
return vars.stringify_keys if vars.is_a?(Hash)
- vars.to_a.map { |var| [var[:key].to_s, var[:value]] }.to_h
+ vars.to_a.to_h { |var| [var[:key].to_s, var[:value]] }
+ end
+
+ def inherit_yaml_variables(from:, to:, inheritance:)
+ merge_variables(apply_inheritance(from, inheritance), to)
+ end
+
+ private
+
+ def apply_inheritance(variables, inheritance)
+ case inheritance
+ when true then variables
+ when false then {}
+ when Array then variables.select { |var| inheritance.include?(var[:key]) }
+ end
end
end
end
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index 3459b69bebc..f96a6629849 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -38,11 +38,12 @@ module Gitlab
.map { |job| build_attributes(job[:name]) }
end
- def workflow_attributes
- {
- rules: hash_config.dig(:workflow, :rules),
- yaml_variables: transform_to_yaml_variables(variables)
- }
+ def workflow_rules
+ @workflow_rules ||= hash_config.dig(:workflow, :rules)
+ end
+
+ def root_variables
+ @root_variables ||= transform_to_yaml_variables(variables)
end
def jobs
@@ -68,7 +69,9 @@ module Gitlab
when: job[:when] || 'on_success',
environment: job[:environment_name],
coverage_regex: job[:coverage],
- yaml_variables: transform_to_yaml_variables(job[:variables]),
+ yaml_variables: transform_to_yaml_variables(job[:variables]), # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+ job_variables: transform_to_yaml_variables(job[:job_variables]),
+ root_variables_inheritance: job[:root_variables_inheritance],
needs_attributes: job.dig(:needs, :job),
interruptible: job[:interruptible],
only: job[:only],
@@ -101,7 +104,7 @@ module Gitlab
end
def merged_yaml
- @ci_config&.to_hash&.to_yaml
+ @ci_config&.to_hash&.deep_stringify_keys&.to_yaml
end
def variables_with_data
diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb
index ac0071cdc53..fdff8fb32d3 100644
--- a/lib/gitlab/composer/version_index.rb
+++ b/lib/gitlab/composer/version_index.rb
@@ -28,20 +28,34 @@ module Gitlab
def package_metadata(package)
json = package.composer_metadatum.composer_json
- json.merge('dist' => package_dist(package), 'uid' => package.id, 'version' => package.version)
+ json.merge(
+ 'dist' => package_dist(package),
+ 'source' => package_source(package),
+ 'uid' => package.id,
+ 'version' => package.version
+ )
end
def package_dist(package)
- sha = package.composer_metadatum.target_sha
archive_api_path = api_v4_projects_packages_composer_archives_package_name_path({ id: package.project_id, package_name: package.name, format: '.zip' }, true)
{
'type' => 'zip',
- 'url' => expose_url(archive_api_path) + "?sha=#{sha}",
- 'reference' => sha,
+ 'url' => expose_url(archive_api_path) + "?sha=#{package.composer_target_sha}",
+ 'reference' => package.composer_target_sha,
'shasum' => ''
}
end
+
+ def package_source(package)
+ git_url = package.project.http_url_to_repo
+
+ {
+ 'type' => 'git',
+ 'url' => git_url,
+ 'reference' => package.composer_target_sha
+ }
+ end
end
end
end
diff --git a/lib/gitlab/conan_token.rb b/lib/gitlab/conan_token.rb
index d03997b4158..c3d90aa78fb 100644
--- a/lib/gitlab/conan_token.rb
+++ b/lib/gitlab/conan_token.rb
@@ -7,7 +7,7 @@
module Gitlab
class ConanToken
- HMAC_KEY = 'gitlab-conan-packages'.freeze
+ HMAC_KEY = 'gitlab-conan-packages'
attr_reader :access_token_id, :user_id
diff --git a/lib/gitlab/contributor.rb b/lib/gitlab/contributor.rb
index d74d5a86aa0..c1c270bc9e6 100644
--- a/lib/gitlab/contributor.rb
+++ b/lib/gitlab/contributor.rb
@@ -5,7 +5,9 @@ module Gitlab
attr_accessor :email, :name, :commits, :additions, :deletions
def initialize
- @commits, @additions, @deletions = 0, 0, 0
+ @commits = 0
+ @additions = 0
+ @deletions = 0
end
end
end
diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb
index 4428354642d..c113cebd72f 100644
--- a/lib/gitlab/crypto_helper.rb
+++ b/lib/gitlab/crypto_helper.rb
@@ -16,34 +16,16 @@ module Gitlab
::Digest::SHA256.base64digest("#{value}#{salt}")
end
- def aes256_gcm_encrypt(value, nonce: nil)
- aes256_gcm_encrypt_using_static_nonce(value)
+ def aes256_gcm_encrypt(value, nonce: AES256_GCM_IV_STATIC)
+ encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: nonce))
+ Base64.strict_encode64(encrypted_token)
end
- def aes256_gcm_decrypt(value)
+ def aes256_gcm_decrypt(value, nonce: AES256_GCM_IV_STATIC)
return unless value
- nonce = Feature.enabled?(:dynamic_nonce_creation) ? dynamic_nonce(value) : AES256_GCM_IV_STATIC
encrypted_token = Base64.decode64(value)
- decrypted_token = Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce))
- decrypted_token
- end
-
- def dynamic_nonce(value)
- TokenWithIv.find_nonce_by_hashed_token(value) || AES256_GCM_IV_STATIC
- end
-
- def aes256_gcm_encrypt_using_static_nonce(value)
- create_encrypted_token(value, AES256_GCM_IV_STATIC)
- end
-
- def read_only?
- Gitlab::Database.read_only?
- end
-
- def create_encrypted_token(value, iv)
- encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: iv))
- Base64.strict_encode64(encrypted_token)
+ Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce))
end
end
end
diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb
index c4af5e6608e..0e4fc8efa95 100644
--- a/lib/gitlab/data_builder/build.rb
+++ b/lib/gitlab/data_builder/build.rb
@@ -12,7 +12,7 @@ module Gitlab
author_url = build_author_url(build.commit, commit)
- data = {
+ {
object_kind: 'build',
ref: build.ref,
@@ -26,6 +26,7 @@ module Gitlab
build_name: build.name,
build_stage: build.stage,
build_status: build.status,
+ build_created_at: build.created_at,
build_started_at: build.started_at,
build_finished_at: build.finished_at,
build_duration: build.duration,
@@ -66,8 +67,6 @@ module Gitlab
environment: build_environment(build)
}
-
- data
end
private
@@ -84,7 +83,6 @@ module Gitlab
id: runner.id,
description: runner.description,
active: runner.active?,
- is_shared: runner.instance_type?,
tags: runner.tags&.map(&:name)
}
end
diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb
index 7fd1b9cd228..a56029c0d1d 100644
--- a/lib/gitlab/data_builder/pipeline.rb
+++ b/lib/gitlab/data_builder/pipeline.rb
@@ -77,7 +77,6 @@ module Gitlab
id: runner.id,
description: runner.description,
active: runner.active?,
- is_shared: runner.instance_type?,
tags: runner.tags&.map(&:name)
}
end
diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb
new file mode 100644
index 00000000000..7c45f416638
--- /dev/null
+++ b/lib/gitlab/database/as_with_materialized.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ # This class is a special Arel node which allows optionally define the `MATERIALIZED` keyword for CTE and Recursive CTE queries.
+ class AsWithMaterialized < Arel::Nodes::Binary
+ extend Gitlab::Utils::StrongMemoize
+
+ MATERIALIZED = Arel.sql(' MATERIALIZED')
+ EMPTY_STRING = Arel.sql('')
+ attr_reader :expr
+
+ def initialize(left, right, materialized: true)
+ @expr = if materialized && self.class.materialized_supported?
+ MATERIALIZED
+ else
+ EMPTY_STRING
+ end
+
+ super(left, right)
+ end
+
+ # Note: to be deleted after the minimum PG version is set to 12.0
+ def self.materialized_supported?
+ strong_memoize(:materialized_supported) do
+ Gitlab::Database.version.match?(/^1[2-9]\./) # version 12.x and above
+ end
+ end
+
+ # Note: to be deleted after the minimum PG version is set to 12.0
+ def self.materialized_if_supported
+ materialized_supported? ? 'MATERIALIZED' : ''
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/background_migration/batch_metrics.rb b/lib/gitlab/database/background_migration/batch_metrics.rb
new file mode 100644
index 00000000000..3e6d7ac3c9f
--- /dev/null
+++ b/lib/gitlab/database/background_migration/batch_metrics.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module BackgroundMigration
+ class BatchMetrics
+ attr_reader :timings
+
+ def initialize
+ @timings = {}
+ end
+
+ def time_operation(label)
+ start_time = monotonic_time
+
+ yield
+
+ timings_for_label(label) << monotonic_time - start_time
+ end
+
+ private
+
+ def timings_for_label(label)
+ timings[label] ||= []
+ end
+
+ def monotonic_time
+ Gitlab::Metrics::System.monotonic_time
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb
index 0c9add9b355..4aa33ed7946 100644
--- a/lib/gitlab/database/background_migration/batched_migration.rb
+++ b/lib/gitlab/database/background_migration/batched_migration.rb
@@ -5,7 +5,7 @@ module Gitlab
module BackgroundMigration
class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord
JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration'
- BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies".freeze
+ BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies"
self.table_name = :batched_background_migrations
@@ -23,8 +23,15 @@ module Gitlab
finished: 3
}
- def interval_elapsed?
- last_job.nil? || last_job.created_at <= Time.current - interval
+ def self.active_migration
+ active.queue_order.first
+ end
+
+ def interval_elapsed?(variance: 0)
+ return true unless last_job
+
+ interval_with_variance = interval - variance
+ last_job.created_at <= Time.current - interval_with_variance
end
def create_batched_job!(min, max)
@@ -50,6 +57,13 @@ module Gitlab
def batch_class_name=(class_name)
write_attribute(:batch_class_name, class_name.demodulize)
end
+
+ def prometheus_labels
+ @prometheus_labels ||= {
+ migration_id: id,
+ migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name]
+ }
+ end
end
end
end
diff --git a/lib/gitlab/database/background_migration/scheduler.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb
index 5f8a5ec06a5..cf8b61f5feb 100644
--- a/lib/gitlab/database/background_migration/scheduler.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb
@@ -3,12 +3,22 @@
module Gitlab
module Database
module BackgroundMigration
- class Scheduler
- def perform(migration_wrapper: BatchedMigrationWrapper.new)
- active_migration = BatchedMigration.active.queue_order.first
-
- return unless active_migration&.interval_elapsed?
+ class BatchedMigrationRunner
+ def initialize(migration_wrapper = BatchedMigrationWrapper.new)
+ @migration_wrapper = migration_wrapper
+ end
+ # Runs the next batched_job for a batched_background_migration.
+ #
+ # The batch bounds of the next job are calculated at runtime, based on the migration
+ # configuration and the bounds of the most recently created batched_job. Updating the
+ # migration configuration will cause future jobs to use the updated batch sizes.
+ #
+ # The job instance will automatically receive a set of arguments based on the migration
+ # configuration. For more details, see the BatchedMigrationWrapper class.
+ #
+ # Note that this method is primarily intended to called by a scheduled worker.
+ def run_migration_job(active_migration)
if next_batched_job = create_next_batched_job!(active_migration)
migration_wrapper.perform(next_batched_job)
else
@@ -16,8 +26,26 @@ module Gitlab
end
end
+ # Runs all remaining batched_jobs for a batched_background_migration.
+ #
+ # This method is intended to be used in a test/dev environment to execute the background
+ # migration inline. It should NOT be used in a real environment for any non-trivial migrations.
+ def run_entire_migration(migration)
+ unless Rails.env.development? || Rails.env.test?
+ raise 'this method is not intended for use in real environments'
+ end
+
+ while migration.active?
+ run_migration_job(migration)
+
+ migration.reload_last_job
+ end
+ end
+
private
+ attr_reader :migration_wrapper
+
def create_next_batched_job!(active_migration)
next_batch_range = find_next_batch_range(active_migration)
diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
index 299bd992197..c276f8ce75b 100644
--- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
+++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb
@@ -4,6 +4,15 @@ module Gitlab
module Database
module BackgroundMigration
class BatchedMigrationWrapper
+ extend Gitlab::Utils::StrongMemoize
+
+ # Wraps the execution of a batched_background_migration.
+ #
+ # Updates the job's tracking records with the status of the migration
+ # when starting and finishing execution, and optionally saves batch_metrics
+ # the migration provides, if any are given.
+ #
+ # The job's batch_metrics are serialized to JSON for storage.
def perform(batch_tracking_record)
start_tracking_execution(batch_tracking_record)
@@ -16,6 +25,7 @@ module Gitlab
raise e
ensure
finish_tracking_execution(batch_tracking_record)
+ track_prometheus_metrics(batch_tracking_record)
end
private
@@ -34,12 +44,75 @@ module Gitlab
tracking_record.migration_column_name,
tracking_record.sub_batch_size,
*tracking_record.migration_job_arguments)
+
+ if job_instance.respond_to?(:batch_metrics)
+ tracking_record.metrics = job_instance.batch_metrics
+ end
end
def finish_tracking_execution(tracking_record)
tracking_record.finished_at = Time.current
tracking_record.save!
end
+
+ def track_prometheus_metrics(tracking_record)
+ migration = tracking_record.batched_migration
+ base_labels = migration.prometheus_labels
+
+ metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size)
+ metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size)
+ metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size)
+
+ # Time efficiency: Ratio of duration to interval (ideal: less than, but close to 1)
+ efficiency = (tracking_record.finished_at - tracking_record.started_at).to_i / migration.interval.to_f
+ metric_for(:histogram_time_efficiency).observe(base_labels, efficiency)
+
+ if metrics = tracking_record.metrics
+ metrics['timings']&.each do |key, timings|
+ summary = metric_for(:histogram_timings)
+ labels = base_labels.merge(operation: key)
+
+ timings.each do |timing|
+ summary.observe(labels, timing)
+ end
+ end
+ end
+ end
+
+ def metric_for(name)
+ self.class.metrics[name]
+ end
+
+ def self.metrics
+ strong_memoize(:metrics) do
+ {
+ gauge_batch_size: Gitlab::Metrics.gauge(
+ :batched_migration_job_batch_size,
+ 'Batch size for a batched migration job'
+ ),
+ gauge_sub_batch_size: Gitlab::Metrics.gauge(
+ :batched_migration_job_sub_batch_size,
+ 'Sub-batch size for a batched migration job'
+ ),
+ counter_updated_tuples: Gitlab::Metrics.counter(
+ :batched_migration_job_updated_tuples_total,
+ 'Number of tuples updated by batched migration job'
+ ),
+ histogram_timings: Gitlab::Metrics.histogram(
+ :batched_migration_job_duration_seconds,
+ 'Timings for a batched migration job',
+ {},
+ [0.1, 0.25, 0.5, 1, 5].freeze
+ ),
+ histogram_time_efficiency: Gitlab::Metrics.histogram(
+ :batched_migration_job_time_efficiency,
+ 'Ratio of job duration to interval',
+ {},
+ [0.5, 0.9, 1, 1.5, 2].freeze
+ )
+ }
+ end
+ end
end
end
end
diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb
index 5a506da0d05..9002d39e1ee 100644
--- a/lib/gitlab/database/batch_count.rb
+++ b/lib/gitlab/database/batch_count.rb
@@ -88,11 +88,16 @@ module Gitlab
batch_start = start
while batch_start < finish
- batch_end = [batch_start + batch_size, finish].min
- batch_relation = build_relation_batch(batch_start, batch_end, mode)
-
begin
- results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend
+ batch_end = [batch_start + batch_size, finish].min
+ batch_relation = build_relation_batch(batch_start, batch_end, mode)
+
+ op_args = @operation_args
+ if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode)
+ op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS]
+ end
+
+ results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend
batch_start = batch_end
rescue ActiveRecord::QueryCanceled => error
# retry with a safe batch size & warmer cache
@@ -102,6 +107,18 @@ module Gitlab
log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error)
return FALLBACK
end
+ rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error
+ Gitlab::AppJsonLogger
+ .error(
+ event: 'batch_count',
+ relation: @relation.table_name,
+ operation: @operation,
+ operation_args: @operation_args,
+ mode: mode,
+ message: "LooseIndexScanDistinctCount column error: #{error.message}"
+ )
+
+ return FALLBACK
end
sleep(SLEEP_TIME_IN_SECONDS)
@@ -123,7 +140,11 @@ module Gitlab
private
def build_relation_batch(start, finish, mode)
- @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend
+ if use_loose_index_scan_for_distinct_values?(mode)
+ Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish)
+ else
+ @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend
+ end
end
def batch_size_for_mode_and_operation(mode, operation)
@@ -165,6 +186,14 @@ module Gitlab
message: "Query has been canceled with message: #{error.message}"
)
end
+
+ def use_loose_index_scan_for_distinct_values?(mode)
+ Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct
+ end
+
+ def not_group_by_query?
+ !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank?
+ end
end
end
end
diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb
index 1403d561890..b1f9da30585 100644
--- a/lib/gitlab/database/bulk_update.rb
+++ b/lib/gitlab/database/bulk_update.rb
@@ -130,7 +130,7 @@ module Gitlab
def sql
<<~SQL
- WITH cte(#{list_of(cte_columns)}) AS (VALUES #{list_of(values)})
+ WITH cte(#{list_of(cte_columns)}) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (VALUES #{list_of(values)})
UPDATE #{table_name} SET #{list_of(updates)} FROM cte WHERE cte_id = id
SQL
end
diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb
index 89190320cf9..a7bfafe2815 100644
--- a/lib/gitlab/database/count/reltuples_count_strategy.rb
+++ b/lib/gitlab/database/count/reltuples_count_strategy.rb
@@ -3,10 +3,6 @@
module Gitlab
module Database
module Count
- class PgClass < ActiveRecord::Base
- self.table_name = 'pg_class'
- end
-
# This strategy counts based on PostgreSQL's statistics in pg_stat_user_tables.
#
# Specifically, it relies on the column reltuples in said table. An additional
@@ -74,7 +70,7 @@ module Gitlab
def get_statistics(table_names, check_statistics: true)
time = 6.hours.ago
- query = PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid")
+ query = ::Gitlab::Database::PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid")
.where(relname: table_names)
.where('schemaname = current_schema()')
.select('pg_class.relname AS table_name, reltuples::bigint AS estimate')
diff --git a/lib/gitlab/database/loose_index_scan_distinct_count.rb b/lib/gitlab/database/loose_index_scan_distinct_count.rb
new file mode 100644
index 00000000000..884f4d47ff8
--- /dev/null
+++ b/lib/gitlab/database/loose_index_scan_distinct_count.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ # This class builds efficient batched distinct query by using loose index scan.
+ # Consider the following example:
+ # > Issue.distinct(:project_id).where(project_id: (1...100)).count
+ #
+ # Note: there is an index on project_id
+ #
+ # This query will read each element in the index matching the project_id filter.
+ # If for a project_id has 100_000 issues, all 100_000 elements will be read.
+ #
+ # A loose index scan will read only one entry from the index for each project_id to reduce the number of disk reads.
+ #
+ # Usage:
+ #
+ # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).count(from: 1, to: 100)
+ #
+ # The query will return the number of distinct projects_ids between 1 and 100
+ #
+ # Getting the Arel query:
+ #
+ # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).build_query(from: 1, to: 100)
+ class LooseIndexScanDistinctCount
+ COLUMN_ALIAS = 'distinct_count_column'
+
+ ColumnConfigurationError = Class.new(StandardError)
+
+ def initialize(scope, column)
+ if scope.is_a?(ActiveRecord::Relation)
+ @scope = scope
+ @model = scope.model
+ else
+ @scope = scope.where({})
+ @model = scope
+ end
+
+ @column = transform_column(column)
+ end
+
+ def count(from:, to:)
+ build_query(from: from, to: to).count(COLUMN_ALIAS)
+ end
+
+ def build_query(from:, to:) # rubocop:disable Metrics/AbcSize
+ cte = Gitlab::SQL::RecursiveCTE.new(:counter_cte, union_args: { remove_order: false })
+ table = model.arel_table
+
+ cte << @scope
+ .dup
+ .select(column.as(COLUMN_ALIAS))
+ .where(column.gteq(from))
+ .where(column.lt(to))
+ .order(column)
+ .limit(1)
+
+ inner_query = @scope
+ .dup
+ .where(column.gt(cte.table[COLUMN_ALIAS]))
+ .where(column.lt(to))
+ .select(column.as(COLUMN_ALIAS))
+ .order(column)
+ .limit(1)
+
+ cte << cte.table
+ .project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(COLUMN_ALIAS))
+ .where(cte.table[COLUMN_ALIAS].lt(to))
+
+ model
+ .with
+ .recursive(cte.to_arel)
+ .from(cte.alias_to(table))
+ .unscope(where: :source_type)
+ .unscope(where: model.inheritance_column) # Remove STI query, not needed here
+ end
+
+ private
+
+ attr_reader :column, :model
+
+ # Transforms the column so it can be used in Arel expressions
+ #
+ # 'table.column' => 'table.column'
+ # 'column' => 'table_name.column'
+ # :column => 'table_name.column'
+ # Arel::Attributes::Attribute => name of the column
+ def transform_column(column)
+ if column.is_a?(String) || column.is_a?(Symbol)
+ column_as_string = column.to_s
+ column_as_string = "#{model.table_name}.#{column_as_string}" unless column_as_string.include?('.')
+
+ Arel.sql(column_as_string)
+ elsif column.is_a?(Arel::Attributes::Attribute)
+ column
+ else
+ raise ColumnConfigurationError.new("Cannot transform the column: #{column.inspect}, please provide the column name as string")
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 31e733050e1..d06a73da8ac 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -4,6 +4,7 @@ module Gitlab
module Database
module MigrationHelpers
include Migrations::BackgroundMigrationHelpers
+ include DynamicModelHelpers
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
MAX_IDENTIFIER_NAME_LENGTH = 63
@@ -576,17 +577,7 @@ module Gitlab
# old_column - The name of the old column.
# new_column - The name of the new column.
def install_rename_triggers(table, old_column, new_column)
- trigger_name = rename_trigger_name(table, old_column, new_column)
- quoted_table = quote_table_name(table)
- quoted_old = quote_column_name(old_column)
- quoted_new = quote_column_name(new_column)
-
- install_rename_triggers_for_postgresql(
- trigger_name,
- quoted_table,
- quoted_old,
- quoted_new
- )
+ install_rename_triggers_for_postgresql(table, old_column, new_column)
end
# Changes the type of a column concurrently.
@@ -927,19 +918,67 @@ module Gitlab
# This is crucial for Primary Key conversions, because setting a column
# as the PK converts even check constraints to NOT NULL constraints
# and forces an inline re-verification of the whole table.
- # - It backfills the new column with the values of the existing primary key
- # by scheduling background jobs.
- # - It tracks the scheduled background jobs through the use of
- # Gitlab::Database::BackgroundMigrationJob
+ # - It sets up a trigger to keep the two columns in sync.
+ #
+ # Note: this helper is intended to be used in a regular (pre-deployment) migration.
+ #
+ # This helper is part 1 of a multi-step migration process:
+ # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers
+ # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations
+ # 3. remaining steps TBD, see #288005
+ #
+ # table - The name of the database table containing the column
+ # column - The name of the column that we want to convert to bigint.
+ # primary_key - The name of the primary key column (most often :id)
+ def initialize_conversion_of_integer_to_bigint(table, column, primary_key: :id)
+ unless table_exists?(table)
+ raise "Table #{table} does not exist"
+ end
+
+ unless column_exists?(table, primary_key)
+ raise "Column #{primary_key} does not exist on #{table}"
+ end
+
+ unless column_exists?(table, column)
+ raise "Column #{column} does not exist on #{table}"
+ end
+
+ check_trigger_permissions!(table)
+
+ old_column = column_for(table, column)
+ tmp_column = "#{column}_convert_to_bigint"
+
+ with_lock_retries do
+ if (column.to_s == primary_key.to_s) || !old_column.null
+ # If the column to be converted is either a PK or is defined as NOT NULL,
+ # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow
+ # That way, we skip the expensive validation step required to add
+ # a NOT NULL constraint at the end of the process
+ add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false)
+ else
+ add_column(table, tmp_column, :bigint, default: old_column.default)
+ end
+
+ install_rename_triggers(table, column, tmp_column)
+ end
+ end
+
+ # Backfills the new column used in the conversion of an integer column to bigint using background migrations.
+ #
+ # - This helper should be called from a post-deployment migration.
+ # - In order for this helper to work properly, the new column must be first initialized with
+ # the `initialize_conversion_of_integer_to_bigint` helper.
+ # - It tracks the scheduled background jobs through Gitlab::Database::BackgroundMigration::BatchedMigration,
# which allows a more thorough check that all jobs succeeded in the
# cleanup migration and is way faster for very large tables.
- # - It sets up a trigger to keep the two columns in sync
- # - It does not schedule a cleanup job: we have to do that with followup
- # post deployment migrations in the next release.
#
- # This needs to be done manually by using the
- # `cleanup_initialize_conversion_of_integer_to_bigint`
- # (not yet implemented - check #288005)
+ # Note: this helper is intended to be used in a post-deployment migration, to ensure any new code is
+ # deployed (including background job changes) before we begin processing the background migration.
+ #
+ # This helper is part 2 of a multi-step migration process:
+ # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers
+ # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations
+ # 3. remaining steps TBD, see #288005
#
# table - The name of the database table containing the column
# column - The name of the column that we want to convert to bigint.
@@ -960,7 +999,7 @@ module Gitlab
# and set the batch_size to 50_000 which will require
# ~50s = (50000 / 200) * (0.1 + 0.1) to complete and leaves breathing space
# between the scheduled jobs
- def initialize_conversion_of_integer_to_bigint(
+ def backfill_conversion_of_integer_to_bigint(
table,
column,
primary_key: :id,
@@ -969,10 +1008,6 @@ module Gitlab
interval: 2.minutes
)
- if transaction_open?
- raise 'initialize_conversion_of_integer_to_bigint can not be run inside a transaction'
- end
-
unless table_exists?(table)
raise "Table #{table} does not exist"
end
@@ -985,87 +1020,42 @@ module Gitlab
raise "Column #{column} does not exist on #{table}"
end
- check_trigger_permissions!(table)
-
- old_column = column_for(table, column)
tmp_column = "#{column}_convert_to_bigint"
- with_lock_retries do
- if (column.to_s == primary_key.to_s) || !old_column.null
- # If the column to be converted is either a PK or is defined as NOT NULL,
- # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow
- # That way, we skip the expensive validation step required to add
- # a NOT NULL constraint at the end of the process
- add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false)
- else
- add_column(table, tmp_column, :bigint, default: old_column.default)
- end
-
- install_rename_triggers(table, column, tmp_column)
- end
-
- source_model = Class.new(ActiveRecord::Base) do
- include EachBatch
-
- self.table_name = table
- self.inheritance_column = :_type_disabled
+ unless column_exists?(table, tmp_column)
+ raise 'The temporary column does not exist, initialize it with `initialize_conversion_of_integer_to_bigint`'
end
- queue_background_migration_jobs_by_range_at_intervals(
- source_model,
+ batched_migration = queue_batched_background_migration(
'CopyColumnUsingBackgroundMigrationJob',
- interval,
+ table,
+ primary_key,
+ column,
+ tmp_column,
+ job_interval: interval,
batch_size: batch_size,
- other_job_arguments: [table, primary_key, sub_batch_size, column, tmp_column],
- track_jobs: true,
- primary_column_name: primary_key
- )
+ sub_batch_size: sub_batch_size)
if perform_background_migration_inline?
# To ensure the schema is up to date immediately we perform the
# migration inline in dev / test environments.
- Gitlab::BackgroundMigration.steal('CopyColumnUsingBackgroundMigrationJob')
+ Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(batched_migration)
end
end
# Performs a concurrent column rename when using PostgreSQL.
- def install_rename_triggers_for_postgresql(trigger, table, old, new)
- execute <<-EOF.strip_heredoc
- CREATE OR REPLACE FUNCTION #{trigger}()
- RETURNS trigger AS
- $BODY$
- BEGIN
- NEW.#{new} := NEW.#{old};
- RETURN NEW;
- END;
- $BODY$
- LANGUAGE 'plpgsql'
- VOLATILE
- EOF
-
- execute <<-EOF.strip_heredoc
- DROP TRIGGER IF EXISTS #{trigger}
- ON #{table}
- EOF
-
- execute <<-EOF.strip_heredoc
- CREATE TRIGGER #{trigger}
- BEFORE INSERT OR UPDATE
- ON #{table}
- FOR EACH ROW
- EXECUTE FUNCTION #{trigger}()
- EOF
+ def install_rename_triggers_for_postgresql(table, old, new, trigger_name: nil)
+ Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).create(old, new, trigger_name: trigger_name)
end
# Removes the triggers used for renaming a PostgreSQL column concurrently.
def remove_rename_triggers_for_postgresql(table, trigger)
- execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}")
- execute("DROP FUNCTION IF EXISTS #{trigger}()")
+ Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).drop(trigger)
end
# Returns the (base) name to use for triggers when renaming columns.
def rename_trigger_name(table, old, new)
- 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12)
+ Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).name(old, new)
end
# Returns an Array containing the indexes for the given column
diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb
index e8cbea72887..8d5ea652bfc 100644
--- a/lib/gitlab/database/migrations/background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/background_migration_helpers.rb
@@ -190,7 +190,7 @@ module Gitlab
migration_status = batch_max_value.nil? ? :finished : :active
batch_max_value ||= batch_min_value
- Gitlab::Database::BackgroundMigration::BatchedMigration.create!(
+ migration = Gitlab::Database::BackgroundMigration::BatchedMigration.create!(
job_class_name: job_class_name,
table_name: batch_table_name,
column_name: batch_column_name,
@@ -202,6 +202,17 @@ module Gitlab
sub_batch_size: sub_batch_size,
job_arguments: job_arguments,
status: migration_status)
+
+ # This guard is necessary since #total_tuple_count was only introduced schema-wise,
+ # after this migration helper had been used for the first time.
+ return migration unless migration.respond_to?(:total_tuple_count)
+
+ # We keep track of the estimated number of tuples to reason later
+ # about the overall progress of a migration.
+ migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate
+ migration.save!
+
+ migration
end
def perform_background_migration_inline?
@@ -236,6 +247,14 @@ module Gitlab
Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block)
end
+ def delete_queued_jobs(class_name)
+ Gitlab::BackgroundMigration.steal(class_name) do |job|
+ job.delete
+
+ false
+ end
+ end
+
private
def track_in_database(class_name, arguments)
diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
index 2def3a4d3a9..4402c42b136 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
@@ -6,6 +6,80 @@ module Gitlab
module ForeignKeyHelpers
include ::Gitlab::Database::SchemaHelpers
+ # Adds a foreign key with only minimal locking on the tables involved.
+ #
+ # In concept it works similarly to add_concurrent_foreign_key, but we have
+ # to add a special helper for partitioned tables for the following reasons:
+ # - add_concurrent_foreign_key sets the constraint to `NOT VALID`
+ # before validating it
+ # - Setting an FK to NOT VALID is not supported currently in Postgres (up to PG13)
+ # - Also, PostgreSQL will currently ignore NOT VALID constraints on partitions
+ # when adding a valid FK to the partitioned table, so they have to
+ # also be validated before we can add the final FK.
+ # Solution:
+ # - Add the foreign key first to each partition by using
+ # add_concurrent_foreign_key and validating it
+ # - Once all partitions have a foreign key, add it also to the partitioned
+ # table (there will be no need for a validation at that level)
+ # For those reasons, this method does not include an option to delay the
+ # validation, we have to force validate: true.
+ #
+ # source - The source (partitioned) table containing the foreign key.
+ # target - The target table the key points to.
+ # column - The name of the column to create the foreign key on.
+ # on_delete - The action to perform when associated data is removed,
+ # defaults to "CASCADE".
+ # name - The name of the foreign key.
+ #
+ def add_concurrent_partitioned_foreign_key(source, target, column:, on_delete: :cascade, name: nil)
+ partition_options = {
+ column: column,
+ on_delete: on_delete,
+
+ # We'll use the same FK name for all partitions and match it to
+ # the name used for the partitioned table to follow the convention
+ # used by PostgreSQL when adding FKs to new partitions
+ name: name.presence || concurrent_partitioned_foreign_key_name(source, column),
+
+ # Force the FK validation to true for partitions (and the partitioned table)
+ validate: true
+ }
+
+ if foreign_key_exists?(source, target, **partition_options)
+ warning_message = "Foreign key not created because it exists already " \
+ "(this may be due to an aborted migration or similar): " \
+ "source: #{source}, target: #{target}, column: #{partition_options[:column]}, "\
+ "name: #{partition_options[:name]}, on_delete: #{partition_options[:on_delete]}"
+
+ Gitlab::AppLogger.warn warning_message
+
+ return
+ end
+
+ partitioned_table = find_partitioned_table(source)
+
+ partitioned_table.postgres_partitions.order(:name).each do |partition|
+ add_concurrent_foreign_key(partition.identifier, target, **partition_options)
+ end
+
+ with_lock_retries do
+ add_foreign_key(source, target, **partition_options)
+ end
+ end
+
+ # Returns the name for a concurrent partitioned foreign key.
+ #
+ # Similar to concurrent_foreign_key_name (Gitlab::Database::MigrationHelpers)
+ # we just keep a separate method in case we want a different behavior
+ # for partitioned tables
+ #
+ def concurrent_partitioned_foreign_key_name(table, column, prefix: 'fk_rails_')
+ identifier = "#{table}_#{column}_fk"
+ hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10)
+
+ "#{prefix}#{hashed_identifier}"
+ end
+
# Creates a "foreign key" that references a partitioned table. Because foreign keys referencing partitioned
# tables are not supported in PG11, this does not create a true database foreign key, but instead implements the
# same functionality at the database level by using triggers.
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index 1c289391e21..9ccbdc9930e 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -223,6 +223,28 @@ module Gitlab
replace_table(table_name, archived_table_name, partitioned_table_name, primary_key_name)
end
+ def drop_nonpartitioned_archive_table(table_name)
+ assert_table_is_allowed(table_name)
+
+ archived_table_name = make_archived_table_name(table_name)
+
+ with_lock_retries do
+ drop_sync_trigger(table_name)
+ end
+
+ drop_table(archived_table_name)
+ end
+
+ def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key)
+ function_name = make_sync_function_name(source_table_name)
+ trigger_name = make_sync_trigger_name(source_table_name)
+
+ create_sync_function(function_name, partitioned_table_name, unique_key)
+ create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table")
+
+ create_sync_trigger(source_table_name, trigger_name, function_name)
+ end
+
private
def assert_table_is_allowed(table_name)
@@ -316,16 +338,6 @@ module Gitlab
create_range_partition(partition_name, table_name, lower_bound, upper_bound)
end
- def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key)
- function_name = make_sync_function_name(source_table_name)
- trigger_name = make_sync_trigger_name(source_table_name)
-
- create_sync_function(function_name, partitioned_table_name, unique_key)
- create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table")
-
- create_sync_trigger(source_table_name, trigger_name, function_name)
- end
-
def drop_sync_trigger(source_table_name)
trigger_name = make_sync_trigger_name(source_table_name)
drop_trigger(source_table_name, trigger_name)
diff --git a/lib/gitlab/database/pg_class.rb b/lib/gitlab/database/pg_class.rb
new file mode 100644
index 00000000000..0ce9eebc14c
--- /dev/null
+++ b/lib/gitlab/database/pg_class.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class PgClass < ActiveRecord::Base
+ self.table_name = 'pg_class'
+
+ def self.for_table(relname)
+ joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid")
+ .where('schemaname = current_schema()')
+ .find_by(relname: relname)
+ end
+
+ def cardinality_estimate
+ tuples = reltuples.to_i
+
+ return if tuples < 1
+
+ tuples
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
index 62dfaeeaae3..e8b49c7f62c 100644
--- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
+++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb
@@ -41,19 +41,6 @@ module Gitlab
BUCKET_ID_MASK = (Buckets::TOTAL_BUCKETS - ZERO_OFFSET).to_s(2)
BIT_31_MASK = "B'0#{'1' * 31}'"
BIT_32_NORMALIZED_BUCKET_ID_MASK = "B'#{'0' * (32 - BUCKET_ID_MASK.size)}#{BUCKET_ID_MASK}'"
- # @example source_query
- # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits
- # FROM %{relation}
- # WHERE %{pkey} >= %{batch_start}
- # AND %{pkey} < %{batch_end}
- # AND %{column} IS NOT NULL
- BUCKETED_DATA_SQL = <<~SQL
- WITH hashed_attributes AS (%{source_query})
- SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num,
- (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash
- FROM hashed_attributes
- GROUP BY 1
- SQL
WRONG_CONFIGURATION_ERROR = Class.new(ActiveRecord::StatementInvalid)
@@ -103,7 +90,7 @@ module Gitlab
def hll_buckets_for_batch(start, finish)
@relation
.connection
- .execute(BUCKETED_DATA_SQL % { source_query: source_query(start, finish) })
+ .execute(bucketed_data_sql % { source_query: source_query(start, finish) })
.map(&:values)
.to_h
end
@@ -139,6 +126,22 @@ module Gitlab
def actual_finish(finish)
finish || @relation.unscope(:group, :having).maximum(@relation.primary_key) || 0
end
+
+ # @example source_query
+ # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits
+ # FROM %{relation}
+ # WHERE %{pkey} >= %{batch_start}
+ # AND %{pkey} < %{batch_end}
+ # AND %{column} IS NOT NULL
+ def bucketed_data_sql
+ <<~SQL
+ WITH hashed_attributes AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (%{source_query})
+ SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num,
+ (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash
+ FROM hashed_attributes
+ GROUP BY 1
+ SQL
+ end
end
end
end
diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb
index 40845c0d5e0..20bf6fa4d30 100644
--- a/lib/gitlab/database/similarity_score.rb
+++ b/lib/gitlab/database/similarity_score.rb
@@ -10,7 +10,7 @@ module Gitlab
# Adds a "magic" comment in the generated SQL expression in order to be able to tell if we're sorting by similarity.
# Example: /* gitlab/database/similarity_score */ SIMILARITY(COALESCE...
- SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY".freeze
+ SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY"
# This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity.
#
diff --git a/lib/gitlab/database/unidirectional_copy_trigger.rb b/lib/gitlab/database/unidirectional_copy_trigger.rb
new file mode 100644
index 00000000000..029c894a5ff
--- /dev/null
+++ b/lib/gitlab/database/unidirectional_copy_trigger.rb
@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class UnidirectionalCopyTrigger
+ def self.on_table(table_name, connection: ActiveRecord::Base.connection)
+ new(table_name, connection)
+ end
+
+ def name(from_column_names, to_column_names)
+ from_column_names, to_column_names = check_column_names!(from_column_names, to_column_names)
+
+ unchecked_name(from_column_names, to_column_names)
+ end
+
+ def create(from_column_names, to_column_names, trigger_name: nil)
+ from_column_names, to_column_names = check_column_names!(from_column_names, to_column_names)
+ trigger_name ||= unchecked_name(from_column_names, to_column_names)
+
+ assignment_clauses = assignment_clauses_for_columns(from_column_names, to_column_names)
+
+ connection.execute(<<~SQL)
+ CREATE OR REPLACE FUNCTION #{trigger_name}()
+ RETURNS trigger AS
+ $BODY$
+ BEGIN
+ #{assignment_clauses};
+ RETURN NEW;
+ END;
+ $BODY$
+ LANGUAGE 'plpgsql'
+ VOLATILE
+ SQL
+
+ connection.execute(<<~SQL)
+ DROP TRIGGER IF EXISTS #{trigger_name}
+ ON #{quoted_table_name}
+ SQL
+
+ connection.execute(<<~SQL)
+ CREATE TRIGGER #{trigger_name}
+ BEFORE INSERT OR UPDATE
+ ON #{quoted_table_name}
+ FOR EACH ROW
+ EXECUTE FUNCTION #{trigger_name}()
+ SQL
+ end
+
+ def drop(trigger_name)
+ connection.execute("DROP TRIGGER IF EXISTS #{trigger_name} ON #{quoted_table_name}")
+ connection.execute("DROP FUNCTION IF EXISTS #{trigger_name}()")
+ end
+
+ private
+
+ attr_reader :table_name, :connection
+
+ def initialize(table_name, connection)
+ @table_name = table_name
+ @connection = connection
+ end
+
+ def quoted_table_name
+ @quoted_table_name ||= connection.quote_table_name(table_name)
+ end
+
+ def check_column_names!(from_column_names, to_column_names)
+ from_column_names = Array.wrap(from_column_names)
+ to_column_names = Array.wrap(to_column_names)
+
+ unless from_column_names.size == to_column_names.size
+ raise ArgumentError, 'number of source and destination columns must match'
+ end
+
+ [from_column_names, to_column_names]
+ end
+
+ def unchecked_name(from_column_names, to_column_names)
+ joined_column_names = from_column_names.zip(to_column_names).flatten.join('_')
+ 'trigger_' + Digest::SHA256.hexdigest("#{table_name}_#{joined_column_names}").first(12)
+ end
+
+ def assignment_clauses_for_columns(from_column_names, to_column_names)
+ combined_column_names = to_column_names.zip(from_column_names)
+
+ assignment_clauses = combined_column_names.map do |(new_name, old_name)|
+ new_name = connection.quote_column_name(new_name)
+ old_name = connection.quote_column_name(old_name)
+
+ "NEW.#{new_name} := NEW.#{old_name}"
+ end
+
+ assignment_clauses.join(";\n ")
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb
index baa46e7e306..8385bbbb3de 100644
--- a/lib/gitlab/diff/highlight.rb
+++ b/lib/gitlab/diff/highlight.rb
@@ -3,7 +3,7 @@
module Gitlab
module Diff
class Highlight
- attr_reader :diff_file, :diff_lines, :raw_lines, :repository, :project
+ attr_reader :diff_file, :diff_lines, :repository, :project
delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff
@@ -22,29 +22,15 @@ module Gitlab
end
def highlight
- @diff_lines.map.with_index do |diff_line, i|
+ populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml)
+
+ @diff_lines.map.with_index do |diff_line, index|
diff_line = diff_line.dup
# ignore highlighting for "match" lines
next diff_line if diff_line.meta?
- rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text)
-
- if line_inline_diffs = inline_diffs[i]
- begin
- # MarkerRange objects are converted to Ranges to keep the previous behavior
- # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068
- if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml)
- line_inline_diffs = line_inline_diffs.map { |marker_range| marker_range.to_range }
- end
-
- rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs)
- # This should only happen when the encoding of the diff doesn't
- # match the blob, which is a bug. But we shouldn't fail to render
- # completely in that case, even though we want to report the error.
- rescue RangeError => e
- Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441')
- end
- end
+ rich_line = apply_syntax_highlight(diff_line)
+ rich_line = apply_marker_ranges_highlight(diff_line, rich_line, index)
diff_line.rich_text = rich_line
@@ -54,9 +40,87 @@ module Gitlab
private
+ def populate_marker_ranges
+ pair_selector = Gitlab::Diff::PairSelector.new(@raw_lines)
+
+ pair_selector.each do |old_index, new_index|
+ old_line = diff_lines[old_index]
+ new_line = diff_lines[new_index]
+
+ old_diffs, new_diffs = Gitlab::Diff::InlineDiff.new(old_line.text, new_line.text, offset: 1).inline_diffs
+
+ old_line.set_marker_ranges(old_diffs)
+ new_line.set_marker_ranges(new_diffs)
+ end
+ end
+
+ def apply_syntax_highlight(diff_line)
+ highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text)
+ end
+
+ def apply_marker_ranges_highlight(diff_line, rich_line, index)
+ marker_ranges = if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml)
+ diff_line.marker_ranges
+ else
+ inline_diffs[index]
+ end
+
+ return rich_line if marker_ranges.blank?
+
+ begin
+ # MarkerRange objects are converted to Ranges to keep the previous behavior
+ # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068
+ if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml)
+ marker_ranges = marker_ranges.map { |marker_range| marker_range.to_range }
+ end
+
+ InlineDiffMarker.new(diff_line.text, rich_line).mark(marker_ranges)
+ # This should only happen when the encoding of the diff doesn't
+ # match the blob, which is a bug. But we shouldn't fail to render
+ # completely in that case, even though we want to report the error.
+ rescue RangeError => e
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441')
+ end
+ end
+
def highlight_line(diff_line)
return unless diff_file && diff_file.diff_refs
+ if Feature.enabled?(:diff_line_syntax_highlighting, project, default_enabled: :yaml)
+ diff_line_highlighting(diff_line)
+ else
+ blob_highlighting(diff_line)
+ end
+ end
+
+ def diff_line_highlighting(diff_line)
+ rich_line = syntax_highlighter(diff_line).highlight(
+ diff_line.text(prefix: false),
+ context: { line_number: diff_line.line }
+ )&.html_safe
+
+ # Only update text if line is found. This will prevent
+ # issues with submodules given the line only exists in diff content.
+ if rich_line
+ line_prefix = diff_line.text =~ /\A(.)/ ? Regexp.last_match(1) : ' '
+ rich_line.prepend(line_prefix).concat("\n")
+ end
+ end
+
+ def syntax_highlighter(diff_line)
+ path = diff_line.removed? ? diff_file.old_path : diff_file.new_path
+
+ @syntax_highlighter ||= {}
+ @syntax_highlighter[path] ||= Gitlab::Highlight.new(
+ path,
+ @raw_lines,
+ language: repository&.gitattribute(path, 'gitlab-language')
+ )
+ end
+
+ # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324159
+ # ------------------------------------------------------------------------
+ def blob_highlighting(diff_line)
rich_line =
if diff_line.unchanged? || diff_line.added?
new_lines[diff_line.new_pos - 1]&.html_safe
@@ -72,6 +136,8 @@ module Gitlab
end
end
+ # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638
+ # ------------------------------------------------------------------------
def inline_diffs
@inline_diffs ||= InlineDiff.for_lines(@raw_lines)
end
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index c5e9bfdc321..209462fd6e9 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -71,9 +71,12 @@ module Gitlab
strong_memoize(:redis_key) do
[
'highlighted-diff-files',
- diffable.cache_key, VERSION,
+ diffable.cache_key,
+ VERSION,
diff_options,
- Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml)
+ Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml),
+ Feature.enabled?(:use_marker_ranges, diffable.project, default_enabled: :yaml),
+ Feature.enabled?(:diff_line_syntax_highlighting, diffable.project, default_enabled: :yaml)
].join(":")
end
end
diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb
index dd73e4d6c15..f70618195d0 100644
--- a/lib/gitlab/diff/inline_diff.rb
+++ b/lib/gitlab/diff/inline_diff.rb
@@ -18,6 +18,7 @@ module Gitlab
CharDiff.new(old_line, new_line).changed_ranges(offset: offset)
end
+ # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638
class << self
def for_lines(lines)
pair_selector = Gitlab::Diff::PairSelector.new(lines)
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 98ed2400d82..6cf414e29cc 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -8,19 +8,24 @@ module Gitlab
#
SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze
- attr_reader :line_code
- attr_writer :rich_text
- attr_accessor :text, :index, :type, :old_pos, :new_pos
+ attr_reader :line_code, :marker_ranges
+ attr_writer :text, :rich_text
+ attr_accessor :index, :type, :old_pos, :new_pos
def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil)
- @text, @type, @index = text, type, index
- @old_pos, @new_pos = old_pos, new_pos
+ @text = text
+ @type = type
+ @index = index
+ @old_pos = old_pos
+ @new_pos = new_pos
@parent_file = parent_file
@rich_text = rich_text
# When line code is not provided from cache store we build it
# using the parent_file(Diff::File or Conflict::File).
@line_code = line_code || calculate_line_code
+
+ @marker_ranges = []
end
def self.init_from_hash(hash)
@@ -48,6 +53,16 @@ module Gitlab
hash
end
+ def set_marker_ranges(marker_ranges)
+ @marker_ranges = marker_ranges
+ end
+
+ def text(prefix: true)
+ return @text if prefix
+
+ @text&.slice(1..).to_s
+ end
+
def old_line
old_pos unless added? || meta?
end
diff --git a/lib/gitlab/diff/suggestions_parser.rb b/lib/gitlab/diff/suggestions_parser.rb
index 6e17ffaf6ff..f3e6fc455ac 100644
--- a/lib/gitlab/diff/suggestions_parser.rb
+++ b/lib/gitlab/diff/suggestions_parser.rb
@@ -17,7 +17,7 @@ module Gitlab
no_original_data: true,
suggestions_filter_enabled: supports_suggestion)
doc = Nokogiri::HTML(html)
- suggestion_nodes = doc.search('pre.suggestion')
+ suggestion_nodes = doc.search('pre.language-suggestion')
return [] if suggestion_nodes.empty?
@@ -29,9 +29,8 @@ module Gitlab
lines_above, lines_below = nil
if lang_param && suggestion_params = fetch_suggestion_params(lang_param)
- lines_above, lines_below =
- suggestion_params[:above],
- suggestion_params[:below]
+ lines_above = suggestion_params[:above]
+ lines_below = suggestion_params[:below]
end
Gitlab::Diff::Suggestion.new(node.text,
diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb
deleted file mode 100644
index 457a3c12206..00000000000
--- a/lib/gitlab/downtime_check.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- # Checks if a set of migrations requires downtime or not.
- class DowntimeCheck
- # The constant containing the boolean that indicates if downtime is needed
- # or not.
- DOWNTIME_CONST = :DOWNTIME
-
- # The constant that specifies the reason for the migration requiring
- # downtime.
- DOWNTIME_REASON_CONST = :DOWNTIME_REASON
-
- # Checks the given migration paths and returns an Array of
- # `Gitlab::DowntimeCheck::Message` instances.
- #
- # migrations - The migration file paths to check.
- def check(migrations)
- migrations.map do |path|
- require(path)
-
- migration_class = class_for_migration_file(path)
-
- unless migration_class.const_defined?(DOWNTIME_CONST)
- raise "The migration in #{path} does not specify if it requires " \
- "downtime or not"
- end
-
- if online?(migration_class)
- Message.new(path)
- else
- reason = downtime_reason(migration_class)
-
- unless reason
- raise "The migration in #{path} requires downtime but no reason " \
- "was given"
- end
-
- Message.new(path, true, reason)
- end
- end
- end
-
- # Checks the given migrations and prints the results to STDOUT/STDERR.
- #
- # migrations - The migration file paths to check.
- def check_and_print(migrations)
- check(migrations).each do |message|
- puts message.to_s # rubocop: disable Rails/Output
- end
- end
-
- # Returns the class for the given migration file path.
- def class_for_migration_file(path)
- File.basename(path, File.extname(path)).split('_', 2).last.camelize
- .constantize
- end
-
- # Returns true if the given migration can be performed without downtime.
- def online?(migration)
- migration.const_get(DOWNTIME_CONST, false) == false
- end
-
- # Returns the downtime reason, or nil if none was defined.
- def downtime_reason(migration)
- if migration.const_defined?(DOWNTIME_REASON_CONST)
- migration.const_get(DOWNTIME_REASON_CONST, false)
- else
- nil
- end
- end
- end
-end
diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb
deleted file mode 100644
index 5debb754943..00000000000
--- a/lib/gitlab/downtime_check/message.rb
+++ /dev/null
@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- class DowntimeCheck
- class Message
- attr_reader :path, :offline
-
- OFFLINE = "\e[31moffline\e[0m"
- ONLINE = "\e[32monline\e[0m"
-
- # path - The file path of the migration.
- # offline - When set to `true` the migration will require downtime.
- # reason - The reason as to why the migration requires downtime.
- def initialize(path, offline = false, reason = nil)
- @path = path
- @offline = offline
- @reason = reason
- end
-
- def to_s
- label = offline ? OFFLINE : ONLINE
-
- message = ["[#{label}]: #{path}"]
-
- if reason?
- message << ":\n\n#{reason}\n\n"
- end
-
- message.join
- end
-
- def reason?
- @reason.present?
- end
-
- def reason
- @reason.strip.lines.map(&:strip).join("\n")
- end
- end
- end
-end
diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb
index dfed8db8df0..47d361fb95c 100644
--- a/lib/gitlab/error_tracking.rb
+++ b/lib/gitlab/error_tracking.rb
@@ -16,6 +16,12 @@ module Gitlab
Rack::Timeout::RequestTimeoutException
].freeze
+ PROCESSORS = [
+ ::Gitlab::ErrorTracking::Processor::SidekiqProcessor,
+ ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor,
+ ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor
+ ].freeze
+
class << self
def configure
Raven.configure do |config|
@@ -97,7 +103,9 @@ module Gitlab
inject_context_for_exception(event, hint[:exception])
custom_fingerprinting(event, hint[:exception])
- event
+ PROCESSORS.reduce(event) do |processed_event, processor|
+ processor.call(processed_event)
+ end
end
def process_exception(exception, sentry: false, logging: true, extra:)
diff --git a/lib/gitlab/error_tracking/processor/context_payload_processor.rb b/lib/gitlab/error_tracking/processor/context_payload_processor.rb
index 5185205e94e..758f6aa11d7 100644
--- a/lib/gitlab/error_tracking/processor/context_payload_processor.rb
+++ b/lib/gitlab/error_tracking/processor/context_payload_processor.rb
@@ -9,9 +9,21 @@ module Gitlab
# integrations are re-implemented and use Gitlab::ErrorTracking, this
# processor should be removed.
def process(payload)
+ return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {})
payload.deep_merge!(context_payload)
end
+
+ def self.call(event)
+ return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
+ Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}).each do |key, value|
+ event.public_send(key).deep_merge!(value) # rubocop:disable GitlabSecurity/PublicSend
+ end
+
+ event
+ end
end
end
end
diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb
index 871e9c4b7c8..419098dbd09 100644
--- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb
+++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb
@@ -6,60 +6,126 @@ module Gitlab
class GrpcErrorProcessor < ::Raven::Processor
DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:(.*)')
- def process(value)
- process_first_exception_value(value)
- process_custom_fingerprint(value)
+ def process(payload)
+ return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
- value
- end
-
- # Sentry can report multiple exceptions in an event. Sanitize
- # only the first one since that's what is used for grouping.
- def process_first_exception_value(value)
- exceptions = value.dig(:exception, :values)
-
- return unless exceptions.is_a?(Array)
-
- entry = exceptions.first
-
- return unless entry.is_a?(Hash)
-
- exception_type = entry[:type]
- raw_message = entry[:value]
-
- return unless exception_type&.start_with?('GRPC::')
- return unless raw_message.present?
-
- message, debug_str = split_debug_error_string(raw_message)
-
- entry[:value] = message if message
- extra = value[:extra] || {}
- extra[:grpc_debug_error_string] = debug_str if debug_str
- end
-
- def process_custom_fingerprint(value)
- fingerprint = value[:fingerprint]
-
- return value unless custom_grpc_fingerprint?(fingerprint)
+ self.class.process_first_exception_value(payload)
+ self.class.process_custom_fingerprint(payload)
- message, _ = split_debug_error_string(fingerprint[1])
- fingerprint[1] = message if message
+ payload
end
- private
-
- def custom_grpc_fingerprint?(fingerprint)
- fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::')
- end
-
- def split_debug_error_string(message)
- return unless message
-
- match = DEBUG_ERROR_STRING_REGEX.match(message)
-
- return unless match
-
- [match[1], match[2]]
+ class << self
+ def call(event)
+ return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
+ process_first_exception_value(event)
+ process_custom_fingerprint(event)
+
+ event
+ end
+
+ # Sentry can report multiple exceptions in an event. Sanitize
+ # only the first one since that's what is used for grouping.
+ def process_first_exception_value(event_or_payload)
+ exceptions = exceptions(event_or_payload)
+
+ return unless exceptions.is_a?(Array)
+
+ exception = exceptions.first
+
+ return unless valid_exception?(exception)
+
+ exception_type, raw_message = type_and_value(exception)
+
+ return unless exception_type&.start_with?('GRPC::')
+ return unless raw_message.present?
+
+ message, debug_str = split_debug_error_string(raw_message)
+
+ set_new_values!(event_or_payload, exception, message, debug_str)
+ end
+
+ def process_custom_fingerprint(event)
+ fingerprint = fingerprint(event)
+
+ return event unless custom_grpc_fingerprint?(fingerprint)
+
+ message, _ = split_debug_error_string(fingerprint[1])
+ fingerprint[1] = message if message
+ end
+
+ private
+
+ def custom_grpc_fingerprint?(fingerprint)
+ fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::')
+ end
+
+ def split_debug_error_string(message)
+ return unless message
+
+ match = DEBUG_ERROR_STRING_REGEX.match(message)
+
+ return unless match
+
+ [match[1], match[2]]
+ end
+
+ # The below methods can be removed once we remove the
+ # sentry_processors_before_send feature flag, and we can
+ # assume we always have an Event object
+ def exceptions(event_or_payload)
+ case event_or_payload
+ when Raven::Event
+ # Better in new version, will be event_or_payload.exception.values
+ event_or_payload.instance_variable_get(:@interfaces)[:exception]&.values
+ when Hash
+ event_or_payload.dig(:exception, :values)
+ end
+ end
+
+ def valid_exception?(exception)
+ case exception
+ when Raven::SingleExceptionInterface
+ exception&.value
+ when Hash
+ true
+ else
+ false
+ end
+ end
+
+ def type_and_value(exception)
+ case exception
+ when Raven::SingleExceptionInterface
+ [exception.type, exception.value]
+ when Hash
+ exception.values_at(:type, :value)
+ end
+ end
+
+ def set_new_values!(event_or_payload, exception, message, debug_str)
+ case event_or_payload
+ when Raven::Event
+ # Worse in new version, no setter! Have to poke at the
+ # instance variable
+ exception.value = message if message
+ event_or_payload.extra[:grpc_debug_error_string] = debug_str if debug_str
+ when Hash
+ exception[:value] = message if message
+ extra = event_or_payload[:extra] || {}
+ extra[:grpc_debug_error_string] = debug_str if debug_str
+ end
+ end
+
+ def fingerprint(event_or_payload)
+ case event_or_payload
+ when Raven::Event
+ event_or_payload.fingerprint
+ when Hash
+ event_or_payload[:fingerprint]
+ end
+ end
end
end
end
diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb
index 272cb689ad5..93310745ece 100644
--- a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb
+++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb
@@ -8,39 +8,66 @@ module Gitlab
class SidekiqProcessor < ::Raven::Processor
FILTERED_STRING = '[FILTERED]'
- def self.filter_arguments(args, klass)
- args.lazy.with_index.map do |arg, i|
- case arg
- when Numeric
- arg
- else
- if permitted_arguments_for_worker(klass).include?(i)
+ class << self
+ def filter_arguments(args, klass)
+ args.lazy.with_index.map do |arg, i|
+ case arg
+ when Numeric
arg
else
- FILTERED_STRING
+ if permitted_arguments_for_worker(klass).include?(i)
+ arg
+ else
+ FILTERED_STRING
+ end
end
end
end
- end
- def self.permitted_arguments_for_worker(klass)
- @permitted_arguments_for_worker ||= {}
- @permitted_arguments_for_worker[klass] ||=
- begin
- klass.constantize&.loggable_arguments&.to_set
- rescue
- Set.new
+ def permitted_arguments_for_worker(klass)
+ @permitted_arguments_for_worker ||= {}
+ @permitted_arguments_for_worker[klass] ||=
+ begin
+ klass.constantize&.loggable_arguments&.to_set
+ rescue
+ Set.new
+ end
+ end
+
+ def loggable_arguments(args, klass)
+ Gitlab::Utils::LogLimitedArray
+ .log_limited_array(filter_arguments(args, klass))
+ .map(&:to_s)
+ .to_a
+ end
+
+ def call(event)
+ return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
+ sidekiq = event&.extra&.dig(:sidekiq)
+
+ return event unless sidekiq
+
+ sidekiq = sidekiq.deep_dup
+ sidekiq.delete(:jobstr)
+
+ # 'args' in this hash => from Gitlab::ErrorTracking.track_*
+ # 'args' in :job => from default error handler
+ job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job]
+
+ if job_holder['args']
+ job_holder['args'] = filter_arguments(job_holder['args'], job_holder['class']).to_a
end
- end
- def self.loggable_arguments(args, klass)
- Gitlab::Utils::LogLimitedArray
- .log_limited_array(filter_arguments(args, klass))
- .map(&:to_s)
- .to_a
+ event.extra[:sidekiq] = sidekiq
+
+ event
+ end
end
def process(value, key = nil)
+ return value if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml)
+
sidekiq = value.dig(:extra, :sidekiq)
return value unless sidekiq
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index b602393b59e..ef0236f8275 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -15,14 +15,14 @@ module Gitlab
PREFIX = 'gitlab:exclusive_lease'
NoKey = Class.new(ArgumentError)
- LUA_CANCEL_SCRIPT = <<~EOS.freeze
+ LUA_CANCEL_SCRIPT = <<~EOS
local key, uuid = KEYS[1], ARGV[1]
if redis.call("get", key) == uuid then
redis.call("del", key)
end
EOS
- LUA_RENEW_SCRIPT = <<~EOS.freeze
+ LUA_RENEW_SCRIPT = <<~EOS
local key, uuid, ttl = KEYS[1], ARGV[1], ARGV[2]
if redis.call("get", key) == uuid then
redis.call("expire", key, ttl)
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
index 1bb29ba3eac..145bb6d7b8f 100644
--- a/lib/gitlab/experimentation.rb
+++ b/lib/gitlab/experimentation.rb
@@ -34,10 +34,6 @@
module Gitlab
module Experimentation
EXPERIMENTS = {
- upgrade_link_in_user_menu_a: {
- tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA',
- use_backwards_compatible_subject_index: true
- },
invite_members_version_b: {
tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB',
use_backwards_compatible_subject_index: true
diff --git a/lib/gitlab/external_authorization/access.rb b/lib/gitlab/external_authorization/access.rb
index e111c41fcc2..21fa728fd3a 100644
--- a/lib/gitlab/external_authorization/access.rb
+++ b/lib/gitlab/external_authorization/access.rb
@@ -10,7 +10,8 @@ module Gitlab
:load_type
def initialize(user, label)
- @user, @label = user, label
+ @user = user
+ @label = label
end
def loaded?
diff --git a/lib/gitlab/external_authorization/cache.rb b/lib/gitlab/external_authorization/cache.rb
index acdc028b4dc..509daeb0248 100644
--- a/lib/gitlab/external_authorization/cache.rb
+++ b/lib/gitlab/external_authorization/cache.rb
@@ -6,7 +6,8 @@ module Gitlab
VALIDITY_TIME = 6.hours
def initialize(user, label)
- @user, @label = user, label
+ @user = user
+ @label = label
end
def load
diff --git a/lib/gitlab/external_authorization/client.rb b/lib/gitlab/external_authorization/client.rb
index fc859304eab..582051010d3 100644
--- a/lib/gitlab/external_authorization/client.rb
+++ b/lib/gitlab/external_authorization/client.rb
@@ -13,7 +13,8 @@ module Gitlab
}.freeze
def initialize(user, label)
- @user, @label = user, label
+ @user = user
+ @label = label
end
def request_access
@@ -51,18 +52,18 @@ module Gitlab
def body
@body ||= begin
- body = {
- user_identifier: @user.email,
- project_classification_label: @label,
- identities: @user.identities.map { |identity| { provider: identity.provider, extern_uid: identity.extern_uid } }
- }
+ body = {
+ user_identifier: @user.email,
+ project_classification_label: @label,
+ identities: @user.identities.map { |identity| { provider: identity.provider, extern_uid: identity.extern_uid } }
+ }
- if @user.ldap_identity
- body[:user_ldap_dn] = @user.ldap_identity.extern_uid
- end
+ if @user.ldap_identity
+ body[:user_ldap_dn] = @user.ldap_identity.extern_uid
+ end
- body
- end
+ body
+ end
end
end
end
diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb
index bd5d2e53180..612865ed1be 100644
--- a/lib/gitlab/fogbugz_import/importer.rb
+++ b/lib/gitlab/fogbugz_import/importer.rb
@@ -199,8 +199,7 @@ module Gitlab
def linkify_issues(str)
str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2')
- str = str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2')
- str
+ str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2')
end
def escape_for_markdown(str)
@@ -208,8 +207,7 @@ module Gitlab
str = str.gsub(/^-/, "\\-")
str = str.gsub("`", "\\~")
str = str.delete("\r")
- str = str.gsub("\n", " \n")
- str
+ str.gsub("\n", " \n")
end
def format_content(raw_content)
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 9e24306c05e..a5b1b7d914b 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -30,8 +30,10 @@ module Gitlab
end
def process_raw_blame(output)
- lines, final = [], []
- info, commits = {}, {}
+ lines = []
+ final = []
+ info = {}
+ commits = {}
# process the output
output.split("\n").each do |line|
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index ff99803d8de..51baed32935 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -390,7 +390,7 @@ module Gitlab
@committer_name = commit.committer.name.dup
@committer_email = commit.committer.email.dup
@parent_ids = Array(commit.parent_ids)
- @trailers = Hash[commit.trailers.map { |t| [t.key, t.value] }]
+ @trailers = commit.trailers.to_h { |t| [t.key, t.value] }
end
# Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index 19462e6cb02..fb947c80b7e 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -82,6 +82,30 @@ module Gitlab
!!@overflow
end
+ def overflow_max_lines?
+ !!@overflow_max_lines
+ end
+
+ def overflow_max_bytes?
+ !!@overflow_max_bytes
+ end
+
+ def overflow_max_files?
+ !!@overflow_max_files
+ end
+
+ def collapsed_safe_lines?
+ !!@collapsed_safe_lines
+ end
+
+ def collapsed_safe_files?
+ !!@collapsed_safe_files
+ end
+
+ def collapsed_safe_bytes?
+ !!@collapsed_safe_bytes
+ end
+
def size
@size ||= count # forces a loop using each method
end
@@ -103,10 +127,9 @@ module Gitlab
end
def decorate!
- collection = each_with_index do |element, i|
+ each_with_index do |element, i|
@array[i] = yield(element)
end
- collection
end
alias_method :to_ary, :to_a
@@ -121,7 +144,15 @@ module Gitlab
end
def over_safe_limits?(files)
- files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes
+ if files >= safe_max_files
+ @collapsed_safe_files = true
+ elsif @line_count > safe_max_lines
+ @collapsed_safe_lines = true
+ elsif @byte_count >= safe_max_bytes
+ @collapsed_safe_bytes = true
+ end
+
+ @collapsed_safe_files || @collapsed_safe_lines || @collapsed_safe_bytes
end
def expand_diff?
@@ -154,6 +185,7 @@ module Gitlab
if @enforce_limits && i >= max_files
@overflow = true
+ @overflow_max_files = true
break
end
@@ -166,10 +198,19 @@ module Gitlab
@line_count += diff.line_count
@byte_count += diff.diff.bytesize
- if @enforce_limits && (@line_count >= max_lines || @byte_count >= max_bytes)
+ if @enforce_limits && @line_count >= max_lines
+ # This last Diff instance pushes us over the lines limit. We stop and
+ # discard it.
+ @overflow = true
+ @overflow_max_lines = true
+ break
+ end
+
+ if @enforce_limits && @byte_count >= max_bytes
# This last Diff instance pushes us over the lines limit. We stop and
# discard it.
@overflow = true
+ @overflow_max_bytes = true
break
end
diff --git a/lib/gitlab/git/merge_base.rb b/lib/gitlab/git/merge_base.rb
index b27f7038c26..905d72cadbf 100644
--- a/lib/gitlab/git/merge_base.rb
+++ b/lib/gitlab/git/merge_base.rb
@@ -6,7 +6,8 @@ module Gitlab
include Gitlab::Utils::StrongMemoize
def initialize(repository, refs)
- @repository, @refs = repository, refs
+ @repository = repository
+ @refs = refs
end
# Returns the SHA of the first common ancestor
diff --git a/lib/gitlab/git/patches/commit_patches.rb b/lib/gitlab/git/patches/commit_patches.rb
index c62994432d3..1182db10c34 100644
--- a/lib/gitlab/git/patches/commit_patches.rb
+++ b/lib/gitlab/git/patches/commit_patches.rb
@@ -7,7 +7,10 @@ module Gitlab
include Gitlab::Git::WrapsGitalyErrors
def initialize(user, repository, branch, patch_collection)
- @user, @repository, @branch, @patches = user, repository, branch, patch_collection
+ @user = user
+ @repository = repository
+ @branch = branch
+ @patches = patch_collection
end
def commit
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index e316d52ac05..3361cee733b 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -599,9 +599,9 @@ module Gitlab
tags.find { |tag| tag.name == name }
end
- def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts)
+ def merge_to_ref(user, **kwargs)
wrapped_gitaly_errors do
- gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts)
+ gitaly_operation_client.user_merge_to_ref(user, **kwargs)
end
end
@@ -1017,6 +1017,10 @@ module Gitlab
gitaly_repository_client.search_files_by_name(ref, safe_query)
end
+ def search_files_by_regexp(filter, ref = 'HEAD')
+ gitaly_repository_client.search_files_by_regexp(ref, filter)
+ end
+
def find_commits_by_message(query, ref, path, limit, offset)
wrapped_gitaly_errors do
gitaly_commit_client
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index da86d6baf4a..568e894a02f 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -87,6 +87,10 @@ module Gitlab
end
end
+ def cache_key
+ "tag:" + Digest::SHA1.hexdigest([name, message, target, target_commit&.sha].join)
+ end
+
private
def message_from_gitaly_tag
diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb
index 55ff3c6caf1..75d6b949874 100644
--- a/lib/gitlab/git/wiki.rb
+++ b/lib/gitlab/git/wiki.rb
@@ -102,12 +102,6 @@ module Gitlab
end
end
- def file(name, version)
- wrapped_gitaly_errors do
- gitaly_find_file(name, version)
- end
- end
-
# options:
# :page - The Integer page number.
# :per_page - The number of items per page.
@@ -161,13 +155,6 @@ module Gitlab
nil
end
- def gitaly_find_file(name, version)
- wiki_file = gitaly_wiki_client.find_file(name, version)
- return unless wiki_file
-
- Gitlab::Git::WikiFile.new(wiki_file)
- end
-
def gitaly_list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false)
params = { limit: limit, sort: sort, direction_desc: direction_desc }
diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb
index 7f09173f05c..c56a17c52f3 100644
--- a/lib/gitlab/git/wiki_file.rb
+++ b/lib/gitlab/git/wiki_file.rb
@@ -5,25 +5,11 @@ module Gitlab
class WikiFile
attr_reader :mime_type, :raw_data, :name, :path
- # This class wraps Gitlab::GitalyClient::WikiFile
- def initialize(gitaly_file)
- @mime_type = gitaly_file.mime_type
- @raw_data = gitaly_file.raw_data
- @name = gitaly_file.name
- @path = gitaly_file.path
- end
-
- def self.from_blob(blob)
- hash = {
- name: File.basename(blob.name),
- mime_type: blob.mime_type,
- path: blob.path,
- raw_data: blob.data
- }
-
- gitaly_file = Gitlab::GitalyClient::WikiFile.new(hash)
-
- Gitlab::Git::WikiFile.new(gitaly_file)
+ def initialize(blob)
+ @mime_type = blob.mime_type
+ @raw_data = blob.data
+ @name = File.basename(blob.name)
+ @path = blob.path
end
end
end
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index c5ca46827cb..31e4755192e 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -91,6 +91,7 @@ module Gitlab
when *PUSH_COMMANDS
check_push_access!
end
+ check_additional_conditions!
success_result
end
@@ -530,6 +531,10 @@ module Gitlab
def size_checker
container.repository_size_checker
end
+
+ # overriden in EE
+ def check_additional_conditions!
+ end
end
end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index e3788814dd5..f4a89edecd1 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -215,7 +215,7 @@ module Gitlab
'client_name' => CLIENT_NAME
}
- context_data = Labkit::Context.current&.to_h
+ context_data = Gitlab::ApplicationContext.current
feature_stack = Thread.current[:gitaly_feature_stack]
feature = feature_stack && feature_stack[0]
diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb
index f935281ac2e..74e6279708e 100644
--- a/lib/gitlab/gitaly_client/attributes_bag.rb
+++ b/lib/gitlab/gitaly_client/attributes_bag.rb
@@ -3,7 +3,7 @@
module Gitlab
module GitalyClient
# This module expects an `ATTRS` const to be defined on the subclass
- # See GitalyClient::WikiFile for an example
+ # See GitalyClient::WikiPage for an example
module AttributesBag
extend ActiveSupport::Concern
diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb
index c66b3335d89..19a473e4785 100644
--- a/lib/gitlab/gitaly_client/blob_service.rb
+++ b/lib/gitlab/gitaly_client/blob_service.rb
@@ -78,17 +78,7 @@ module Gitlab
end
def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil)
- request = Gitaly::GetNewLFSPointersRequest.new(
- repository: @gitaly_repo,
- revision: encode_binary(revision),
- limit: limit || 0
- )
-
- if not_in.nil? || not_in == :all
- request.not_in_all = true
- else
- request.not_in_refs += not_in
- end
+ request, rpc = create_new_lfs_pointers_request(revision, limit, not_in)
timeout =
if dynamic_timeout
@@ -100,7 +90,7 @@ module Gitlab
response = GitalyClient.call(
@gitaly_repo.storage_name,
:blob_service,
- :get_new_lfs_pointers,
+ rpc,
request,
timeout: timeout
)
@@ -108,16 +98,51 @@ module Gitlab
end
def get_all_lfs_pointers
- request = Gitaly::GetAllLFSPointersRequest.new(
- repository: @gitaly_repo
+ request = Gitaly::ListLFSPointersRequest.new(
+ repository: @gitaly_repo,
+ revisions: [encode_binary("--all")]
)
- response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout)
+ response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_lfs_pointers, request, timeout: GitalyClient.medium_timeout)
map_lfs_pointers(response)
end
private
+ def create_new_lfs_pointers_request(revision, limit, not_in)
+ # If the check happens for a change which is using a quarantine
+ # environment for incoming objects, then we can avoid doing the
+ # necessary graph walk to detect only new LFS pointers and instead scan
+ # through all quarantined objects.
+ git_env = ::Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository)
+ if Feature.enabled?(:lfs_integrity_inspect_quarantined_objects, @project, default_enabled: :yaml) && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present?
+ repository = @gitaly_repo.dup
+ repository.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string)
+
+ request = Gitaly::ListAllLFSPointersRequest.new(
+ repository: repository,
+ limit: limit || 0
+ )
+
+ [request, :list_all_lfs_pointers]
+ else
+ revisions = [revision]
+ revisions += if not_in.nil? || not_in == :all
+ ["--not", "--all"]
+ else
+ not_in.prepend "--not"
+ end
+
+ request = Gitaly::ListLFSPointersRequest.new(
+ repository: @gitaly_repo,
+ limit: limit || 0,
+ revisions: revisions.map { |rev| encode_binary(rev) }
+ )
+
+ [request, :list_lfs_pointers]
+ end
+ end
+
def consume_blob_response(response)
data = []
blob = nil
diff --git a/lib/gitlab/gitaly_client/call.rb b/lib/gitlab/gitaly_client/call.rb
index 9d4d86997ad..4bb184bee2f 100644
--- a/lib/gitlab/gitaly_client/call.rb
+++ b/lib/gitlab/gitaly_client/call.rb
@@ -50,11 +50,11 @@ module Gitlab
end
def recording_request
- start = Gitlab::Metrics::System.monotonic_time
+ @start = Gitlab::Metrics::System.monotonic_time
yield
ensure
- @duration += Gitlab::Metrics::System.monotonic_time - start
+ @duration += Gitlab::Metrics::System.monotonic_time - @start
end
def store_timings
@@ -64,8 +64,14 @@ module Gitlab
request_hash = @request.is_a?(Google::Protobuf::MessageExts) ? @request.to_h : {}
- GitalyClient.add_call_details(feature: "#{@service}##{@rpc}", duration: @duration, request: request_hash, rpc: @rpc,
- backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller))
+ GitalyClient.add_call_details(
+ start: @start,
+ feature: "#{@service}##{@rpc}",
+ duration: @duration,
+ request: request_hash,
+ rpc: @rpc,
+ backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)
+ )
end
end
end
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index ef5221a8042..3d24b4d53a4 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -107,6 +107,8 @@ module Gitlab
entry.data = data.join
entry unless entry.oid.blank?
+ rescue GRPC::NotFound
+ nil
end
def tree_entries(repository, revision, path, recursive)
diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb
index 6f302b2c4e7..5ce1b1f0c87 100644
--- a/lib/gitlab/gitaly_client/operation_service.rb
+++ b/lib/gitlab/gitaly_client/operation_service.rb
@@ -103,7 +103,7 @@ module Gitlab
end
end
- def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts)
+ def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:, allow_conflicts: false)
request = Gitaly::UserMergeToRefRequest.new(
repository: @gitaly_repo,
source_sha: source_sha,
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index bd450249355..a93f4071efc 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -339,6 +339,11 @@ module Gitlab
search_results_from_response(response, options)
end
+ def search_files_by_regexp(ref, filter)
+ request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter)
+ GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files)
+ end
+
def disconnect_alternates
request = Gitaly::DisconnectGitAlternatesRequest.new(
repository: @gitaly_repo
diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb
index 7edd42f9ef7..dd9e3d5d28b 100644
--- a/lib/gitlab/gitaly_client/storage_settings.rb
+++ b/lib/gitlab/gitaly_client/storage_settings.rb
@@ -11,7 +11,7 @@ module Gitlab
DirectPathAccessError = Class.new(StandardError)
InvalidConfigurationError = Class.new(StandardError)
- INVALID_STORAGE_MESSAGE = <<~MSG.freeze
+ INVALID_STORAGE_MESSAGE = <<~MSG
Storage is invalid because it has no `path` key.
For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example.
diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb
deleted file mode 100644
index ef2b23732d1..00000000000
--- a/lib/gitlab/gitaly_client/wiki_file.rb
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module GitalyClient
- class WikiFile
- ATTRS = %i(name mime_type path raw_data).freeze
-
- include AttributesBag
- end
- end
-end
diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb
index 9034edb6263..fecc2b7023d 100644
--- a/lib/gitlab/gitaly_client/wiki_service.rb
+++ b/lib/gitlab/gitaly_client/wiki_service.rb
@@ -153,32 +153,6 @@ module Gitlab
versions
end
- def find_file(name, revision)
- request = Gitaly::WikiFindFileRequest.new(
- repository: @gitaly_repo,
- name: encode_binary(name),
- revision: encode_binary(revision)
- )
-
- response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request, timeout: GitalyClient.fast_timeout)
- wiki_file = nil
-
- response.each do |message|
- next unless message.name.present? || wiki_file
-
- if wiki_file
- wiki_file.raw_data = "#{wiki_file.raw_data}#{message.raw_data}"
- else
- wiki_file = GitalyClient::WikiFile.new(message.to_h)
- # All gRPC strings in a response are frozen, so we get
- # an unfrozen version here so appending in the else clause below doesn't blow up.
- wiki_file.raw_data = wiki_file.raw_data.dup
- end
- end
-
- wiki_file
- end
-
private
# If a block is given and the yielded value is truthy, iteration will be
diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb
index f2dc668c482..31b7a198b92 100644
--- a/lib/gitlab/golang.rb
+++ b/lib/gitlab/golang.rb
@@ -2,10 +2,12 @@
module Gitlab
module Golang
+ PseudoVersion = Struct.new(:semver, :timestamp, :commit_id)
+
extend self
def local_module_prefix
- @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/".freeze
+ @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/"
end
def semver_tag?(tag)
@@ -37,11 +39,11 @@ module Gitlab
end
# This pattern is intentionally more forgiving than the patterns
- # above. Correctness is verified by #pseudo_version_commit.
+ # above. Correctness is verified by #validate_pseudo_version.
/\A\d{14}-\h+\z/.freeze.match? pre
end
- def pseudo_version_commit(project, semver)
+ def parse_pseudo_version(semver)
# Per Go's implementation of pseudo-versions, a tag should be
# considered a pseudo-version if it matches one of the patterns
# listed in #pseudo_version?, regardless of the content of the
@@ -55,9 +57,14 @@ module Gitlab
# - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go)
# Go ignores anything before '.' or after the second '-', so we will do the same
- timestamp, sha = semver.prerelease.split('-').last 2
+ timestamp, commit_id = semver.prerelease.split('-').last 2
timestamp = timestamp.split('.').last
- commit = project.repository.commit_by(oid: sha)
+
+ PseudoVersion.new(semver, timestamp, commit_id)
+ end
+
+ def validate_pseudo_version(project, version, commit = nil)
+ commit ||= project.repository.commit_by(oid: version.commit_id)
# Error messages are based on the responses of proxy.golang.org
@@ -65,10 +72,10 @@ module Gitlab
raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit
# Require the SHA fragment to be 12 characters long
- raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12
+ raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12
# Require the timestamp to match that of the commit
- raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp
+ raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp
commit
end
@@ -77,6 +84,14 @@ module Gitlab
Packages::SemVer.parse(str, prefixed: true)
end
+ def go_path(project, path = nil)
+ if path.blank?
+ "#{local_module_prefix}/#{project.full_path}"
+ else
+ "#{local_module_prefix}/#{project.full_path}/#{path}"
+ end
+ end
+
def pkg_go_dev_url(name, version = nil)
if version
"https://pkg.go.dev/#{name}@#{version}"
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index c7e215c143f..08c17058fcb 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -45,7 +45,7 @@ module Gitlab
# Initialize gon.features with any flags that should be
# made globally available to the frontend
push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false)
- push_frontend_feature_flag(:usage_data_api, default_enabled: true)
+ push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml)
push_frontend_feature_flag(:security_auto_fix, default_enabled: false)
end
diff --git a/lib/gitlab/grape_logging/loggers/context_logger.rb b/lib/gitlab/grape_logging/loggers/context_logger.rb
index 0a8f0872fbe..468a296886e 100644
--- a/lib/gitlab/grape_logging/loggers/context_logger.rb
+++ b/lib/gitlab/grape_logging/loggers/context_logger.rb
@@ -6,7 +6,7 @@ module Gitlab
module Loggers
class ContextLogger < ::GrapeLogging::Loggers::Base
def parameters(_, _)
- Labkit::Context.current.to_h
+ Gitlab::ApplicationContext.current
end
end
end
diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb
deleted file mode 100644
index e83b567308b..00000000000
--- a/lib/gitlab/graphql/authorize.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Graphql
- # Allow fields to declare permissions their objects must have. The field
- # will be set to nil unless all required permissions are present.
- module Authorize
- extend ActiveSupport::Concern
-
- def self.use(schema_definition)
- schema_definition.instrument(:field, Gitlab::Graphql::Authorize::Instrumentation.new, after_built_ins: true)
- end
- end
- end
-end
diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb
deleted file mode 100644
index e8db619f88a..00000000000
--- a/lib/gitlab/graphql/authorize/authorize_field_service.rb
+++ /dev/null
@@ -1,147 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Graphql
- module Authorize
- class AuthorizeFieldService
- def initialize(field)
- @field = field
- @old_resolve_proc = @field.resolve_proc
- end
-
- def authorizations?
- authorizations.present?
- end
-
- def authorized_resolve
- proc do |parent_typed_object, args, ctx|
- resolved_type = @old_resolve_proc.call(parent_typed_object, args, ctx)
- authorizing_object = authorize_against(parent_typed_object, resolved_type)
-
- filter_allowed(ctx[:current_user], resolved_type, authorizing_object)
- end
- end
-
- private
-
- def authorizations
- @authorizations ||= (type_authorizations + field_authorizations).uniq
- end
-
- # Returns any authorize metadata from the return type of @field
- def type_authorizations
- type = @field.type
-
- # When the return type of @field is a collection, find the singular type
- if @field.connection?
- type = node_type_for_relay_connection(type)
- elsif type.list?
- type = node_type_for_basic_connection(type)
- end
-
- type = type.unwrap if type.kind.non_null?
-
- Array.wrap(type.metadata[:authorize])
- end
-
- # Returns any authorize metadata from @field
- def field_authorizations
- return [] if @field.metadata[:authorize] == true
-
- Array.wrap(@field.metadata[:authorize])
- end
-
- def authorize_against(parent_typed_object, resolved_type)
- if scalar_type?
- # The field is a built-in/scalar type, or a list of scalars
- # authorize using the parent's object
- parent_typed_object.object
- elsif @field.connection? || @field.type.list? || resolved_type.is_a?(Array)
- # The field is a connection or a list of non-built-in types, we'll
- # authorize each element when rendering
- nil
- elsif resolved_type.respond_to?(:object)
- # The field is a type representing a single object, we'll authorize
- # against the object directly
- resolved_type.object
- else
- # Resolved type is a single object that might not be loaded yet by
- # the batchloader, we'll authorize that
- resolved_type
- end
- end
-
- def filter_allowed(current_user, resolved_type, authorizing_object)
- if resolved_type.nil?
- # We're not rendering anything, for example when a record was not found
- # no need to do anything
- elsif authorizing_object
- # Authorizing fields representing scalars, or a simple field with an object
- ::Gitlab::Graphql::Lazy.with_value(authorizing_object) do |object|
- resolved_type if allowed_access?(current_user, object)
- end
- elsif @field.connection?
- ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |type|
- # A connection with pagination, modify the visible nodes on the
- # connection type in place
- nodes = to_nodes(type)
- nodes.keep_if { |node| allowed_access?(current_user, node) } if nodes
- type
- end
- elsif @field.type.list? || resolved_type.is_a?(Array)
- # A simple list of rendered types each object being an object to authorize
- ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |items|
- items.select do |single_object_type|
- object_type = realized(single_object_type)
- object = object_type.try(:object) || object_type
- allowed_access?(current_user, object)
- end
- end
- else
- raise "Can't authorize #{@field}"
- end
- end
-
- # Ensure that we are dealing with realized objects, not delayed promises
- def realized(thing)
- ::Gitlab::Graphql::Lazy.force(thing)
- end
-
- # Try to get the connection
- # can be at type.object or at type
- def to_nodes(type)
- if type.respond_to?(:nodes)
- type.nodes
- elsif type.respond_to?(:object)
- to_nodes(type.object)
- else
- nil
- end
- end
-
- def allowed_access?(current_user, object)
- object = realized(object)
-
- authorizations.all? do |ability|
- Ability.allowed?(current_user, ability, object)
- end
- end
-
- # Returns the singular type for relay connections.
- # This will be the type class of edges.node
- def node_type_for_relay_connection(type)
- type.unwrap.get_field('edges').type.unwrap.get_field('node').type
- end
-
- # Returns the singular type for basic connections, for example `[Types::ProjectType]`
- def node_type_for_basic_connection(type)
- type.unwrap
- end
-
- def scalar_type?
- node_type_for_basic_connection(@field.type).kind.scalar?
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb
index 6ee446011d4..4d575b964e5 100644
--- a/lib/gitlab/graphql/authorize/authorize_resource.rb
+++ b/lib/gitlab/graphql/authorize/authorize_resource.rb
@@ -5,15 +5,17 @@ module Gitlab
module Authorize
module AuthorizeResource
extend ActiveSupport::Concern
+ ConfigurationError = Class.new(StandardError)
- RESOURCE_ACCESS_ERROR = "The resource that you are attempting to access does not exist or you don't have permission to perform this action"
+ RESOURCE_ACCESS_ERROR = "The resource that you are attempting to access does " \
+ "not exist or you don't have permission to perform this action"
class_methods do
def required_permissions
# If the `#authorize` call is used on multiple classes, we add the
# permissions specified on a subclass, to the ones that were specified
- # on it's superclass.
- @required_permissions ||= if self.respond_to?(:superclass) && superclass.respond_to?(:required_permissions)
+ # on its superclass.
+ @required_permissions ||= if respond_to?(:superclass) && superclass.respond_to?(:required_permissions)
superclass.required_permissions.dup
else
[]
@@ -23,6 +25,18 @@ module Gitlab
def authorize(*permissions)
required_permissions.concat(permissions)
end
+
+ def authorizes_object?
+ defined?(@authorizes_object) ? @authorizes_object : false
+ end
+
+ def authorizes_object!
+ @authorizes_object = true
+ end
+
+ def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR)
+ raise ::Gitlab::Graphql::Errors::ResourceNotAvailable, msg
+ end
end
def find_object(*args)
@@ -37,33 +51,21 @@ module Gitlab
object
end
+ # authorizes the object using the current class authorization.
def authorize!(object)
- unless authorized_resource?(object)
- raise_resource_not_available_error!
- end
+ raise_resource_not_available_error! unless authorized_resource?(object)
end
- # this was named `#authorized?`, however it conflicts with the native
- # graphql gem version
- # TODO consider adopting the gem's built in authorization system
- # https://gitlab.com/gitlab-org/gitlab/issues/13984
def authorized_resource?(object)
# Sanity check. We don't want to accidentally allow a developer to authorize
# without first adding permissions to authorize against
- if self.class.required_permissions.empty?
- raise Gitlab::Graphql::Errors::ArgumentError, "#{self.class.name} has no authorizations"
- end
+ raise ConfigurationError, "#{self.class.name} has no authorizations" if self.class.authorization.none?
- self.class.required_permissions.all? do |ability|
- # The actions could be performed across multiple objects. In which
- # case the current user is common, and we could benefit from the
- # caching in `DeclarativePolicy`.
- Ability.allowed?(current_user, ability, object, scope: :user)
- end
+ self.class.authorization.ok?(object, current_user)
end
- def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR)
- raise Gitlab::Graphql::Errors::ResourceNotAvailable, msg
+ def raise_resource_not_available_error!(*args)
+ self.class.raise_resource_not_available_error!(*args)
end
end
end
diff --git a/lib/gitlab/graphql/authorize/connection_filter_extension.rb b/lib/gitlab/graphql/authorize/connection_filter_extension.rb
new file mode 100644
index 00000000000..c75510df3e3
--- /dev/null
+++ b/lib/gitlab/graphql/authorize/connection_filter_extension.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module Authorize
+ class ConnectionFilterExtension < GraphQL::Schema::FieldExtension
+ class Redactor
+ include ::Gitlab::Graphql::Laziness
+
+ def initialize(type, context)
+ @type = type
+ @context = context
+ end
+
+ def redact(nodes)
+ remove_unauthorized(nodes)
+
+ nodes
+ end
+
+ def active?
+ # some scalar types (such as integers) do not respond to :authorized?
+ return false unless @type.respond_to?(:authorized?)
+
+ auth = @type.try(:authorization)
+
+ auth.nil? || auth.any?
+ end
+
+ private
+
+ def remove_unauthorized(nodes)
+ nodes
+ .map! { |lazy| force(lazy) }
+ .keep_if { |forced| @type.authorized?(forced, @context) }
+ end
+ end
+
+ def after_resolve(value:, context:, **rest)
+ return value if value.is_a?(GraphQL::Execution::Execute::Skip)
+
+ if @field.connection?
+ redact_connection(value, context)
+ elsif @field.type.list?
+ redact_list(value.to_a, context) unless value.nil?
+ end
+
+ value
+ end
+
+ def redact_connection(conn, context)
+ redactor = Redactor.new(@field.type.unwrap.node_type, context)
+ return unless redactor.active?
+
+ conn.redactor = redactor if conn.respond_to?(:redactor=)
+ end
+
+ def redact_list(list, context)
+ redactor = Redactor.new(@field.type.unwrap, context)
+ redactor.redact(list) if redactor.active?
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb
deleted file mode 100644
index 15ecc3b04f0..00000000000
--- a/lib/gitlab/graphql/authorize/instrumentation.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Graphql
- module Authorize
- class Instrumentation
- # Replace the resolver for the field with one that will only return the
- # resolved object if the permissions check is successful.
- def instrument(_type, field)
- service = AuthorizeFieldService.new(field)
-
- if service.authorizations?
- field.redefine { resolve(service.authorized_resolve) }
- else
- field
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/graphql/authorize/object_authorization.rb b/lib/gitlab/graphql/authorize/object_authorization.rb
new file mode 100644
index 00000000000..0bc87108871
--- /dev/null
+++ b/lib/gitlab/graphql/authorize/object_authorization.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module Authorize
+ class ObjectAuthorization
+ attr_reader :abilities
+
+ def initialize(abilities)
+ @abilities = Array.wrap(abilities).flatten
+ end
+
+ def none?
+ abilities.empty?
+ end
+
+ def any?
+ abilities.present?
+ end
+
+ def ok?(object, current_user)
+ return true if none?
+
+ subject = object.try(:declarative_policy_subject) || object
+ abilities.all? do |ability|
+ Ability.allowed?(current_user, ability, subject)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb
new file mode 100644
index 00000000000..e0176e2d6e0
--- /dev/null
+++ b/lib/gitlab/graphql/deprecation.rb
@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ class Deprecation
+ REASONS = {
+ renamed: 'This was renamed.',
+ discouraged: 'Use of this is not recommended.'
+ }.freeze
+
+ include ActiveModel::Validations
+
+ validates :milestone, presence: true, format: { with: /\A\d+\.\d+\z/, message: 'must be milestone-ish' }
+ validates :reason, presence: true
+ validates :reason,
+ format: { with: /.*[^.]\z/, message: 'must not end with a period' },
+ if: :reason_is_string?
+ validate :milestone_is_string
+ validate :reason_known_or_string
+
+ def self.parse(options)
+ new(**options) if options
+ end
+
+ def initialize(reason: nil, milestone: nil, replacement: nil)
+ @reason = reason.presence
+ @milestone = milestone.presence
+ @replacement = replacement.presence
+ end
+
+ def ==(other)
+ return false unless other.is_a?(self.class)
+
+ [reason_text, milestone, replacement] == [:reason_text, :milestone, :replacement].map do |attr|
+ other.send(attr) # rubocop: disable GitlabSecurity/PublicSend
+ end
+ end
+ alias_method :eql, :==
+
+ def markdown(context: :inline)
+ parts = [
+ "#{deprecated_in(format: :markdown)}.",
+ reason_text,
+ replacement.then { |r| "Use: `#{r}`." if r }
+ ].compact
+
+ case context
+ when :block
+ ['WARNING:', *parts].join("\n")
+ when :inline
+ parts.join(' ')
+ end
+ end
+
+ def edit_description(original_description)
+ @original_description = original_description
+ return unless original_description
+
+ original_description + description_suffix
+ end
+
+ def original_description
+ return unless @original_description
+ return @original_description if @original_description.ends_with?('.')
+
+ "#{@original_description}."
+ end
+
+ def deprecation_reason
+ [
+ reason_text,
+ replacement && "Please use `#{replacement}`.",
+ "#{deprecated_in}."
+ ].compact.join(' ')
+ end
+
+ private
+
+ attr_reader :reason, :milestone, :replacement
+
+ def milestone_is_string
+ return if milestone.is_a?(String)
+
+ errors.add(:milestone, 'must be a string')
+ end
+
+ def reason_known_or_string
+ return if REASONS.key?(reason)
+ return if reason_is_string?
+
+ errors.add(:reason, 'must be a known reason or a string')
+ end
+
+ def reason_is_string?
+ reason.is_a?(String)
+ end
+
+ def reason_text
+ @reason_text ||= REASONS[reason] || "#{reason.to_s.strip}."
+ end
+
+ def description_suffix
+ " #{deprecated_in}: #{reason_text}"
+ end
+
+ def deprecated_in(format: :plain)
+ case format
+ when :plain
+ "Deprecated in #{milestone}"
+ when :markdown
+ "**Deprecated** in #{milestone}"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb
index e9ff85d9ca9..f4173e26224 100644
--- a/lib/gitlab/graphql/docs/helper.rb
+++ b/lib/gitlab/graphql/docs/helper.rb
@@ -27,7 +27,10 @@ module Gitlab
MD
end
- def render_name_and_description(object, level = 3)
+ # Template methods:
+ # Methods that return chunks of Markdown for insertion into the document
+
+ def render_name_and_description(object, owner: nil, level: 3)
content = []
content << "#{'#' * level} `#{object[:name]}`"
@@ -35,10 +38,22 @@ module Gitlab
if object[:description].present?
desc = object[:description].strip
desc += '.' unless desc.ends_with?('.')
+ end
+
+ if object[:is_deprecated]
+ owner = Array.wrap(owner)
+ deprecation = schema_deprecation(owner, object[:name])
+ content << (deprecation&.original_description || desc)
+ content << render_deprecation(object, owner, :block)
+ else
content << desc
end
- content.join("\n\n")
+ content.compact.join("\n\n")
+ end
+
+ def render_return_type(query)
+ "Returns #{render_field_type(query[:type])}.\n"
end
def sorted_by_name(objects)
@@ -47,39 +62,25 @@ module Gitlab
objects.sort_by { |o| o[:name] }
end
- def render_field(field)
- row(render_name(field), render_field_type(field[:type]), render_description(field))
+ def render_field(field, owner)
+ render_row(
+ render_name(field, owner),
+ render_field_type(field[:type]),
+ render_description(field, owner, :inline)
+ )
end
- def render_enum_value(value)
- row(render_name(value), render_description(value))
+ def render_enum_value(enum, value)
+ render_row(render_name(value, enum[:name]), render_description(value, enum[:name], :inline))
end
- def row(*values)
- "| #{values.join(' | ')} |"
+ def render_union_member(member)
+ "- [`#{member}`](##{member.downcase})"
end
- def render_name(object)
- rendered_name = "`#{object[:name]}`"
- rendered_name += ' **{warning-solid}**' if object[:is_deprecated]
- rendered_name
- end
+ # QUERIES:
- # Returns the object description. If the object has been deprecated,
- # the deprecation reason will be returned in place of the description.
- def render_description(object)
- return object[:description] unless object[:is_deprecated]
-
- "**Deprecated:** #{object[:deprecation_reason]}"
- end
-
- def render_field_type(type)
- "[`#{type[:info]}`](##{type[:name].downcase})"
- end
-
- def render_return_type(query)
- "Returns #{render_field_type(query[:type])}.\n"
- end
+ # Methods that return parts of the schema, or related information:
# We are ignoring connections and built in types for now,
# they should be added when queries are generated.
@@ -103,6 +104,83 @@ module Gitlab
!enum_type[:name].in?(%w[__DirectiveLocation __TypeKind])
end
end
+
+ private # DO NOT CALL THESE METHODS IN TEMPLATES
+
+ # Template methods
+
+ def render_row(*values)
+ "| #{values.map { |val| val.to_s.squish }.join(' | ')} |"
+ end
+
+ def render_name(object, owner = nil)
+ rendered_name = "`#{object[:name]}`"
+ rendered_name += ' **{warning-solid}**' if object[:is_deprecated]
+ rendered_name
+ end
+
+ # Returns the object description. If the object has been deprecated,
+ # the deprecation reason will be returned in place of the description.
+ def render_description(object, owner = nil, context = :block)
+ owner = Array.wrap(owner)
+ return render_deprecation(object, owner, context) if object[:is_deprecated]
+ return if object[:description].blank?
+
+ desc = object[:description].strip
+ desc += '.' unless desc.ends_with?('.')
+ desc
+ end
+
+ def render_deprecation(object, owner, context)
+ deprecation = schema_deprecation(owner, object[:name])
+ return deprecation.markdown(context: context) if deprecation
+
+ reason = object[:deprecation_reason] || 'Use of this is deprecated.'
+ "**Deprecated:** #{reason}"
+ end
+
+ def render_field_type(type)
+ "[`#{type[:info]}`](##{type[:name].downcase})"
+ end
+
+ # Queries
+
+ # returns the deprecation information for a field or argument
+ # See: Gitlab::Graphql::Deprecation
+ def schema_deprecation(type_name, field_name)
+ schema_member(type_name, field_name)&.deprecation
+ end
+
+ # Return a part of the schema.
+ #
+ # This queries the Schema by owner and name to find:
+ #
+ # - fields (e.g. `schema_member('Query', 'currentUser')`)
+ # - arguments (e.g. `schema_member(['Query', 'project], 'fullPath')`)
+ def schema_member(type_name, field_name)
+ type_name = Array.wrap(type_name)
+ if type_name.size == 2
+ arg_name = field_name
+ type_name, field_name = type_name
+ else
+ type_name = type_name.first
+ arg_name = nil
+ end
+
+ return if type_name.nil? || field_name.nil?
+
+ type = schema.types[type_name]
+ return unless type && type.kind.fields?
+
+ field = type.fields[field_name]
+ return field if arg_name.nil?
+
+ args = field.arguments
+ is_mutation = field.mutation && field.mutation <= ::Mutations::BaseMutation
+ args = args['input'].type.unwrap.arguments if is_mutation
+
+ args[arg_name]
+ end
end
end
end
diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb
index 6abd56c89c6..497567f9389 100644
--- a/lib/gitlab/graphql/docs/renderer.rb
+++ b/lib/gitlab/graphql/docs/renderer.rb
@@ -10,17 +10,20 @@ module Gitlab
# It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs.
#
# Arguments:
- # schema - the GraphQL schema definition. For GitLab should be: GitlabSchema.graphql_definition
+ # schema - the GraphQL schema definition. For GitLab should be: GitlabSchema
# output_dir: The folder where the markdown files will be saved
# template: The path of the haml template to be parsed
class Renderer
include Gitlab::Graphql::Docs::Helper
+ attr_reader :schema
+
def initialize(schema, output_dir:, template:)
@output_dir = output_dir
@template = template
@layout = Haml::Engine.new(File.read(template))
- @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse
+ @parsed_schema = GraphQLDocs::Parser.new(schema.graphql_definition, {}).parse
+ @schema = schema
end
def contents
diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml
index 847f1777b08..fe73297d0d9 100644
--- a/lib/gitlab/graphql/docs/templates/default.md.haml
+++ b/lib/gitlab/graphql/docs/templates/default.md.haml
@@ -27,7 +27,7 @@
\
- sorted_by_name(queries).each do |query|
- = render_name_and_description(query)
+ = render_name_and_description(query, owner: 'Query')
\
= render_return_type(query)
- unless query[:arguments].empty?
@@ -35,7 +35,7 @@
~ "| Name | Type | Description |"
~ "| ---- | ---- | ----------- |"
- sorted_by_name(query[:arguments]).each do |argument|
- = render_field(argument)
+ = render_field(argument, query[:type][:name])
\
:plain
@@ -58,7 +58,7 @@
~ "| Field | Type | Description |"
~ "| ----- | ---- | ----------- |"
- sorted_by_name(type[:fields]).each do |field|
- = render_field(field)
+ = render_field(field, type[:name])
\
:plain
@@ -79,7 +79,7 @@
~ "| Value | Description |"
~ "| ----- | ----------- |"
- sorted_by_name(enum[:values]).each do |value|
- = render_enum_value(value)
+ = render_enum_value(enum, value)
\
:plain
@@ -121,12 +121,12 @@
\
- graphql_union_types.each do |type|
- = render_name_and_description(type, 4)
+ = render_name_and_description(type, level: 4)
\
One of:
\
- - type[:possible_types].each do |type_name|
- ~ "- [`#{type_name}`](##{type_name.downcase})"
+ - type[:possible_types].each do |member|
+ = render_union_member(member)
\
:plain
@@ -134,7 +134,7 @@
\
- graphql_interface_types.each do |type|
- = render_name_and_description(type, 4)
+ = render_name_and_description(type, level: 4)
\
Implementations:
\
@@ -144,5 +144,5 @@
~ "| Field | Type | Description |"
~ "| ----- | ---- | ----------- |"
- sorted_by_name(type[:fields] + type[:connections]).each do |field|
- = render_field(field)
+ = render_field(field, type[:name])
\
diff --git a/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb b/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb
index 67511c124e4..1945388cdd4 100644
--- a/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb
+++ b/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb
@@ -5,7 +5,8 @@ module Gitlab
module Loaders
class BatchLfsOidLoader
def initialize(repository, blob_id)
- @repository, @blob_id = repository, blob_id
+ @repository = repository
+ @blob_id = blob_id
end
def find
diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb
index 9b85ba164d4..805864cdd4c 100644
--- a/lib/gitlab/graphql/loaders/batch_model_loader.rb
+++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb
@@ -7,7 +7,8 @@ module Gitlab
attr_reader :model_class, :model_id
def initialize(model_class, model_id)
- @model_class, @model_id = model_class, model_id
+ @model_class = model_class
+ @model_id = model_id
end
# rubocop: disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb
index 0aa237c78de..26c1ce64a83 100644
--- a/lib/gitlab/graphql/loaders/full_path_model_loader.rb
+++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb
@@ -9,7 +9,8 @@ module Gitlab
attr_reader :model_class, :full_path
def initialize(model_class, full_path)
- @model_class, @full_path = model_class, full_path
+ @model_class = model_class
+ @full_path = full_path
end
def find
diff --git a/lib/gitlab/graphql/negatable_arguments.rb b/lib/gitlab/graphql/negatable_arguments.rb
new file mode 100644
index 00000000000..b4ab31ed51a
--- /dev/null
+++ b/lib/gitlab/graphql/negatable_arguments.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module NegatableArguments
+ class TypeDefiner
+ def initialize(resolver_class, type_definition)
+ @resolver_class = resolver_class
+ @type_definition = type_definition
+ end
+
+ def define!
+ negated_params_type.instance_eval(&@type_definition)
+ end
+
+ def negated_params_type
+ @negated_params_type ||= existing_type || build_type
+ end
+
+ private
+
+ def existing_type
+ ::Types.const_get(type_class_name, false) if ::Types.const_defined?(type_class_name)
+ end
+
+ def build_type
+ klass = Class.new(::Types::BaseInputObject)
+ ::Types.const_set(type_class_name, klass)
+ klass
+ end
+
+ def type_class_name
+ @type_class_name ||= begin
+ base_name = @resolver_class.name.sub('Resolvers::', '')
+ base_name + 'NegatedParamsType'
+ end
+ end
+ end
+
+ def negated(param_key: :not, &block)
+ definer = ::Gitlab::Graphql::NegatableArguments::TypeDefiner.new(self, block)
+ definer.define!
+
+ argument param_key, definer.negated_params_type,
+ required: false,
+ description: <<~MD
+ List of negated arguments.
+ Warning: this argument is experimental and a subject to change in future.
+ MD
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb
index bd785880b57..6645dac36fa 100644
--- a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb
+++ b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb
@@ -13,7 +13,11 @@ module Gitlab
# @param [Symbol] before_or_after indicates whether we want
# items :before the cursor or :after the cursor
def initialize(arel_table, order_list, values, operators, before_or_after)
- @arel_table, @order_list, @values, @operators, @before_or_after = arel_table, order_list, values, operators, before_or_after
+ @arel_table = arel_table
+ @order_list = order_list
+ @values = values
+ @operators = operators
+ @before_or_after = before_or_after
@before_or_after = :after unless [:after, :before].include?(@before_or_after)
end
diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb
index 3164598b7b9..ec70f5c5a24 100644
--- a/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb
+++ b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb
@@ -30,15 +30,13 @@ module Gitlab
# ex: " OR (relative_position = 23 AND id > 500)"
def second_attribute_condition
- condition = <<~SQL
+ <<~SQL
OR (
#{table_condition(order_list.first, values.first, '=').to_sql}
AND
#{table_condition(order_list[1], values[1], operators[1]).to_sql}
)
SQL
-
- condition
end
# ex: " OR (relative_position IS NULL)"
diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb
index fa25181d663..1aae1020e79 100644
--- a/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb
+++ b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb
@@ -14,15 +14,13 @@ module Gitlab
# ex: "(relative_position IS NULL AND id > 500)"
def first_attribute_condition
- condition = <<~SQL
+ <<~SQL
(
#{table_condition(order_list.first, nil, 'is_null').to_sql}
AND
#{table_condition(order_list[1], values[1], operators[1]).to_sql}
)
SQL
-
- condition
end
# ex: " OR (relative_position IS NOT NULL)"
diff --git a/lib/gitlab/graphql/pagination/keyset/query_builder.rb b/lib/gitlab/graphql/pagination/keyset/query_builder.rb
index 29169449843..ee9c902c735 100644
--- a/lib/gitlab/graphql/pagination/keyset/query_builder.rb
+++ b/lib/gitlab/graphql/pagination/keyset/query_builder.rb
@@ -6,7 +6,10 @@ module Gitlab
module Keyset
class QueryBuilder
def initialize(arel_table, order_list, decoded_cursor, before_or_after)
- @arel_table, @order_list, @decoded_cursor, @before_or_after = arel_table, order_list, decoded_cursor, before_or_after
+ @arel_table = arel_table
+ @order_list = order_list
+ @decoded_cursor = decoded_cursor
+ @before_or_after = before_or_after
if order_list.empty?
raise ArgumentError.new('No ordering scopes have been supplied')
diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb
index fcf293fb13e..74f55abccbc 100644
--- a/lib/gitlab/graphql/queries.rb
+++ b/lib/gitlab/graphql/queries.rb
@@ -224,11 +224,9 @@ module Gitlab
frag_path = frag_path.gsub(DOTS_RE) do |dots|
rel_dir(dots.split('/').count)
end
- frag_path = frag_path.gsub(IMPLICIT_ROOT) do
+ frag_path.gsub(IMPLICIT_ROOT) do
(Rails.root / 'app').to_s + '/'
end
-
- frag_path
end
def rel_dir(n_steps_up)
diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
index 8acd27869a9..c6f22e0bd4f 100644
--- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
+++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
@@ -12,6 +12,7 @@ module Gitlab
def initial_value(query)
variables = process_variables(query.provided_variables)
default_initial_values(query).merge({
+ operation_name: query.operation_name,
query_string: query.query_string,
variables: variables
})
@@ -20,8 +21,8 @@ module Gitlab
default_initial_values(query)
end
- def call(memo, visit_type, irep_node)
- RequestStore.store[:graphql_logs] = memo
+ def call(memo, *)
+ memo
end
def final_value(memo)
@@ -37,6 +38,8 @@ module Gitlab
memo[:used_fields] = field_usages.first
memo[:used_deprecated_fields] = field_usages.second
+ RequestStore.store[:graphql_logs] ||= []
+ RequestStore.store[:graphql_logs] << memo
GraphqlLogger.info(memo.except!(:time_started, :query))
rescue => e
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb
index e780bf8a986..f5f142c251f 100644
--- a/lib/gitlab/health_checks/gitaly_check.rb
+++ b/lib/gitlab/health_checks/gitaly_check.rb
@@ -5,7 +5,7 @@ module Gitlab
class GitalyCheck
extend BaseAbstractCheck
- METRIC_PREFIX = 'gitaly_health_check'.freeze
+ METRIC_PREFIX = 'gitaly_health_check'
class << self
def readiness
diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb
index 40dee0142b9..765d3dfca56 100644
--- a/lib/gitlab/highlight.rb
+++ b/lib/gitlab/highlight.rb
@@ -20,7 +20,9 @@ module Gitlab
@blob_content = blob_content
end
- def highlight(text, continue: true, plain: false)
+ def highlight(text, continue: false, plain: false, context: {})
+ @context = context
+
plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE
highlighted_text = highlight_text(text, continue: continue, plain: plain)
@@ -31,13 +33,15 @@ module Gitlab
def lexer
@lexer ||= custom_language || begin
Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new
- rescue Rouge::Guesser::Ambiguous => e
- e.alternatives.min_by(&:tag)
+ rescue Rouge::Guesser::Ambiguous => e
+ e.alternatives.min_by(&:tag)
end
end
private
+ attr_reader :context
+
def custom_language
return unless @language
@@ -53,13 +57,13 @@ module Gitlab
end
def highlight_plain(text)
- @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe
+ @formatter.format(Rouge::Lexers::PlainText.lex(text), context).html_safe
end
def highlight_rich(text, continue: true)
tag = lexer.tag
tokens = lexer.lex(text, continue: continue)
- Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe }
+ Timeout.timeout(timeout_time) { @formatter.format(tokens, context.merge(tag: tag)).html_safe }
rescue Timeout::Error => e
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
highlight_plain(text)
diff --git a/lib/gitlab/hook_data/user_builder.rb b/lib/gitlab/hook_data/user_builder.rb
new file mode 100644
index 00000000000..537245e948f
--- /dev/null
+++ b/lib/gitlab/hook_data/user_builder.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module HookData
+ class UserBuilder < BaseBuilder
+ alias_method :user, :object
+
+ # Sample data
+ # {
+ # :created_at=>"2021-04-02T10:00:26Z",
+ # :updated_at=>"2021-04-02T10:00:26Z",
+ # :event_name=>"user_create",
+ # :name=>"John Doe",
+ # :email=>"john@example.com",
+ # :user_id=>1,
+ # :username=>"johndoe"
+ # }
+
+ def build(event)
+ [
+ timestamps_data,
+ event_data(event),
+ user_data,
+ event_specific_user_data(event)
+ ].reduce(:merge)
+ end
+
+ private
+
+ def user_data
+ {
+ name: user.name,
+ email: user.email,
+ user_id: user.id,
+ username: user.username
+ }
+ end
+
+ def event_specific_user_data(event)
+ case event
+ when :rename
+ { old_username: user.username_before_last_save }
+ when :failed_login
+ { state: user.state }
+ else
+ {}
+ end
+ end
+ end
+ end
+end
+
+Gitlab::HookData::UserBuilder.prepend_if_ee('EE::Gitlab::HookData::UserBuilder')
diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb
index 37f618ae879..f7a3da53fdb 100644
--- a/lib/gitlab/http_connection_adapter.rb
+++ b/lib/gitlab/http_connection_adapter.rb
@@ -17,14 +17,6 @@ module Gitlab
def connection
@uri, hostname = validate_url!(uri)
- if options.key?(:http_proxyaddr)
- proxy_uri_with_port = uri_with_port(options[:http_proxyaddr], options[:http_proxyport])
- proxy_uri_validated = validate_url!(proxy_uri_with_port).first
-
- @options[:http_proxyaddr] = proxy_uri_validated.omit(:port).to_s
- @options[:http_proxyport] = proxy_uri_validated.port
- end
-
super.tap do |http|
http.hostname_override = hostname if hostname
end
@@ -53,11 +45,5 @@ module Gitlab
def allow_settings_local_requests?
Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?
end
-
- def uri_with_port(address, port)
- uri = Addressable::URI.parse(address)
- uri.port = port if port.present?
- uri
- end
end
end
diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb
index d60bc79df4c..05a4a8f4c93 100644
--- a/lib/gitlab/import_export/base/relation_factory.rb
+++ b/lib/gitlab/import_export/base/relation_factory.rb
@@ -6,7 +6,7 @@ module Gitlab
class RelationFactory
include Gitlab::Utils::StrongMemoize
- IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
+ IMPORTED_OBJECT_MAX_RETRIES = 5
OVERRIDES = {}.freeze
EXISTING_OBJECT_RELATIONS = %i[].freeze
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 778b42f4358..42d32593cbd 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -265,6 +265,7 @@ excluded_attributes:
- :issue_id
push_event_payload:
- :event_id
+ - :event_id_convert_to_bigint
project_badges:
- :group_id
resource_label_events:
@@ -287,6 +288,7 @@ excluded_attributes:
- :label_id
events:
- :target_id
+ - :id_convert_to_bigint
timelogs:
- :issue_id
- :merge_request_id
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index 428bcbe8dc5..2f15cdd7506 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -76,7 +76,7 @@ module Gitlab
def project_uploads_except_avatar(avatar_path)
return @project.uploads unless avatar_path
- @project.uploads.where("path != ?", avatar_path)
+ @project.uploads.where.not(path: avatar_path)
end
def download_and_copy(upload)
diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb
index 88753e80391..95c002edf0a 100644
--- a/lib/gitlab/import_sources.rb
+++ b/lib/gitlab/import_sources.rb
@@ -28,7 +28,7 @@ module Gitlab
prepend_if_ee('EE::Gitlab::ImportSources') # rubocop: disable Cop/InjectEnterpriseEditionModule
def options
- Hash[import_table.map { |importer| [importer.title, importer.name] }]
+ import_table.to_h { |importer| [importer.title, importer.name] }
end
def values
diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index 61de6b02453..a865a6392f0 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -6,24 +6,6 @@ module Gitlab
DURATION_PRECISION = 6 # microseconds
- def keys
- @keys ||= [
- :cpu_s,
- :gitaly_calls,
- :gitaly_duration_s,
- :rugged_calls,
- :rugged_duration_s,
- :elasticsearch_calls,
- :elasticsearch_duration_s,
- :elasticsearch_timed_out_count,
- *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values,
- *::Gitlab::Instrumentation::Redis.known_payload_keys,
- *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS,
- *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS,
- *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS
- ]
- end
-
def init_instrumentation_data(request_ip: nil)
# Set `request_start_time` only if this is request
# This is done, as `request_start_time` imply `request_deadline`
diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb
index 945ab7f40c2..6b33b60e850 100644
--- a/lib/gitlab/issuables_count_for_state.rb
+++ b/lib/gitlab/issuables_count_for_state.rb
@@ -78,7 +78,7 @@ module Gitlab
# to perform the calculation more efficiently. Until then, use a shorter
# timeout and return -1 as a sentinel value if it is triggered
begin
- ApplicationRecord.with_fast_statement_timeout do
+ ApplicationRecord.with_fast_read_statement_timeout do
finder.count_by_state
end
rescue ActiveRecord::QueryCanceled => err
diff --git a/lib/gitlab/jira/dvcs.rb b/lib/gitlab/jira/dvcs.rb
index 4415f98fc7f..ddf2cd76709 100644
--- a/lib/gitlab/jira/dvcs.rb
+++ b/lib/gitlab/jira/dvcs.rb
@@ -3,8 +3,8 @@
module Gitlab
module Jira
module Dvcs
- ENCODED_SLASH = '@'.freeze
- SLASH = '/'.freeze
+ ENCODED_SLASH = '@'
+ SLASH = '/'
ENCODED_ROUTE_REGEX = /[a-zA-Z0-9_\-\.#{ENCODED_SLASH}]+/.freeze
def self.encode_slash(path)
diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb
index 8565f664cd4..b51c0a33457 100644
--- a/lib/gitlab/json.rb
+++ b/lib/gitlab/json.rb
@@ -186,9 +186,14 @@ module Gitlab
# The `env` param is ignored because it's not needed in either our formatter or Grape's,
# but it is passed through for consistency.
#
+ # If explicitly supplied with a `PrecompiledJson` instance it will skip conversion
+ # and return it directly. This is mostly used in caching.
+ #
# @param object [Object]
# @return [String]
def self.call(object, env = nil)
+ return object.to_s if object.is_a?(PrecompiledJson)
+
if Feature.enabled?(:grape_gitlab_json, default_enabled: true)
Gitlab::Json.dump(object)
else
@@ -197,6 +202,34 @@ module Gitlab
end
end
+ # Wrapper class used to skip JSON dumping on Grape endpoints.
+
+ class PrecompiledJson
+ UnsupportedFormatError = Class.new(StandardError)
+
+ # @overload PrecompiledJson.new("foo")
+ # @param value [String]
+ #
+ # @overload PrecompiledJson.new(["foo", "bar"])
+ # @param value [Array<String>]
+ def initialize(value)
+ @value = value
+ end
+
+ # Convert the value to a String. This will invoke
+ # `#to_s` on the members of the value if it's an array.
+ #
+ # @return [String]
+ # @raise [NoMethodError] if the objects in an array doesn't support to_s
+ # @raise [PrecompiledJson::UnsupportedFormatError] if the value is neither a String or Array
+ def to_s
+ return @value if @value.is_a?(String)
+ return "[#{@value.join(',')}]" if @value.is_a?(Array)
+
+ raise UnsupportedFormatError
+ end
+ end
+
class LimitedEncoder
LimitExceeded = Class.new(StandardError)
diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb
index 329c0f221b5..7a674cb5c21 100644
--- a/lib/gitlab/kas.rb
+++ b/lib/gitlab/kas.rb
@@ -27,7 +27,7 @@ module Gitlab
def included_in_gitlab_com_rollout?(project)
return true unless ::Gitlab.com?
- Feature.enabled?(:kubernetes_agent_on_gitlab_com, project)
+ Feature.enabled?(:kubernetes_agent_on_gitlab_com, project, default_enabled: :yaml)
end
end
end
diff --git a/lib/gitlab/kubernetes/deployment.rb b/lib/gitlab/kubernetes/deployment.rb
index 55ed9a7517e..f2e3a0e6810 100644
--- a/lib/gitlab/kubernetes/deployment.rb
+++ b/lib/gitlab/kubernetes/deployment.rb
@@ -5,7 +5,7 @@ module Gitlab
class Deployment
include Gitlab::Utils::StrongMemoize
- STABLE_TRACK_VALUE = 'stable'.freeze
+ STABLE_TRACK_VALUE = 'stable'
def initialize(attributes = {}, pods: [])
@attributes = attributes
diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb
index 7600e60b904..1e5edb79f10 100644
--- a/lib/gitlab/language_detection.rb
+++ b/lib/gitlab/language_detection.rb
@@ -20,7 +20,7 @@ module Gitlab
# Newly detected languages, returned in a structure accepted by
# Gitlab::Database.bulk_insert
def insertions(programming_languages)
- lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h
+ lang_to_id = programming_languages.to_h { |p| [p.name, p.id] }
(languages - previous_language_names).map do |new_lang|
{
@@ -63,8 +63,7 @@ module Gitlab
@repository
.languages
.first(MAX_LANGUAGES)
- .map { |l| [l[:label], l] }
- .to_h
+ .to_h { |l| [l[:label], l] }
end
end
end
diff --git a/lib/gitlab/manifest_import/manifest.rb b/lib/gitlab/manifest_import/manifest.rb
index 7208fe5bbc5..618ddf37b88 100644
--- a/lib/gitlab/manifest_import/manifest.rb
+++ b/lib/gitlab/manifest_import/manifest.rb
@@ -47,6 +47,10 @@ module Gitlab
@errors << 'Make sure every <project> tag has name and path attributes.'
end
+ unless validate_scheme
+ @errors << 'Make sure the url does not start with javascript'
+ end
+
@errors.empty?
end
@@ -64,6 +68,10 @@ module Gitlab
end
end
+ def validate_scheme
+ remote !~ /\Ajavascript/i
+ end
+
def repository_url(name)
Gitlab::Utils.append_path(remote, name)
end
diff --git a/lib/gitlab/marker_range.rb b/lib/gitlab/marker_range.rb
index 50a59adebdf..73e4a545679 100644
--- a/lib/gitlab/marker_range.rb
+++ b/lib/gitlab/marker_range.rb
@@ -24,6 +24,12 @@ module Gitlab
Range.new(self.begin, self.end, self.exclude_end?)
end
+ def ==(other)
+ return false unless other.is_a?(self.class)
+
+ self.mode == other.mode && super
+ end
+
attr_reader :mode
end
end
diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb
index d419fa66e57..45c6205b36b 100644
--- a/lib/gitlab/markup_helper.rb
+++ b/lib/gitlab/markup_helper.rb
@@ -4,7 +4,7 @@ module Gitlab
module MarkupHelper
extend self
- MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown].freeze
+ MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown rmd].freeze
ASCIIDOC_EXTENSIONS = %w[adoc ad asciidoc].freeze
OTHER_EXTENSIONS = %w[textile rdoc org creole wiki mediawiki rst].freeze
EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS
diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb
index 3dda68bf93f..a1fabe75a97 100644
--- a/lib/gitlab/metrics/background_transaction.rb
+++ b/lib/gitlab/metrics/background_transaction.rb
@@ -34,8 +34,9 @@ module Gitlab
def labels
@labels ||= {
- endpoint_id: current_context&.get_attribute(:caller_id),
- feature_category: current_context&.get_attribute(:feature_category)
+ endpoint_id: endpoint_id,
+ feature_category: feature_category,
+ queue: queue
}
end
@@ -44,6 +45,21 @@ module Gitlab
def current_context
Labkit::Context.current
end
+
+ def feature_category
+ current_context&.get_attribute(:feature_category)
+ end
+
+ def endpoint_id
+ current_context&.get_attribute(:caller_id)
+ end
+
+ def queue
+ worker_class = endpoint_id.to_s.safe_constantize
+ return if worker_class.blank? || !worker_class.respond_to?(:queue)
+
+ worker_class.queue.to_s
+ end
end
end
end
diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb
index c90c1e3f0bc..55d14d6f94a 100644
--- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb
+++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb
@@ -104,9 +104,7 @@ module Gitlab
def format_query(metric)
expression = remove_new_lines(metric[:expr])
expression = replace_variables(expression)
- expression = replace_global_variables(expression, metric)
-
- expression
+ replace_global_variables(expression, metric)
end
# Accomodates instance-defined Grafana variables.
@@ -135,9 +133,7 @@ module Gitlab
def replace_global_variables(expression, metric)
expression = expression.gsub('$__interval', metric[:interval]) if metric[:interval]
expression = expression.gsub('$__from', query_params[:from])
- expression = expression.gsub('$__to', query_params[:to])
-
- expression
+ expression.gsub('$__to', query_params[:to])
end
# Removes new lines from expression.
diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb
index 60ae22df607..c0336a4d0fb 100644
--- a/lib/gitlab/metrics/samplers/database_sampler.rb
+++ b/lib/gitlab/metrics/samplers/database_sampler.rb
@@ -32,9 +32,9 @@ module Gitlab
private
def init_metrics
- METRIC_DESCRIPTIONS.map do |name, description|
+ METRIC_DESCRIPTIONS.to_h do |name, description|
[name, ::Gitlab::Metrics.gauge(:"#{METRIC_PREFIX}#{name}", description)]
- end.to_h
+ end
end
def host_stats
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index 5eefef02507..0d1cd641ffe 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -11,13 +11,16 @@ module Gitlab
DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze
SQL_COMMANDS_WITH_COMMENTS_REGEX = /\A(\/\*.*\*\/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i.freeze
- DURATION_BUCKET = [0.05, 0.1, 0.25].freeze
+ SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze
+ TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze
# This event is published from ActiveRecordBaseTransactionMetrics and
# used to record a database transaction duration when calling
# ActiveRecord::Base.transaction {} block.
def transaction(event)
- observe(:gitlab_database_transaction_seconds, event)
+ observe(:gitlab_database_transaction_seconds, event) do
+ buckets TRANSACTION_DURATION_BUCKET
+ end
end
def sql(event)
@@ -33,7 +36,9 @@ module Gitlab
increment(:db_cached_count) if cached_query?(payload)
increment(:db_write_count) unless select_sql_command?(payload)
- observe(:gitlab_sql_duration_seconds, event)
+ observe(:gitlab_sql_duration_seconds, event) do
+ buckets SQL_DURATION_BUCKET
+ end
end
def self.db_counter_payload
@@ -46,6 +51,10 @@ module Gitlab
payload
end
+ def self.known_payload_keys
+ DB_COUNTERS
+ end
+
private
def ignored_query?(payload)
@@ -66,10 +75,8 @@ module Gitlab
Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1
end
- def observe(histogram, event)
- current_transaction&.observe(histogram, event.duration / 1000.0) do
- buckets DURATION_BUCKET
- end
+ def observe(histogram, event, &block)
+ current_transaction&.observe(histogram, event.duration / 1000.0, &block)
end
def current_transaction
diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb
index 94c5d965200..0df64f2897e 100644
--- a/lib/gitlab/metrics/subscribers/external_http.rb
+++ b/lib/gitlab/metrics/subscribers/external_http.rb
@@ -37,7 +37,7 @@ module Gitlab
def request(event)
payload = event.payload
- add_to_detail_store(payload)
+ add_to_detail_store(event.time, payload)
add_to_request_store(payload)
expose_metrics(payload)
end
@@ -48,10 +48,11 @@ module Gitlab
::Gitlab::Metrics::Transaction.current
end
- def add_to_detail_store(payload)
+ def add_to_detail_store(start, payload)
return unless Gitlab::PerformanceBar.enabled_for_request?
self.class.detail_store << {
+ start: start,
duration: payload[:duration],
scheme: payload[:scheme],
method: payload[:method],
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index 79f1abe820f..329041e3ba2 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -31,7 +31,7 @@ module Gitlab
RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS'
JWT_PARAM_SUFFIX = '.gitlab-workhorse-upload'
JWT_PARAM_FIXED_KEY = 'upload'
- REWRITTEN_FIELD_NAME_MAX_LENGTH = 10000.freeze
+ REWRITTEN_FIELD_NAME_MAX_LENGTH = 10000
class Handler
def initialize(env, message)
diff --git a/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb
new file mode 100644
index 00000000000..d16c068c3c0
--- /dev/null
+++ b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Middleware
+ class RackMultipartTempfileFactory
+ # Immediately unlink the created temporary file so we don't have to rely
+ # on Rack::TempfileReaper catching this after the fact.
+ FACTORY = lambda do |filename, content_type|
+ Rack::Multipart::Parser::TEMPFILE_FACTORY.call(filename, content_type).tap(&:unlink)
+ end
+
+ def initialize(app)
+ @app = app
+ end
+
+ def call(env)
+ if ENV['GITLAB_TEMPFILE_IMMEDIATE_UNLINK'] == '1'
+ env[Rack::RACK_MULTIPART_TEMPFILE_FACTORY] = FACTORY
+ end
+
+ @app.call(env)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/middleware/same_site_cookies.rb b/lib/gitlab/middleware/same_site_cookies.rb
index 37ccc5abb10..405732e8015 100644
--- a/lib/gitlab/middleware/same_site_cookies.rb
+++ b/lib/gitlab/middleware/same_site_cookies.rb
@@ -17,7 +17,7 @@
module Gitlab
module Middleware
class SameSiteCookies
- COOKIE_SEPARATOR = "\n".freeze
+ COOKIE_SEPARATOR = "\n"
def initialize(app)
@app = app
diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb
index b1a1045a1f0..9a74266693b 100644
--- a/lib/gitlab/object_hierarchy.rb
+++ b/lib/gitlab/object_hierarchy.rb
@@ -68,13 +68,22 @@ module Gitlab
expose_depth = hierarchy_order.present?
hierarchy_order ||= :asc
- recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct
-
# if hierarchy_order is given, the calculated `depth` should be present in SELECT
if expose_depth
+ recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct
read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order))
else
- read_only(remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order))
+ recursive_query = base_and_ancestors_cte(upto).apply_to(model.all)
+
+ if skip_ordering?
+ recursive_query = recursive_query.distinct
+ else
+ recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct
+ recursive_query = model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table))
+ recursive_query = remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order)
+ end
+
+ read_only(recursive_query)
end
else
recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all)
@@ -93,12 +102,21 @@ module Gitlab
def base_and_descendants(with_depth: false)
if use_distinct?
# Always calculate `depth`, remove it later if with_depth is false
- base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct
-
if with_depth
- read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: :asc))
+ base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct
+ read_only(model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc))
else
- read_only(remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc))
+ base_cte = base_and_descendants_cte.apply_to(model.all)
+
+ if skip_ordering?
+ base_cte = base_cte.distinct
+ else
+ base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct
+ base_cte = model.from(Arel::Nodes::As.new(base_cte.arel, objects_table))
+ base_cte = remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc)
+ end
+
+ read_only(base_cte)
end
else
read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all))
@@ -161,7 +179,19 @@ module Gitlab
# Use distinct on the Namespace queries to avoid bad planner behavior in PG11.
def use_distinct?
- (model <= Namespace) && options[:use_distinct]
+ return unless model <= Namespace
+ # Global use_distinct_for_all_object_hierarchy takes precedence over use_distinct_in_object_hierarchy
+ return true if Feature.enabled?(:use_distinct_for_all_object_hierarchy)
+ return options[:use_distinct] if options.key?(:use_distinct)
+
+ false
+ end
+
+ # Skips the extra ordering when using distinct on the namespace queries
+ def skip_ordering?
+ return options[:skip_ordering] if options.key?(:skip_ordering)
+
+ false
end
# Remove the extra `depth` field using an INNER JOIN to avoid breaking UNION queries
diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb
index 33e709360ad..98e87e9e915 100644
--- a/lib/gitlab/pages.rb
+++ b/lib/gitlab/pages.rb
@@ -3,7 +3,7 @@
module Gitlab
module Pages
VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze
- INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'.freeze
+ INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'
MAX_SIZE = 1.terabyte
include JwtAuthenticatable
diff --git a/lib/gitlab/pages/migration_helper.rb b/lib/gitlab/pages/migration_helper.rb
new file mode 100644
index 00000000000..8f8667fafd9
--- /dev/null
+++ b/lib/gitlab/pages/migration_helper.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Pages
+ class MigrationHelper
+ def initialize(logger = nil)
+ @logger = logger
+ end
+
+ def migrate_to_remote_storage
+ deployments = ::PagesDeployment.with_files_stored_locally
+ migrate(deployments, ObjectStorage::Store::REMOTE)
+ end
+
+ def migrate_to_local_storage
+ deployments = ::PagesDeployment.with_files_stored_remotely
+ migrate(deployments, ObjectStorage::Store::LOCAL)
+ end
+
+ private
+
+ def batch_size
+ ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i
+ end
+
+ def migrate(deployments, store)
+ deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord
+ deployment.file.migrate!(store)
+
+ log_success(deployment, store)
+ rescue => e
+ log_error(e, deployment)
+ end
+ end
+
+ def log_success(deployment, store)
+ logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage")
+ end
+
+ def log_error(err, deployment)
+ logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}")
+ end
+
+ def storage_label(store)
+ if store == ObjectStorage::Store::LOCAL
+ 'local'
+ else
+ 'object'
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb
index 8650a80a85e..be71018e851 100644
--- a/lib/gitlab/pages/settings.rb
+++ b/lib/gitlab/pages/settings.rb
@@ -6,12 +6,28 @@ module Gitlab
DiskAccessDenied = Class.new(StandardError)
def path
- if ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite?
- raise DiskAccessDenied
- end
+ report_denied_disk_access
super
end
+
+ def local_store
+ @local_store ||= ::Gitlab::Pages::Stores::LocalStore.new(super)
+ end
+
+ private
+
+ def disk_access_denied?
+ return true unless ::Settings.pages.local_store&.enabled
+
+ ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite?
+ end
+
+ def report_denied_disk_access
+ raise DiskAccessDenied if disk_access_denied?
+ rescue => e
+ ::Gitlab::ErrorTracking.track_exception(e)
+ end
end
end
end
diff --git a/lib/gitlab/pages/stores/local_store.rb b/lib/gitlab/pages/stores/local_store.rb
new file mode 100644
index 00000000000..68a7ebaceff
--- /dev/null
+++ b/lib/gitlab/pages/stores/local_store.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Pages
+ module Stores
+ class LocalStore < ::SimpleDelegator
+ def enabled
+ return false unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true)
+
+ super
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb
index c1ccfae3e1f..ae5539c03b1 100644
--- a/lib/gitlab/pages_transfer.rb
+++ b/lib/gitlab/pages_transfer.rb
@@ -12,7 +12,7 @@ module Gitlab
class Async
METHODS.each do |meth|
define_method meth do |*args|
- next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true)
+ next unless Settings.pages.local_store.enabled
PagesTransferWorker.perform_async(meth, args)
end
@@ -21,7 +21,7 @@ module Gitlab
METHODS.each do |meth|
define_method meth do |*args|
- next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true)
+ next unless Settings.pages.local_store.enabled
super(*args)
end
diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb
index e8e68a5c4a5..e596e1bac9d 100644
--- a/lib/gitlab/pagination/keyset/order.rb
+++ b/lib/gitlab/pagination/keyset/order.rb
@@ -55,14 +55,14 @@ module Gitlab
# scope :created_at_ordered, -> {
# keyset_order = Gitlab::Pagination::Keyset::Order.build([
# Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
- # attribute: :created_at,
+ # attribute_name: :created_at,
# column_expression: Project.arel_table[:created_at],
# order_expression: Project.arel_table[:created_at].asc,
# distinct: false, # values in the column are not unique
# nullable: :nulls_last # we might see NULL values (bottom)
# ),
# Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
- # attribute: :id,
+ # attribute_name: :id,
# order_expression: Project.arel_table[:id].asc
# )
# ])
@@ -93,7 +93,7 @@ module Gitlab
end
def cursor_attributes_for_node(node)
- column_definitions.each_with_object({}) do |column_definition, hash|
+ column_definitions.each_with_object({}.with_indifferent_access) do |column_definition, hash|
field_value = node[column_definition.attribute_name]
hash[column_definition.attribute_name] = if field_value.is_a?(Time)
field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z')
@@ -162,7 +162,7 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def apply_cursor_conditions(scope, values = {})
scope = apply_custom_projections(scope)
- scope.where(build_where_values(values))
+ scope.where(build_where_values(values.with_indifferent_access))
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/pagination/offset_header_builder.rb b/lib/gitlab/pagination/offset_header_builder.rb
index 32089e40932..555f0e5a607 100644
--- a/lib/gitlab/pagination/offset_header_builder.rb
+++ b/lib/gitlab/pagination/offset_header_builder.rb
@@ -5,9 +5,9 @@ module Gitlab
class OffsetHeaderBuilder
attr_reader :request_context, :per_page, :page, :next_page, :prev_page, :total, :total_pages
- delegate :params, :header, :request, to: :request_context
+ delegate :request, to: :request_context
- def initialize(request_context:, per_page:, page:, next_page:, prev_page: nil, total:, total_pages:)
+ def initialize(request_context:, per_page:, page:, next_page:, prev_page: nil, total: nil, total_pages: nil, params: nil)
@request_context = request_context
@per_page = per_page
@page = page
@@ -15,6 +15,7 @@ module Gitlab
@prev_page = prev_page
@total = total
@total_pages = total_pages
+ @params = params
end
def execute(exclude_total_headers: false, data_without_counts: false)
@@ -56,10 +57,24 @@ module Gitlab
end
def page_href(next_page_params = {})
- query_params = params.merge(**next_page_params, per_page: params[:per_page]).to_query
+ query_params = params.merge(**next_page_params, per_page: per_page).to_query
build_page_url(query_params: query_params)
end
+
+ def params
+ @params || request_context.params
+ end
+
+ def header(name, value)
+ if request_context.respond_to?(:header)
+ # For Grape API
+ request_context.header(name, value)
+ else
+ # For rails controllers
+ request_context.response.headers[name] = value
+ end
+ end
end
end
end
diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb
index 380340b80be..c2a4602fd16 100644
--- a/lib/gitlab/performance_bar/stats.rb
+++ b/lib/gitlab/performance_bar/stats.rb
@@ -5,6 +5,12 @@ module Gitlab
# This class fetches Peek stats stored in redis and logs them in a
# structured log (so these can be then analyzed in Kibana)
class Stats
+ IGNORED_BACKTRACE_LOCATIONS = %w[
+ ee/lib/ee/peek
+ lib/peek
+ lib/gitlab/database
+ ].freeze
+
def initialize(redis)
@redis = redis
end
@@ -53,7 +59,8 @@ module Gitlab
end
def parse_backtrace(backtrace)
- return unless match = /(?<filename>.*):(?<filenum>\d+):in `(?<method>.*)'/.match(backtrace.first)
+ return unless backtrace_row = find_caller(backtrace)
+ return unless match = /(?<filename>.*):(?<filenum>\d+):in `(?<method>.*)'/.match(backtrace_row)
{
filename: match[:filename],
@@ -65,6 +72,12 @@ module Gitlab
}
end
+ def find_caller(backtrace)
+ backtrace.find do |line|
+ !line.start_with?(*IGNORED_BACKTRACE_LOCATIONS)
+ end
+ end
+
def logger
@logger ||= Gitlab::PerformanceBar::Logger.build
end
diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb
index 3885a9934d5..4c9d54a93ce 100644
--- a/lib/gitlab/phabricator_import.rb
+++ b/lib/gitlab/phabricator_import.rb
@@ -5,7 +5,7 @@ module Gitlab
BaseError = Class.new(StandardError)
def self.available?
- Feature.enabled?(:phabricator_import) &&
+ Feature.enabled?(:phabricator_import, default_enabled: :yaml) &&
Gitlab::CurrentSettings.import_sources.include?('phabricator')
end
end
diff --git a/lib/gitlab/phabricator_import/issues/importer.rb b/lib/gitlab/phabricator_import/issues/importer.rb
index a58438452ff..478c26af030 100644
--- a/lib/gitlab/phabricator_import/issues/importer.rb
+++ b/lib/gitlab/phabricator_import/issues/importer.rb
@@ -4,7 +4,8 @@ module Gitlab
module Issues
class Importer
def initialize(project, after = nil)
- @project, @after = project, after
+ @project = project
+ @after = after
end
def execute
diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb
index c17f3e1729a..9c419ecb700 100644
--- a/lib/gitlab/phabricator_import/issues/task_importer.rb
+++ b/lib/gitlab/phabricator_import/issues/task_importer.rb
@@ -4,7 +4,8 @@ module Gitlab
module Issues
class TaskImporter
def initialize(project, task)
- @project, @task = project, task
+ @project = project
+ @task = task
end
def execute
diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb
index b37a5b44980..c842798ca74 100644
--- a/lib/gitlab/phabricator_import/project_creator.rb
+++ b/lib/gitlab/phabricator_import/project_creator.rb
@@ -55,12 +55,13 @@ module Gitlab
end
def project_feature_attributes
- @project_features_attributes ||= begin
- # everything disabled except for issues
- ProjectFeature::FEATURES.map do |feature|
- [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED]
- end.to_h.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED)
- end
+ @project_features_attributes ||=
+ begin
+ # everything disabled except for issues
+ ProjectFeature::FEATURES.to_h do |feature|
+ [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED]
+ end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED)
+ end
end
def import_data
diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb
index 4b50431e0e0..c6058d12527 100644
--- a/lib/gitlab/phabricator_import/user_finder.rb
+++ b/lib/gitlab/phabricator_import/user_finder.rb
@@ -4,7 +4,8 @@ module Gitlab
module PhabricatorImport
class UserFinder
def initialize(project, phids)
- @project, @phids = project, phids
+ @project = project
+ @phids = phids
@loaded_phids = Set.new
end
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
index 56eeea6e746..32d3eeb8cd2 100644
--- a/lib/gitlab/project_template.rb
+++ b/lib/gitlab/project_template.rb
@@ -5,7 +5,11 @@ module Gitlab
attr_reader :title, :name, :description, :preview, :logo
def initialize(name, title, description, preview, logo = 'illustrations/gitlab_logo.svg')
- @name, @title, @description, @preview, @logo = name, title, description, preview, logo
+ @name = name
+ @title = title
+ @description = description
+ @preview = preview
+ @logo = logo
end
def file
diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb
index ed10ef2917f..76e65d29c7a 100644
--- a/lib/gitlab/prometheus/adapter.rb
+++ b/lib/gitlab/prometheus/adapter.rb
@@ -19,6 +19,10 @@ module Gitlab
end
def cluster_prometheus_adapter
+ if cluster&.integration_prometheus
+ return cluster.integration_prometheus
+ end
+
application = cluster&.application_prometheus
application if application&.available?
diff --git a/lib/gitlab/prometheus/queries/matched_metric_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb
index e4d44df3baf..73de5a11998 100644
--- a/lib/gitlab/prometheus/queries/matched_metric_query.rb
+++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb
@@ -4,7 +4,7 @@ module Gitlab
module Prometheus
module Queries
class MatchedMetricQuery < BaseQuery
- MAX_QUERY_ITEMS = 40.freeze
+ MAX_QUERY_ITEMS = 40
def query
groups_data.map do |group, data|
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index 965349ad711..0fcf63d03fc 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -140,7 +140,7 @@ module Gitlab
end
def mapped_options
- options.keys.map { |k| [gitlab_http_key(k), options[k]] }.to_h
+ options.keys.to_h { |k| [gitlab_http_key(k), options[k]] }
end
def http_options
diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb
index 02446a7953b..ce9fced9465 100644
--- a/lib/gitlab/push_options.rb
+++ b/lib/gitlab/push_options.rb
@@ -5,6 +5,7 @@ module Gitlab
VALID_OPTIONS = HashWithIndifferentAccess.new({
merge_request: {
keys: [
+ :assign,
:create,
:description,
:label,
@@ -12,6 +13,7 @@ module Gitlab
:remove_source_branch,
:target,
:title,
+ :unassign,
:unlabel
]
},
@@ -23,7 +25,9 @@ module Gitlab
MULTI_VALUE_OPTIONS = [
%w[ci variable],
%w[merge_request label],
- %w[merge_request unlabel]
+ %w[merge_request unlabel],
+ %w[merge_request assign],
+ %w[merge_request unassign]
].freeze
NAMESPACE_ALIASES = HashWithIndifferentAccess.new({
diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb
index 5e46e26e14e..03386dca141 100644
--- a/lib/gitlab/query_limiting.rb
+++ b/lib/gitlab/query_limiting.rb
@@ -6,28 +6,36 @@ module Gitlab
#
# This is only enabled in development and test to ensure we don't produce
# any errors that users of other environments can't do anything about themselves.
- def self.enable?
+ def self.enabled_for_env?
Rails.env.development? || Rails.env.test?
end
+ def self.enabled?
+ enabled_for_env? &&
+ !Gitlab::SafeRequestStore[:query_limiting_disabled]
+ end
+
# Allows the current request to execute any number of SQL queries.
#
# This method should _only_ be used when there's a corresponding issue to
# reduce the number of queries.
#
# The issue URL is only meant to push developers into creating an issue
- # instead of blindly whitelisting offending blocks of code.
- def self.whitelist(issue_url)
- return unless enable?
-
+ # instead of blindly disabling for offending blocks of code.
+ def self.disable!(issue_url)
unless issue_url.start_with?('https://')
raise(
ArgumentError,
- 'You must provide a valid issue URL in order to whitelist a block of code'
+ 'You must provide a valid issue URL in order to allow a block of code'
)
end
- Transaction&.current&.whitelisted = true
+ Gitlab::SafeRequestStore[:query_limiting_disabled] = true
+ end
+
+ # Enables query limiting for the request.
+ def self.enable!
+ Gitlab::SafeRequestStore[:query_limiting_disabled] = nil
end
end
end
diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb
index 196072dddda..643b2540c37 100644
--- a/lib/gitlab/query_limiting/transaction.rb
+++ b/lib/gitlab/query_limiting/transaction.rb
@@ -5,7 +5,7 @@ module Gitlab
class Transaction
THREAD_KEY = :__gitlab_query_counts_transaction
- attr_accessor :count, :whitelisted
+ attr_accessor :count
# The name of the action (e.g. `UsersController#show`) that is being
# executed.
@@ -45,7 +45,6 @@ module Gitlab
def initialize
@action = nil
@count = 0
- @whitelisted = false
@sql_executed = []
end
@@ -59,7 +58,7 @@ module Gitlab
end
def increment
- @count += 1 unless whitelisted
+ @count += 1 if enabled?
end
def executed_sql(sql)
@@ -83,6 +82,10 @@ module Gitlab
["#{header}: #{msg}", log, ellipsis].compact.join("\n")
end
+
+ def enabled?
+ ::Gitlab::QueryLimiting.enabled?
+ end
end
end
end
diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb
index b17a0208f95..8ce13db4c03 100644
--- a/lib/gitlab/quick_actions/command_definition.rb
+++ b/lib/gitlab/quick_actions/command_definition.rb
@@ -56,15 +56,18 @@ module Gitlab
end
def execute(context, arg)
- return unless executable?(context)
+ return if noop?
count_commands_executed_in(context)
+ return unless available?(context)
+
execute_block(action_block, context, arg)
end
def execute_message(context, arg)
- return unless executable?(context)
+ return if noop?
+ return _('Could not apply %{name} command.') % { name: name } unless available?(context)
if execution_message.respond_to?(:call)
execute_block(execution_message, context, arg)
@@ -101,10 +104,6 @@ module Gitlab
private
- def executable?(context)
- !noop? && available?(context)
- end
-
def count_commands_executed_in(context)
return unless context.respond_to?(:commands_executed_count=)
diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
index 4934c12a339..b7d58e05651 100644
--- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
+++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb
@@ -182,7 +182,7 @@ module Gitlab
parse_params do |raw_time_date|
Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute
end
- command :spend do |time_spent, time_spent_date|
+ command :spend, :spent do |time_spent, time_spent_date|
if time_spent
@updates[:spend_time] = {
duration: time_spent,
diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb
index 67e3a5de223..bd6d2e016b4 100644
--- a/lib/gitlab/rack_attack/request.rb
+++ b/lib/gitlab/rack_attack/request.rb
@@ -34,12 +34,16 @@ module Gitlab
path =~ %r{^/-/(health|liveness|readiness|metrics)}
end
+ def container_registry_event?
+ path =~ %r{^/api/v\d+/container_registry_event/}
+ end
+
def product_analytics_collector_request?
path.start_with?('/-/collector/i')
end
def should_be_skipped?
- api_internal_request? || health_check_request?
+ api_internal_request? || health_check_request? || container_registry_event?
end
def web_request?
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 00739c05386..488ba04f87c 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -181,7 +181,7 @@ module Gitlab
end
def generic_package_version_regex
- /\A\d+\.\d+\.\d+\z/
+ maven_version_regex
end
def generic_package_name_regex
@@ -385,11 +385,11 @@ module Gitlab
end
def merge_request_wip
- /(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/
+ /(?i)(\[WIP\]\s*|WIP:\s*|\AWIP\z)/
end
def merge_request_draft
- /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/
+ /\A(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft\z)/
end
def issue
diff --git a/lib/gitlab/relative_positioning/closed_range.rb b/lib/gitlab/relative_positioning/closed_range.rb
index 8916d1face5..11fba05edee 100644
--- a/lib/gitlab/relative_positioning/closed_range.rb
+++ b/lib/gitlab/relative_positioning/closed_range.rb
@@ -4,7 +4,8 @@ module Gitlab
module RelativePositioning
class ClosedRange < RelativePositioning::Range
def initialize(lhs, rhs)
- @lhs, @rhs = lhs, rhs
+ @lhs = lhs
+ @rhs = rhs
raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs
raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs
end
diff --git a/lib/gitlab/relative_positioning/gap.rb b/lib/gitlab/relative_positioning/gap.rb
index ab894141a60..2e30e598eb0 100644
--- a/lib/gitlab/relative_positioning/gap.rb
+++ b/lib/gitlab/relative_positioning/gap.rb
@@ -6,7 +6,8 @@ module Gitlab
attr_reader :start_pos, :end_pos
def initialize(start_pos, end_pos)
- @start_pos, @end_pos = start_pos, end_pos
+ @start_pos = start_pos
+ @end_pos = end_pos
end
def ==(other)
diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb
index eb7c9bccf96..d0230c035cc 100644
--- a/lib/gitlab/repository_cache_adapter.rb
+++ b/lib/gitlab/repository_cache_adapter.rb
@@ -60,14 +60,17 @@ module Gitlab
define_method("#{name}_include?") do |value|
ivar = "@#{name}_include"
memoized = instance_variable_get(ivar) || {}
+ lookup = proc { __send__(name).include?(value) } # rubocop:disable GitlabSecurity/PublicSend
next memoized[value] if memoized.key?(value)
memoized[value] =
- if strong_memoized?(name) || !redis_set_cache.exist?(name)
- __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend
+ if strong_memoized?(name)
+ lookup.call
else
- redis_set_cache.include?(name, value)
+ result, exists = redis_set_cache.try_include?(name, value)
+
+ exists ? result : lookup.call
end
instance_variable_set(ivar, memoized)[value]
diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb
index d479d3115a6..430f3e8d162 100644
--- a/lib/gitlab/repository_hash_cache.rb
+++ b/lib/gitlab/repository_hash_cache.rb
@@ -148,7 +148,7 @@ module Gitlab
# @param hash [Hash]
# @return [Hash] the stringified hash
def standardize_hash(hash)
- hash.map { |k, v| [k.to_s, v.to_s] }.to_h
+ hash.to_h { |k, v| [k.to_s, v.to_s] }
end
# Record metrics in Prometheus.
diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb
index 69c1688767c..f73ac628bce 100644
--- a/lib/gitlab/repository_set_cache.rb
+++ b/lib/gitlab/repository_set_cache.rb
@@ -36,10 +36,32 @@ module Gitlab
end
def fetch(key, &block)
- if exist?(key)
- read(key)
- else
- write(key, yield)
+ full_key = cache_key(key)
+
+ smembers, exists = with do |redis|
+ redis.multi do
+ redis.smembers(full_key)
+ redis.exists(full_key)
+ end
+ end
+
+ return smembers if exists
+
+ write(key, yield)
+ end
+
+ # Searches the cache set using SSCAN with the MATCH option. The MATCH
+ # parameter is the pattern argument.
+ # See https://redis.io/commands/scan#the-match-option for more information.
+ # Returns an Enumerator that enumerates all SSCAN hits.
+ def search(key, pattern, &block)
+ full_key = cache_key(key)
+
+ with do |redis|
+ exists = redis.exists(full_key)
+ write(key, yield) unless exists
+
+ redis.sscan_each(full_key, match: pattern)
end
end
end
diff --git a/lib/gitlab/search_context.rb b/lib/gitlab/search_context.rb
index c3bb0ff26f2..0323220690a 100644
--- a/lib/gitlab/search_context.rb
+++ b/lib/gitlab/search_context.rb
@@ -129,7 +129,10 @@ module Gitlab
'wiki_blobs'
elsif view_context.current_controller?(:commits)
'commits'
- else nil
+ elsif view_context.current_controller?(:groups)
+ if %w(issues merge_requests).include?(view_context.controller.action_name)
+ view_context.controller.action_name
+ end
end
end
end
@@ -160,3 +163,5 @@ module Gitlab
end
end
end
+
+Gitlab::SearchContext::Builder.prepend_ee_mod
diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb
index 591265d014e..0f2b7b194c9 100644
--- a/lib/gitlab/set_cache.rb
+++ b/lib/gitlab/set_cache.rb
@@ -51,6 +51,19 @@ module Gitlab
with { |redis| redis.sismember(cache_key(key), value) }
end
+ # Like include?, but also tells us if the cache was populated when it ran
+ # by returning two booleans: [member_exists, set_exists]
+ def try_include?(key, value)
+ full_key = cache_key(key)
+
+ with do |redis|
+ redis.multi do
+ redis.sismember(full_key, value)
+ redis.exists(full_key)
+ end
+ end
+ end
+
def ttl(key)
with { |redis| redis.ttl(cache_key(key)) }
end
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 7561e36cc33..3ac20724403 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -98,6 +98,10 @@ module Gitlab
if Rails.env.test?
socket_filename = options[:gitaly_socket] || "gitaly.socket"
+ prometheus_listen_addr = options[:prometheus_listen_addr]
+
+ git_bin_path = File.expand_path('../gitaly/_build/deps/git/install/bin/git')
+ git_bin_path = nil unless File.exist?(git_bin_path)
config = {
# Override the set gitaly_address since Praefect is in the loop
@@ -106,8 +110,12 @@ module Gitlab
# Compared to production, tests run in constrained environments. This
# number is meant to grow with the number of concurrent rails requests /
# sidekiq jobs, and concurrency will be low anyway in test.
- git: { catfile_cache_size: 5 }
- }
+ git: {
+ catfile_cache_size: 5,
+ bin_path: git_bin_path
+ }.compact,
+ prometheus_listen_addr: prometheus_listen_addr
+ }.compact
storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s
storages << { name: 'test_second_storage', path: storage_path }
diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb
index e471517c50a..9490d543dd1 100644
--- a/lib/gitlab/sidekiq_cluster/cli.rb
+++ b/lib/gitlab/sidekiq_cluster/cli.rb
@@ -53,11 +53,11 @@ module Gitlab
'You cannot specify --queue-selector and --experimental-queue-selector together'
end
- all_queues = SidekiqConfig::CliMethods.all_queues(@rails_path)
- queue_names = SidekiqConfig::CliMethods.worker_queues(@rails_path)
+ worker_metadatas = SidekiqConfig::CliMethods.worker_metadatas(@rails_path)
+ worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path)
- queue_groups = argv.map do |queues|
- next queue_names if queues == '*'
+ queue_groups = argv.map do |queues_or_query_string|
+ next worker_queues if queues_or_query_string == SidekiqConfig::WorkerMatcher::WILDCARD_MATCH
# When using the queue query syntax, we treat each queue group
# as a worker attribute query, and resolve the queues for the
@@ -65,14 +65,14 @@ module Gitlab
# Simplify with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646
if @queue_selector || @experimental_queue_selector
- SidekiqConfig::CliMethods.query_workers(queues, all_queues)
+ SidekiqConfig::CliMethods.query_queues(queues_or_query_string, worker_metadatas)
else
- SidekiqConfig::CliMethods.expand_queues(queues.split(','), queue_names)
+ SidekiqConfig::CliMethods.expand_queues(queues_or_query_string.split(','), worker_queues)
end
end
if @negate_queues
- queue_groups.map! { |queues| queue_names - queues }
+ queue_groups.map! { |queues| worker_queues - queues }
end
if queue_groups.all?(&:empty?)
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index 633291dcdf3..78d45b5f3f0 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -13,10 +13,17 @@ module Gitlab
(EE_QUEUE_CONFIG_PATH if Gitlab.ee?)
].compact.freeze
- DEFAULT_WORKERS = [
- DummyWorker.new('default', weight: 1, tags: []),
- DummyWorker.new('mailers', weight: 2, tags: [])
- ].map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze
+ # This maps workers not in our application code to queues. We need
+ # these queues in our YAML files to ensure we don't accidentally
+ # miss jobs from these queues.
+ #
+ # The default queue should be unused, which is why it maps to an
+ # invalid class name. We keep it in the YAML file for safety, just
+ # in case anything does get scheduled to run there.
+ DEFAULT_WORKERS = {
+ '_' => DummyWorker.new('default', weight: 1, tags: []),
+ 'ActionMailer::MailDeliveryJob' => DummyWorker.new('mailers', feature_category: :issue_tracking, urgency: 'low', weight: 2, tags: [])
+ }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze
class << self
include Gitlab::SidekiqConfig::CliMethods
@@ -40,7 +47,7 @@ module Gitlab
def workers
@workers ||= begin
result = []
- result.concat(DEFAULT_WORKERS)
+ result.concat(DEFAULT_WORKERS.values)
result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false))
if Gitlab.ee?
diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb
index a256632bc12..8eef15f9ccb 100644
--- a/lib/gitlab/sidekiq_config/cli_methods.rb
+++ b/lib/gitlab/sidekiq_config/cli_methods.rb
@@ -12,35 +12,19 @@ module Gitlab
# rubocop:disable Gitlab/ModuleWithInstanceVariables
extend self
+ # The file names are misleading. Those files contain the metadata of the
+ # workers. They should be renamed to all_workers instead.
+ # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1018
QUEUE_CONFIG_PATHS = begin
result = %w[app/workers/all_queues.yml]
result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
result
end.freeze
- QUERY_OR_OPERATOR = '|'
- QUERY_AND_OPERATOR = '&'
- QUERY_CONCATENATE_OPERATOR = ','
- QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze
+ def worker_metadatas(rails_path = Rails.root.to_s)
+ @worker_metadatas ||= {}
- QUERY_PREDICATES = {
- feature_category: :to_sym,
- has_external_dependencies: lambda { |value| value == 'true' },
- name: :to_s,
- resource_boundary: :to_sym,
- tags: :to_sym,
- urgency: :to_sym
- }.freeze
-
- QueryError = Class.new(StandardError)
- InvalidTerm = Class.new(QueryError)
- UnknownOperator = Class.new(QueryError)
- UnknownPredicate = Class.new(QueryError)
-
- def all_queues(rails_path = Rails.root.to_s)
- @worker_queues ||= {}
-
- @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
+ @worker_metadatas[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
full_path = File.join(rails_path, path)
File.exist?(full_path) ? YAML.load_file(full_path) : []
@@ -49,7 +33,7 @@ module Gitlab
# rubocop:enable Gitlab/ModuleWithInstanceVariables
def worker_queues(rails_path = Rails.root.to_s)
- worker_names(all_queues(rails_path))
+ worker_names(worker_metadatas(rails_path))
end
def expand_queues(queues, all_queues = self.worker_queues)
@@ -62,13 +46,18 @@ module Gitlab
end
end
- def query_workers(query_string, queues)
- worker_names(queues.select(&query_string_to_lambda(query_string)))
+ def query_queues(query_string, worker_metadatas)
+ matcher = SidekiqConfig::WorkerMatcher.new(query_string)
+ selected_metadatas = worker_metadatas.select do |worker_metadata|
+ matcher.match?(worker_metadata)
+ end
+
+ worker_names(selected_metadatas)
end
def clear_memoization!
- if instance_variable_defined?('@worker_queues')
- remove_instance_variable('@worker_queues')
+ if instance_variable_defined?('@worker_metadatas')
+ remove_instance_variable('@worker_metadatas')
end
end
@@ -77,53 +66,6 @@ module Gitlab
def worker_names(workers)
workers.map { |queue| queue[:name] }
end
-
- def query_string_to_lambda(query_string)
- or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string|
- and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term|
- predicate_for_term(term)
- end
-
- lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } }
- end
-
- lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } }
- end
-
- def predicate_for_term(term)
- match = term.match(QUERY_TERM_REGEX)
-
- raise InvalidTerm.new("Invalid term: #{term}") unless match
-
- _, lhs, op, rhs = *match
-
- predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR)))
- end
-
- def predicate_for_op(op, predicate)
- case op
- when '='
- predicate
- when '!='
- lambda { |worker| !predicate.call(worker) }
- else
- # This is unreachable because InvalidTerm will be raised instead, but
- # keeping it allows to guard against that changing in future.
- raise UnknownOperator.new("Unknown operator: #{op}")
- end
- end
-
- def predicate_factory(lhs, values)
- values_block = QUERY_PREDICATES[lhs.to_sym]
-
- raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block
-
- lambda do |queue|
- comparator = Array(queue[lhs.to_sym]).to_set
-
- values.map(&values_block).to_set.intersect?(comparator)
- end
- end
end
end
end
diff --git a/lib/gitlab/sidekiq_config/worker_matcher.rb b/lib/gitlab/sidekiq_config/worker_matcher.rb
new file mode 100644
index 00000000000..fe5ac10c65a
--- /dev/null
+++ b/lib/gitlab/sidekiq_config/worker_matcher.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SidekiqConfig
+ class WorkerMatcher
+ WILDCARD_MATCH = '*'
+ QUERY_OR_OPERATOR = '|'
+ QUERY_AND_OPERATOR = '&'
+ QUERY_CONCATENATE_OPERATOR = ','
+ QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze
+
+ QUERY_PREDICATES = {
+ feature_category: :to_sym,
+ has_external_dependencies: lambda { |value| value == 'true' },
+ name: :to_s,
+ resource_boundary: :to_sym,
+ tags: :to_sym,
+ urgency: :to_sym
+ }.freeze
+
+ QueryError = Class.new(StandardError)
+ InvalidTerm = Class.new(QueryError)
+ UnknownOperator = Class.new(QueryError)
+ UnknownPredicate = Class.new(QueryError)
+
+ def initialize(query_string)
+ @match_lambda = query_string_to_lambda(query_string)
+ end
+
+ def match?(worker_metadata)
+ @match_lambda.call(worker_metadata)
+ end
+
+ private
+
+ def query_string_to_lambda(query_string)
+ return lambda { |_worker| true } if query_string.strip == WILDCARD_MATCH
+
+ or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string|
+ and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term|
+ predicate_for_term(term)
+ end
+
+ lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } }
+ end
+
+ lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } }
+ end
+
+ def predicate_for_term(term)
+ match = term.match(QUERY_TERM_REGEX)
+
+ raise InvalidTerm.new("Invalid term: #{term}") unless match
+
+ _, lhs, op, rhs = *match
+
+ predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR)))
+ end
+
+ def predicate_for_op(op, predicate)
+ case op
+ when '='
+ predicate
+ when '!='
+ lambda { |worker| !predicate.call(worker) }
+ else
+ # This is unreachable because InvalidTerm will be raised instead, but
+ # keeping it allows to guard against that changing in future.
+ raise UnknownOperator.new("Unknown operator: #{op}")
+ end
+ end
+
+ def predicate_factory(lhs, values)
+ values_block = QUERY_PREDICATES[lhs.to_sym]
+
+ raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block
+
+ lambda do |queue|
+ comparator = Array(queue[lhs.to_sym]).to_set
+
+ values.map(&values_block).to_set.intersect?(comparator)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index 654b17c5740..b1fb3771c78 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -39,9 +39,7 @@ module Gitlab
private
def add_instrumentation_keys!(job, output_payload)
- instrumentation_values = job.slice(*::Gitlab::InstrumentationHelper.keys).stringify_keys
-
- output_payload.merge!(instrumentation_values)
+ output_payload.merge!(job[:instrumentation].stringify_keys)
end
def add_logging_extras!(job, output_payload)
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index a2696e17078..563a105484d 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -43,3 +43,5 @@ module Gitlab
end
end
end
+
+Gitlab::SidekiqMiddleware.singleton_class.prepend_if_ee('EE::Gitlab::SidekiqMiddleware')
diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb
index 36204e1bee0..1b33743a0e9 100644
--- a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb
+++ b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb
@@ -8,7 +8,8 @@ module Gitlab
# If enabled then it injects a job field that persists through the job execution
class Client
def call(_worker_class, job, _queue, _redis_pool)
- return yield unless ::Feature.enabled?(:user_mode_in_session)
+ # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware
+ # Only when admin mode application setting is enabled might the admin_mode_user_id be non-nil here
# Admin mode enabled in the original request or in a nested sidekiq job
admin_mode_user_id = find_admin_user_id
diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb
index 6366867a0fa..c4e64705d6e 100644
--- a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb
+++ b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb
@@ -5,7 +5,8 @@ module Gitlab
module AdminMode
class Server
def call(_worker, job, _queue)
- return yield unless Feature.enabled?(:user_mode_in_session)
+ # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware
+ # Only when admin_mode setting is enabled can it be true here
admin_mode_user_id = job['admin_mode_user_id']
diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb
index a66a4de4655..b542aa4fe4c 100644
--- a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb
+++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb
@@ -3,6 +3,24 @@
module Gitlab
module SidekiqMiddleware
class InstrumentationLogger
+ def self.keys
+ @keys ||= [
+ :cpu_s,
+ :gitaly_calls,
+ :gitaly_duration_s,
+ :rugged_calls,
+ :rugged_duration_s,
+ :elasticsearch_calls,
+ :elasticsearch_duration_s,
+ :elasticsearch_timed_out_count,
+ *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values,
+ *::Gitlab::Instrumentation::Redis.known_payload_keys,
+ *::Gitlab::Metrics::Subscribers::ActiveRecord.known_payload_keys,
+ *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS,
+ *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS
+ ]
+ end
+
def call(worker, job, queue)
::Gitlab::InstrumentationHelper.init_instrumentation_data
@@ -17,7 +35,10 @@ module Gitlab
# because Sidekiq keeps a pristine copy of the original hash
# before sending it to the middleware:
# https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118
- ::Gitlab::InstrumentationHelper.add_instrumentation_data(job)
+ job[:instrumentation] = {}.tap do |instrumentation_values|
+ ::Gitlab::InstrumentationHelper.add_instrumentation_data(instrumentation_values)
+ instrumentation_values.slice!(*self.class.keys)
+ end
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware/metrics_helper.rb b/lib/gitlab/sidekiq_middleware/metrics_helper.rb
index 60e79ee1188..66930a34319 100644
--- a/lib/gitlab/sidekiq_middleware/metrics_helper.rb
+++ b/lib/gitlab/sidekiq_middleware/metrics_helper.rb
@@ -10,6 +10,7 @@ module Gitlab
def create_labels(worker_class, queue, job)
worker_name = (job['wrapped'].presence || worker_class).to_s
+ worker = find_worker(worker_name, worker_class)
labels = { queue: queue.to_s,
worker: worker_name,
@@ -18,15 +19,15 @@ module Gitlab
feature_category: "",
boundary: "" }
- return labels unless worker_class && worker_class.include?(WorkerAttributes)
+ return labels unless worker.respond_to?(:get_urgency)
- labels[:urgency] = worker_class.get_urgency.to_s
- labels[:external_dependencies] = bool_as_label(worker_class.worker_has_external_dependencies?)
+ labels[:urgency] = worker.get_urgency.to_s
+ labels[:external_dependencies] = bool_as_label(worker.worker_has_external_dependencies?)
- feature_category = worker_class.get_feature_category
+ feature_category = worker.get_feature_category
labels[:feature_category] = feature_category.to_s
- resource_boundary = worker_class.get_worker_resource_boundary
+ resource_boundary = worker.get_worker_resource_boundary
labels[:boundary] = resource_boundary == :unknown ? "" : resource_boundary.to_s
labels
@@ -35,6 +36,10 @@ module Gitlab
def bool_as_label(value)
value ? TRUE_LABEL : FALSE_LABEL
end
+
+ def find_worker(worker_name, worker_class)
+ Gitlab::SidekiqConfig::DEFAULT_WORKERS.fetch(worker_name, worker_class)
+ end
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb
index cf768811ffd..f5fee8050ac 100644
--- a/lib/gitlab/sidekiq_middleware/server_metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb
@@ -21,6 +21,16 @@ module Gitlab
Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME
labels = create_labels(worker.class, queue, job)
+ instrument(job, labels) do
+ yield
+ end
+ end
+
+ protected
+
+ attr_reader :metrics
+
+ def instrument(job, labels)
queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job)
@metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration
@@ -50,19 +60,18 @@ module Gitlab
# job_status: done, fail match the job_status attribute in structured logging
labels[:job_status] = job_succeeded ? "done" : "fail"
+ instrumentation = job[:instrumentation] || {}
@metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime)
@metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time)
@metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000)
- @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(job))
- @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(job))
- @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(job))
- @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(job))
- @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(job))
+ @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(instrumentation))
+ @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(instrumentation))
+ @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation))
+ @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation))
+ @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation))
end
end
- private
-
def init_metrics
{
sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
@@ -81,29 +90,33 @@ module Gitlab
}
end
+ private
+
def get_thread_cputime
defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
end
- def get_redis_time(job)
- job.fetch(:redis_duration_s, 0)
+ def get_redis_time(payload)
+ payload.fetch(:redis_duration_s, 0)
end
- def get_redis_calls(job)
- job.fetch(:redis_calls, 0)
+ def get_redis_calls(payload)
+ payload.fetch(:redis_calls, 0)
end
- def get_elasticsearch_time(job)
- job.fetch(:elasticsearch_duration_s, 0)
+ def get_elasticsearch_time(payload)
+ payload.fetch(:elasticsearch_duration_s, 0)
end
- def get_elasticsearch_calls(job)
- job.fetch(:elasticsearch_calls, 0)
+ def get_elasticsearch_calls(payload)
+ payload.fetch(:elasticsearch_calls, 0)
end
- def get_gitaly_time(job)
- job.fetch(:gitaly_duration_s, 0)
+ def get_gitaly_time(payload)
+ payload.fetch(:gitaly_duration_s, 0)
end
end
end
end
+
+Gitlab::SidekiqMiddleware::ServerMetrics.prepend_if_ee('EE::Gitlab::SidekiqMiddleware::ServerMetrics')
diff --git a/lib/gitlab/sidekiq_queue.rb b/lib/gitlab/sidekiq_queue.rb
index 807c27a71ff..4b71dfc0c1b 100644
--- a/lib/gitlab/sidekiq_queue.rb
+++ b/lib/gitlab/sidekiq_queue.rb
@@ -21,7 +21,7 @@ module Gitlab
job_search_metadata =
search_metadata
.stringify_keys
- .slice(*Labkit::Context::KNOWN_KEYS)
+ .slice(*Gitlab::ApplicationContext::KNOWN_KEYS)
.transform_keys { |key| "meta.#{key}" }
.compact
diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb
index fcc120112f2..e184afa0032 100644
--- a/lib/gitlab/slash_commands/base_command.rb
+++ b/lib/gitlab/slash_commands/base_command.rb
@@ -36,7 +36,9 @@ module Gitlab
attr_accessor :project, :current_user, :params, :chat_name
def initialize(project, chat_name, params = {})
- @project, @current_user, @params = project, chat_name.user, params.dup
+ @project = project
+ @current_user = chat_name.user
+ @params = params.dup
@chat_name = chat_name
end
diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb
index 552456f5836..8841fef702e 100644
--- a/lib/gitlab/slash_commands/presenters/issue_new.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_new.rb
@@ -12,16 +12,18 @@ module Gitlab
private
- def fallback_message
- "New issue #{issue.to_reference}: #{issue.title}"
+ def pretext
+ "I created an issue on #{author_profile_link}'s behalf: *#{issue_link}* in #{project_link}"
end
- def fields_with_markdown
- %i(title pretext text fields)
+ def issue_link
+ "[#{issue.to_reference}](#{project_issue_url(issue.project, issue)})"
end
- def pretext
- "I created an issue on #{author_profile_link}'s behalf: *#{issue.to_reference}* in #{project_link}"
+ def response_message(custom_pretext: pretext)
+ {
+ text: pretext
+ }
end
end
end
diff --git a/lib/gitlab/slash_commands/run.rb b/lib/gitlab/slash_commands/run.rb
index 10a545e28ac..40fd7ee4f20 100644
--- a/lib/gitlab/slash_commands/run.rb
+++ b/lib/gitlab/slash_commands/run.rb
@@ -5,7 +5,7 @@ module Gitlab
# Slash command for triggering chatops jobs.
class Run < BaseCommand
def self.match(text)
- /\Arun\s+(?<command>\S+)(\s+(?<arguments>.+))?\z/.match(text)
+ /\Arun\s+(?<command>\S+)(\s+(?<arguments>.+))?\z/m.match(text)
end
def self.help_message
diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb
index 1b87d3bb626..fd70def8e7c 100644
--- a/lib/gitlab/slug/environment.rb
+++ b/lib/gitlab/slug/environment.rb
@@ -26,16 +26,13 @@ module Gitlab
# Repeated dashes are invalid (OpenShift limitation)
slugified.squeeze!('-')
- slugified =
- if slugified.size > 24 || slugified != name
- # Maximum length: 24 characters (OpenShift limitation)
- shorten_and_add_suffix(slugified)
- else
- # Cannot end with a dash (Kubernetes label limitation)
- slugified.chomp('-')
- end
-
- slugified
+ if slugified.size > 24 || slugified != name
+ # Maximum length: 24 characters (OpenShift limitation)
+ shorten_and_add_suffix(slugified)
+ else
+ # Cannot end with a dash (Kubernetes label limitation)
+ slugified.chomp('-')
+ end
end
private
diff --git a/lib/gitlab/sql/cte.rb b/lib/gitlab/sql/cte.rb
index 7817a2a1ce2..8f37602aeaa 100644
--- a/lib/gitlab/sql/cte.rb
+++ b/lib/gitlab/sql/cte.rb
@@ -15,20 +15,27 @@ module Gitlab
# Namespace
# with(cte.to_arel).
# from(cte.alias_to(ns))
+ #
+ # To skip materialization of the CTE query by passing materialized: false
+ # More context: https://www.postgresql.org/docs/12/queries-with.html
+ #
+ # cte = CTE.new(:my_cte_name, materialized: false)
+ #
class CTE
attr_reader :table, :query
# name - The name of the CTE as a String or Symbol.
- def initialize(name, query)
+ def initialize(name, query, materialized: true)
@table = Arel::Table.new(name)
@query = query
+ @materialized = materialized
end
# Returns the Arel relation for this CTE.
def to_arel
sql = Arel::Nodes::SqlLiteral.new("(#{query.to_sql})")
- Arel::Nodes::As.new(table, sql)
+ Gitlab::Database::AsWithMaterialized.new(table, sql, materialized: @materialized)
end
# Returns an "AS" statement that aliases the CTE name as the given table
diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb
index e45ac5d4765..607ce10d778 100644
--- a/lib/gitlab/sql/recursive_cte.rb
+++ b/lib/gitlab/sql/recursive_cte.rb
@@ -23,9 +23,11 @@ module Gitlab
attr_reader :table
# name - The name of the CTE as a String or Symbol.
- def initialize(name)
+ # union_args - The arguments supplied to Gitlab::SQL::Union class when building inner recursive query
+ def initialize(name, union_args: {})
@table = Arel::Table.new(name)
@queries = []
+ @union_args = union_args
end
# Adds a query to the body of the CTE.
@@ -37,7 +39,7 @@ module Gitlab
# Returns the Arel relation for this CTE.
def to_arel
- sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries).to_sql)
+ sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries, **@union_args).to_sql)
Arel::Nodes::As.new(table, Arel::Nodes::Grouping.new(sql))
end
diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb
index d58a1415493..59a808eafa9 100644
--- a/lib/gitlab/sql/set_operator.rb
+++ b/lib/gitlab/sql/set_operator.rb
@@ -8,6 +8,9 @@ module Gitlab
# ORDER BYs are dropped from the relations as the final sort order is not
# guaranteed any way.
#
+ # remove_order: false option can be used in special cases where the
+ # ORDER BY is necessary for the query.
+ #
# Example usage:
#
# union = Gitlab::SQL::Union.new([user.personal_projects, user.projects])
@@ -15,9 +18,10 @@ module Gitlab
#
# Project.where("id IN (#{sql})")
class SetOperator
- def initialize(relations, remove_duplicates: true)
+ def initialize(relations, remove_duplicates: true, remove_order: true)
@relations = relations
@remove_duplicates = remove_duplicates
+ @remove_order = remove_order
end
def self.operator_keyword
@@ -30,7 +34,9 @@ module Gitlab
# By using "unprepared_statements" we remove the usage of placeholders
# (thus fixing this problem), at a slight performance cost.
fragments = ActiveRecord::Base.connection.unprepared_statement do
- relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?)
+ relations.map do |rel|
+ remove_order ? rel.reorder(nil).to_sql : rel.to_sql
+ end.reject(&:blank?)
end
if fragments.any?
@@ -47,7 +53,7 @@ module Gitlab
private
- attr_reader :relations, :remove_duplicates
+ attr_reader :relations, :remove_duplicates, :remove_order
end
end
end
diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb
index 7fb3487a5e5..c4e95284c50 100644
--- a/lib/gitlab/sql/union.rb
+++ b/lib/gitlab/sql/union.rb
@@ -4,8 +4,8 @@ module Gitlab
module SQL
# Class for building SQL UNION statements.
#
- # ORDER BYs are dropped from the relations as the final sort order is not
- # guaranteed any way.
+ # By default ORDER BYs are dropped from the relations as the final sort
+ # order is not guaranteed any way.
#
# Example usage:
#
diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb
index 315c603c1dd..4180f6ccf00 100644
--- a/lib/gitlab/static_site_editor/config/file_config.rb
+++ b/lib/gitlab/static_site_editor/config/file_config.rb
@@ -28,7 +28,7 @@ module Gitlab
def to_hash_with_defaults
# NOTE: The current approach of simply mapping all the descendents' keys and values ('config')
# into a flat hash may need to be enhanced as we add more complex, non-scalar entries.
- @global.descendants.map { |descendant| [descendant.key, descendant.config] }.to_h
+ @global.descendants.to_h { |descendant| [descendant.key, descendant.config] }
end
private
diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb
index a918e7bec80..3072210d7c8 100644
--- a/lib/gitlab/subscription_portal.rb
+++ b/lib/gitlab/subscription_portal.rb
@@ -6,6 +6,11 @@ module Gitlab
::Gitlab.dev_or_test_env? ? 'https://customers.stg.gitlab.com' : 'https://customers.gitlab.com'
end
- SUBSCRIPTIONS_URL = ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url).freeze
+ def self.subscriptions_url
+ ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url)
+ end
end
end
+
+Gitlab::SubscriptionPortal.prepend_if_jh('JH::Gitlab::SubscriptionPortal')
+Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze
diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb
index dc006877129..31e11f73fe7 100644
--- a/lib/gitlab/template/base_template.rb
+++ b/lib/gitlab/template/base_template.rb
@@ -130,10 +130,10 @@ module Gitlab
return [] if project && !project.repository.exists?
if categories.any?
- categories.keys.map do |category|
+ categories.keys.to_h do |category|
files = self.by_category(category, project)
[category, files.map { |t| { key: t.key, name: t.name, content: t.content } }]
- end.to_h
+ end
else
files = self.all(project)
files.map { |t| { key: t.key, name: t.name, content: t.content } }
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index 9bb793a75cc..b16ae39bcee 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -4,35 +4,18 @@ module Gitlab
module Tracking
SNOWPLOW_NAMESPACE = 'gl'
- module ControllerConcern
- extend ActiveSupport::Concern
-
- protected
-
- def track_event(action = action_name, **args)
- category = args.delete(:category) || self.class.name
- Gitlab::Tracking.event(category, action.to_s, **args)
- end
-
- def track_self_describing_event(schema_url, data:, **args)
- Gitlab::Tracking.self_describing_event(schema_url, data: data, **args)
- end
- end
-
class << self
def enabled?
Gitlab::CurrentSettings.snowplow_enabled?
end
- def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil) # rubocop:disable Metrics/ParameterLists
- contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace).to_context, *context]
+ def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists
+ contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context]
snowplow.event(category, action, label: label, property: property, value: value, context: contexts)
product_analytics.event(category, action, label: label, property: property, value: value, context: contexts)
- end
-
- def self_describing_event(schema_url, data:, context: nil)
- snowplow.self_describing_event(schema_url, data: data, context: context)
+ rescue => error
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action)
end
def snowplow_options(group)
diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb
index 4fa844de325..e548532e061 100644
--- a/lib/gitlab/tracking/destinations/snowplow.rb
+++ b/lib/gitlab/tracking/destinations/snowplow.rb
@@ -15,13 +15,6 @@ module Gitlab
tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i)
end
- def self_describing_event(schema_url, data:, context: nil)
- return unless enabled?
-
- event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, data)
- tracker.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i)
- end
-
private
def enabled?
diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb
index 8ce16c11267..da030649f76 100644
--- a/lib/gitlab/tracking/standard_context.rb
+++ b/lib/gitlab/tracking/standard_context.rb
@@ -3,11 +3,11 @@
module Gitlab
module Tracking
class StandardContext
- GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-3'.freeze
- GITLAB_RAILS_SOURCE = 'gitlab-rails'.freeze
+ GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-4'
+ GITLAB_RAILS_SOURCE = 'gitlab-rails'
- def initialize(namespace: nil, project: nil, user: nil, **data)
- @data = data
+ def initialize(namespace: nil, project: nil, user: nil, **extra)
+ @extra = extra
end
def to_context
@@ -35,8 +35,9 @@ module Gitlab
def to_h
{
environment: environment,
- source: source
- }.merge(@data)
+ source: source,
+ extra: @extra
+ }
end
end
end
diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb
index 6a3e2062070..706c0925302 100644
--- a/lib/gitlab/untrusted_regexp.rb
+++ b/lib/gitlab/untrusted_regexp.rb
@@ -35,6 +35,10 @@ module Gitlab
matches
end
+ def match(text)
+ scan_regexp.match(text)
+ end
+
def match?(text)
text.present? && scan(text).present?
end
diff --git a/lib/gitlab/updated_notes_paginator.rb b/lib/gitlab/updated_notes_paginator.rb
index 3d3d0e5bf9e..d5c01bde6b3 100644
--- a/lib/gitlab/updated_notes_paginator.rb
+++ b/lib/gitlab/updated_notes_paginator.rb
@@ -37,8 +37,8 @@ module Gitlab
end
def fetch_page(relation)
- relation = relation.by_updated_at
- notes = relation.at_most(LIMIT + 1).to_a
+ relation = relation.order_updated_asc.with_order_id_asc
+ notes = relation.limit(LIMIT + 1).to_a
return [notes, false] unless notes.size > LIMIT
diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb
index 1dc660e574b..6b185a5a1e9 100644
--- a/lib/gitlab/usage/docs/helper.rb
+++ b/lib/gitlab/usage/docs/helper.rb
@@ -33,6 +33,10 @@ module Gitlab
object[:description]
end
+ def render_object_schema(object)
+ "[Object JSON schema](#{object.json_schema_path})"
+ end
+
def render_yaml_link(yaml_path)
"[YAML definition](#{yaml_path})"
end
diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml
index 19ad668019e..26f1aa4396d 100644
--- a/lib/gitlab/usage/docs/templates/default.md.haml
+++ b/lib/gitlab/usage/docs/templates/default.md.haml
@@ -27,6 +27,9 @@
= render_name(name)
\
= render_description(object.attributes)
+ - if object.has_json_schema?
+ \
+ = render_object_schema(object)
\
= render_yaml_link(object.yaml_path)
\
diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index 4cb83348478..9c4255a7c92 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -5,6 +5,7 @@ module Gitlab
class MetricDefinition
METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json')
BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master'
+ SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze
attr_reader :path
attr_reader :attributes
@@ -22,6 +23,16 @@ module Gitlab
attributes
end
+ def json_schema_path
+ return '' unless has_json_schema?
+
+ "#{BASE_REPO_PATH}/#{attributes[:object_json_schema]}"
+ end
+
+ def has_json_schema?
+ attributes[:value_type] == 'object' && attributes[:object_json_schema].present?
+ end
+
def yaml_path
"#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}"
end
@@ -29,7 +40,15 @@ module Gitlab
def validate!
unless skip_validation?
self.class.schemer.validate(attributes.stringify_keys).each do |error|
- Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`"))
+ error_message = <<~ERROR_MSG
+ Error type: #{error['type']}
+ Data: #{error['data']}
+ Path: #{error['data_pointer']}
+ Details: #{error['details']}
+ Metric file: #{path}
+ ERROR_MSG
+
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(error_message))
end
end
end
@@ -38,10 +57,11 @@ module Gitlab
class << self
def paths
- @paths ||= [Rails.root.join('config', 'metrics', '**', '*.yml')]
+ @paths ||= [Rails.root.join('config', 'metrics', '[^agg]*', '*.yml')]
end
- def definitions
+ def definitions(skip_validation: false)
+ @skip_validation = skip_validation
@definitions ||= load_all!
end
@@ -49,6 +69,10 @@ module Gitlab
@schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH))
end
+ def dump_metrics_yaml
+ @metrics_yaml ||= definitions.values.map(&:to_h).map(&:deep_stringify_keys).to_yaml
+ end
+
private
def load_all!
@@ -87,7 +111,7 @@ module Gitlab
end
def skip_validation?
- !!attributes[:skip_validation]
+ !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status])
end
end
end
diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
index 1aeca87d849..f77c8cab39c 100644
--- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb
+++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb
@@ -7,7 +7,7 @@ module Gitlab
UNION_OF_AGGREGATED_METRICS = 'OR'
INTERSECTION_OF_AGGREGATED_METRICS = 'AND'
ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze
- AGGREGATED_METRICS_PATH = Rails.root.join('lib/gitlab/usage_data_counters/aggregated_metrics/*.yml')
+ AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml')
AggregatedMetricError = Class.new(StandardError)
UnknownAggregationOperator = Class.new(AggregatedMetricError)
UnknownAggregationSource = Class.new(AggregatedMetricError)
diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
index 33f025770e0..49581169452 100644
--- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb
+++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb
@@ -6,6 +6,8 @@ module Gitlab
module NamesSuggestions
class Generator < ::Gitlab::UsageData
FREE_TEXT_METRIC_NAME = "<please fill metric name>"
+ REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>"
+ CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>"
class << self
def generate(key_path)
@@ -23,7 +25,7 @@ module Gitlab
end
def redis_usage_counter
- FREE_TEXT_METRIC_NAME
+ REDIS_EVENT_METRIC_NAME
end
def alt_usage_data(*)
@@ -31,7 +33,7 @@ module Gitlab
end
def redis_usage_data_totals(counter)
- counter.fallback_totals.transform_values { |_| FREE_TEXT_METRIC_NAME}
+ counter.fallback_totals.transform_values { |_| REDIS_EVENT_METRIC_NAME }
end
def sum(relation, column, *rest)
@@ -47,49 +49,160 @@ module Gitlab
end
def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil)
- parts = [prefix]
+ # rubocop: disable CodeReuse/ActiveRecord
+ relation = relation.unscope(where: :created_at)
+ # rubocop: enable CodeReuse/ActiveRecord
- if column
- parts << parse_target(column)
+ parts = [prefix]
+ arel_column = arelize_column(relation, column)
+
+ # nil as column indicates that the counting would use fallback value of primary key.
+ # Because counting primary key from relation is the conceptual equal to counting all
+ # records from given relation, in order to keep name suggestion more condensed
+ # primary key column is skipped.
+ # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not
+ # as count_id_from_issues since it does not add more information to the name suggestion
+ if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key]
+ parts << arel_column.name
parts << 'from'
end
- source = parse_source(relation)
- constraints = parse_constraints(relation: relation, column: column, distinct: distinct)
+ arel = arel_query(relation: relation, column: arel_column, distinct: distinct)
+ constraints = parse_constraints(relation: relation, arel: arel)
+
+ # In some cases due to performance reasons metrics are instrumented with joined relations
+ # where relation listed in FROM statement is not the one that includes counted attribute
+ # in such situations to make name suggestion more intuitive source should be inferred based
+ # on the relation that provide counted attribute
+ # EG: SELECT COUNT(deployments.environment_id) FROM clusters
+ # JOIN deployments ON deployments.cluster_id = cluster.id
+ # should be translated into:
+ # count_environment_id_from_deployments_with_clusters
+ # instead of
+ # count_environment_id_from_clusters_with_deployments
+ actual_source = parse_source(relation, arel_column)
+
+ append_constraints_prompt(actual_source, [constraints], parts)
+
+ parts << actual_source
+ parts += process_joined_relations(actual_source, arel, relation, constraints)
+ parts.compact.join('_').delete('"')
+ end
- if constraints.include?(source)
- parts << "<adjective describing: '#{constraints}'>"
- end
+ def append_constraints_prompt(target, constraints, parts)
+ applicable_constraints = constraints.select { |constraint| constraint.include?(target) }
+ return unless applicable_constraints.any?
- parts << source
- parts.compact.join('_')
+ parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') }
end
- def parse_constraints(relation:, column: nil, distinct: nil)
+ def parse_constraints(relation:, arel:)
connection = relation.connection
::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints
.new(connection)
- .accept(arel(relation: relation, column: column, distinct: distinct), collector(connection))
+ .accept(arel, collector(connection))
.value
end
- def parse_target(column)
- if column.is_a?(Arel::Attribute)
- "#{column.relation.name}.#{column.name}"
- else
+ # TODO: joins with `USING` keyword
+ def process_joined_relations(actual_source, arel, relation, where_constraints)
+ joins = parse_joins(connection: relation.connection, arel: arel)
+ return [] unless joins.any?
+
+ sources = [relation.table_name, *joins.map { |join| join[:source] }]
+ joins = extract_joins_targets(joins, sources)
+
+ relations = if actual_source != relation.table_name
+ build_relations_tree(joins + [{ source: relation.table_name }], actual_source)
+ else
+ # in case where counter attribute comes from joined relations, the relations
+ # diagram has to be built bottom up, thus source and target are reverted
+ build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source)
+ end
+
+ collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints)
+ end
+
+ def parse_joins(connection:, arel:)
+ ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins
+ .new(connection)
+ .accept(arel)
+ end
+
+ def extract_joins_targets(joins, sources)
+ joins.map do |join|
+ source_regex = /(#{join[:source]})\.(\w+_)*id/i
+
+ tables_except_src = (sources - [join[:source]]).join('|')
+ target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i
+
+ join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i
+ matched = join_cond_regex.match(join[:constraints])
+
+ if matched
+ join[:target] = matched[:target]
+ join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '')
+ end
+
+ join
+ end
+ end
+
+ def build_relations_tree(joins, parent, source_key: :source, target_key: :target)
+ return [] if joins.blank?
+
+ tree = {}
+ tree[parent] = []
+
+ joins.each do |join|
+ if join[source_key] == parent
+ tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key)
+ end
+ end
+ tree
+ end
+
+ def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle)
+ conjunction = conjunctions.next
+ relations.each do |subtree|
+ subtree.each do |parent, children|
+ parts << "<#{conjunction}>"
+ join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints)
+ append_constraints_prompt(parent, [wheres, join_constraints].compact, parts)
+ parts << parent
+ collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions)
+ end
+ end
+ parts
+ end
+
+ def arelize_column(relation, column)
+ case column
+ when Arel::Attribute
column
+ when NilClass
+ Arel::Table.new(relation.table_name)[relation.primary_key]
+ when String
+ if column.include?('.')
+ table, col = column.split('.')
+ Arel::Table.new(table)[col]
+ else
+ Arel::Table.new(relation.table_name)[column]
+ end
+ when Symbol
+ arelize_column(relation, column.to_s)
end
end
- def parse_source(relation)
- relation.table_name
+ def parse_source(relation, column)
+ column.relation.name || relation.table_name
end
def collector(connection)
Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new)
end
- def arel(relation:, column: nil, distinct: nil)
+ def arel_query(relation:, column: nil, distinct: nil)
column ||= relation.primary_key
if column.is_a?(Arel::Attribute)
diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb
new file mode 100644
index 00000000000..d52e4903f3c
--- /dev/null
+++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Usage
+ module Metrics
+ module NamesSuggestions
+ module RelationParsers
+ class Joins < ::Arel::Visitors::PostgreSQL
+ def accept(object)
+ object.source.right.map do |join|
+ visit(join, collector)
+ end
+ end
+
+ private
+
+ # rubocop:disable Naming/MethodName
+ def visit_Arel_Nodes_StringJoin(object, collector)
+ result = visit(object.left, collector)
+ source, constraints = result.value.split('ON')
+ {
+ source: source.split('JOIN').last&.strip,
+ constraints: constraints&.strip
+ }.compact
+ end
+
+ def visit_Arel_Nodes_FullOuterJoin(object, _)
+ parse_join(object)
+ end
+
+ def visit_Arel_Nodes_OuterJoin(object, _)
+ parse_join(object)
+ end
+
+ def visit_Arel_Nodes_RightOuterJoin(object, _)
+ parse_join(object)
+ end
+
+ def visit_Arel_Nodes_InnerJoin(object, _)
+ {
+ source: visit(object.left, collector).value,
+ constraints: object.right ? visit(object.right.expr, collector).value : nil
+ }.compact
+ end
+ # rubocop:enable Naming/MethodName
+
+ def parse_join(object)
+ {
+ source: visit(object.left, collector).value,
+ constraints: visit(object.right.expr, collector).value
+ }
+ end
+
+ def quote(value)
+ "#{value}"
+ end
+
+ def quote_table_name(name)
+ "#{name}"
+ end
+
+ def quote_column_name(name)
+ "#{name}"
+ end
+
+ def collector
+ Arel::Collectors::SubstituteBinds.new(@connection, Arel::Collectors::SQLString.new)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 5dc3f71329d..b36ca38cd64 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -87,7 +87,7 @@ module Gitlab
# rubocop: disable Metrics/AbcSize
# rubocop: disable CodeReuse/ActiveRecord
def system_usage_data
- issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
+ issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue))
{
counts: {
@@ -138,7 +138,7 @@ module Gitlab
in_review_folder: count(::Environment.in_review_folder),
grafana_integrated_projects: count(GrafanaIntegration.enabled),
groups: count(Group),
- issues: count(Issue, start: issue_minimum_id, finish: issue_maximum_id),
+ issues: count(Issue, start: minimum_id(Issue), finish: maximum_id(Issue)),
issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
@@ -146,9 +146,9 @@ module Gitlab
issues_created_from_alerts: total_alert_issues,
issues_created_gitlab_alerts: issues_created_manually_from_alerts,
issues_created_manually_from_alerts: issues_created_manually_from_alerts,
- incident_issues: count(::Issue.incident, start: issue_minimum_id, finish: issue_maximum_id),
- alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id),
- incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id),
+ incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)),
+ alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)),
+ incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: minimum_id(Issue), finish: maximum_id(Issue)),
keys: count(Key),
label_lists: count(List.label),
lfs_objects: count(LfsObject),
@@ -389,8 +389,8 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def container_expiration_policies_usage
results = {}
- start = ::Project.minimum(:id)
- finish = ::Project.maximum(:id)
+ start = minimum_id(Project)
+ finish = maximum_id(Project)
results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish)
# rubocop: disable UsageData/LargeTable
@@ -591,7 +591,7 @@ module Gitlab
{
events: distinct_count(::Event.where(time_period), :author_id),
groups: distinct_count(::GroupMember.where(time_period), :user_id),
- users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id),
+ users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)),
omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
user_auth_by_provider: distinct_count_user_auth_by_provider(time_period),
unique_users_all_imports: unique_users_all_imports(time_period),
@@ -636,8 +636,8 @@ module Gitlab
clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id),
clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period),
operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
- start: user_minimum_id,
- finish: user_maximum_id),
+ start: minimum_id(User),
+ finish: maximum_id(User)),
projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id),
projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id),
projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id),
@@ -691,12 +691,12 @@ module Gitlab
def usage_activity_by_stage_verify(time_period)
{
ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id),
- ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
- ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
- ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
- ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+ ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
+ ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
+ ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
+ ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
- ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+ ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)),
ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id),
clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period)
}
@@ -711,6 +711,8 @@ module Gitlab
end
def redis_hll_counters
+ return {} unless Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml)
+
{ redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data }
end
@@ -799,8 +801,8 @@ module Gitlab
end
def distinct_count_service_desk_enabled_projects(time_period)
- project_creator_id_start = user_minimum_id
- project_creator_id_finish = user_maximum_id
+ project_creator_id_start = minimum_id(User)
+ project_creator_id_finish = maximum_id(User)
distinct_count(::Project.service_desk_enabled.where(time_period), :creator_id, start: project_creator_id_start, finish: project_creator_id_finish) # rubocop: disable CodeReuse/ActiveRecord
end
@@ -832,57 +834,9 @@ module Gitlab
def total_alert_issues
# Remove prometheus table queries once they are deprecated
# To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407.
- add count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id),
- count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id),
- count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id)
- end
-
- def user_minimum_id
- strong_memoize(:user_minimum_id) do
- ::User.minimum(:id)
- end
- end
-
- def user_maximum_id
- strong_memoize(:user_maximum_id) do
- ::User.maximum(:id)
- end
- end
-
- def issue_minimum_id
- strong_memoize(:issue_minimum_id) do
- ::Issue.minimum(:id)
- end
- end
-
- def issue_maximum_id
- strong_memoize(:issue_maximum_id) do
- ::Issue.maximum(:id)
- end
- end
-
- def deployment_minimum_id
- strong_memoize(:deployment_minimum_id) do
- ::Deployment.minimum(:id)
- end
- end
-
- def deployment_maximum_id
- strong_memoize(:deployment_maximum_id) do
- ::Deployment.maximum(:id)
- end
- end
-
- def project_minimum_id
- strong_memoize(:project_minimum_id) do
- ::Project.minimum(:id)
- end
- end
-
- def project_maximum_id
- strong_memoize(:project_maximum_id) do
- ::Project.maximum(:id)
- end
+ add count(Issue.with_alert_management_alerts, start: minimum_id(Issue), finish: maximum_id(Issue)),
+ count(::Issue.with_self_managed_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)),
+ count(::Issue.with_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue))
end
def self_monitoring_project
@@ -916,7 +870,7 @@ module Gitlab
end
def deployment_count(relation)
- count relation, start: deployment_minimum_id, finish: deployment_maximum_id
+ count relation, start: minimum_id(Deployment), finish: maximum_id(Deployment)
end
def project_imports(time_period)
diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml
deleted file mode 100644
index 4c2355d526a..00000000000
--- a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml
+++ /dev/null
@@ -1,108 +0,0 @@
-# code_review_extension_category_monthly_active_users
-# This is only metrics related to the VS Code Extension for now.
-#
-# code_review_category_monthly_active_users
-# This is the user based metrics. These should only be user based metrics and only be related to the Code Review things inside of GitLab.
-#
-# code_review_group_monthly_active_users
-# This is an aggregation of both of the above aggregations. It's intended to represent all users who interact with our group across all of our categories.
----
-- name: code_review_group_monthly_active_users
- operator: OR
- feature_flag: usage_data_code_review_aggregation
- source: redis
- time_frame: [7d, 28d]
- events: [
- 'i_code_review_user_single_file_diffs',
- 'i_code_review_user_create_mr',
- 'i_code_review_user_close_mr',
- 'i_code_review_user_reopen_mr',
- 'i_code_review_user_resolve_thread',
- 'i_code_review_user_unresolve_thread',
- 'i_code_review_edit_mr_title',
- 'i_code_review_edit_mr_desc',
- 'i_code_review_user_merge_mr',
- 'i_code_review_user_create_mr_comment',
- 'i_code_review_user_edit_mr_comment',
- 'i_code_review_user_remove_mr_comment',
- 'i_code_review_user_create_review_note',
- 'i_code_review_user_publish_review',
- 'i_code_review_user_create_multiline_mr_comment',
- 'i_code_review_user_edit_multiline_mr_comment',
- 'i_code_review_user_remove_multiline_mr_comment',
- 'i_code_review_user_add_suggestion',
- 'i_code_review_user_apply_suggestion',
- 'i_code_review_user_assigned',
- 'i_code_review_user_review_requested',
- 'i_code_review_user_approve_mr',
- 'i_code_review_user_unapprove_mr',
- 'i_code_review_user_marked_as_draft',
- 'i_code_review_user_unmarked_as_draft',
- 'i_code_review_user_approval_rule_added',
- 'i_code_review_user_approval_rule_deleted',
- 'i_code_review_user_approval_rule_edited',
- 'i_code_review_user_vs_code_api_request',
- 'i_code_review_user_toggled_task_item_status',
- 'i_code_review_user_create_mr_from_issue',
- 'i_code_review_user_mr_discussion_locked',
- 'i_code_review_user_mr_discussion_unlocked',
- 'i_code_review_user_time_estimate_changed',
- 'i_code_review_user_time_spent_changed',
- 'i_code_review_user_assignees_changed',
- 'i_code_review_user_reviewers_changed',
- 'i_code_review_user_milestone_changed',
- 'i_code_review_user_labels_changed'
- ]
-- name: code_review_category_monthly_active_users
- operator: OR
- feature_flag: usage_data_code_review_aggregation
- source: redis
- time_frame: [7d, 28d]
- events: [
- 'i_code_review_user_single_file_diffs',
- 'i_code_review_user_create_mr',
- 'i_code_review_user_close_mr',
- 'i_code_review_user_reopen_mr',
- 'i_code_review_user_resolve_thread',
- 'i_code_review_user_unresolve_thread',
- 'i_code_review_edit_mr_title',
- 'i_code_review_edit_mr_desc',
- 'i_code_review_user_merge_mr',
- 'i_code_review_user_create_mr_comment',
- 'i_code_review_user_edit_mr_comment',
- 'i_code_review_user_remove_mr_comment',
- 'i_code_review_user_create_review_note',
- 'i_code_review_user_publish_review',
- 'i_code_review_user_create_multiline_mr_comment',
- 'i_code_review_user_edit_multiline_mr_comment',
- 'i_code_review_user_remove_multiline_mr_comment',
- 'i_code_review_user_add_suggestion',
- 'i_code_review_user_apply_suggestion',
- 'i_code_review_user_assigned',
- 'i_code_review_user_review_requested',
- 'i_code_review_user_approve_mr',
- 'i_code_review_user_unapprove_mr',
- 'i_code_review_user_marked_as_draft',
- 'i_code_review_user_unmarked_as_draft',
- 'i_code_review_user_approval_rule_added',
- 'i_code_review_user_approval_rule_deleted',
- 'i_code_review_user_approval_rule_edited',
- 'i_code_review_user_toggled_task_item_status',
- 'i_code_review_user_create_mr_from_issue',
- 'i_code_review_user_mr_discussion_locked',
- 'i_code_review_user_mr_discussion_unlocked',
- 'i_code_review_user_time_estimate_changed',
- 'i_code_review_user_time_spent_changed',
- 'i_code_review_user_assignees_changed',
- 'i_code_review_user_reviewers_changed',
- 'i_code_review_user_milestone_changed',
- 'i_code_review_user_labels_changed'
- ]
-- name: code_review_extension_category_monthly_active_users
- operator: OR
- feature_flag: usage_data_code_review_aggregation
- source: redis
- time_frame: [7d, 28d]
- events: [
- 'i_code_review_user_vs_code_api_request'
- ]
diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml
deleted file mode 100644
index 73a55b5d5fa..00000000000
--- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml
+++ /dev/null
@@ -1,72 +0,0 @@
-# Aggregated metrics that include EE only event names within `events:` attribute have to be defined at ee/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml
-# instead of this file.
-#- name: unique name of aggregated metric
-# operator: aggregation operator. Valid values are:
-# - "OR": counts unique elements that were observed triggering any of following events
-# - "AND": counts unique elements that were observed triggering all of following events
-# events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes
-# see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events.
-# source: defines which datasource will be used to locate events that should be included in aggregated metric. Valid values are:
-# - database
-# - redis
-# time_frame: defines time frames for aggregated metrics:
-# - 7d - last 7 days
-# - 28d - last 28 days
-# - all - all historical available data, this time frame is not available for redis source
-# feature_flag: name of development feature flag that will be checked before metrics aggregation is performed.
-# Corresponding feature flag should have `default_enabled` attribute set to `false`.
-# This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked.
----
-- name: compliance_features_track_unique_visits_union
- operator: OR
- source: redis
- time_frame: [7d, 28d]
- events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory']
-- name: product_analytics_test_metrics_union
- operator: OR
- source: redis
- time_frame: [7d, 28d]
- events: ['i_search_total', 'i_search_advanced', 'i_search_paid']
-- name: product_analytics_test_metrics_intersection
- operator: AND
- source: redis
- time_frame: [7d, 28d]
- events: ['i_search_total', 'i_search_advanced', 'i_search_paid']
-- name: incident_management_alerts_total_unique_counts
- operator: OR
- source: redis
- time_frame: [7d, 28d]
- events: [
- 'incident_management_alert_status_changed',
- 'incident_management_alert_assigned',
- 'incident_management_alert_todo',
- 'incident_management_alert_create_incident'
- ]
-- name: incident_management_incidents_total_unique_counts
- operator: OR
- source: redis
- time_frame: [7d, 28d]
- events: [
- 'incident_management_incident_created',
- 'incident_management_incident_reopened',
- 'incident_management_incident_closed',
- 'incident_management_incident_assigned',
- 'incident_management_incident_todo',
- 'incident_management_incident_comment',
- 'incident_management_incident_zoom_meeting',
- 'incident_management_incident_published',
- 'incident_management_incident_relate',
- 'incident_management_incident_unrelate',
- 'incident_management_incident_change_confidential'
- ]
-- name: i_testing_paid_monthly_active_user_total
- operator: OR
- source: redis
- time_frame: [7d, 28d]
- events: [
- 'i_testing_web_performance_widget_total',
- 'i_testing_full_code_quality_report_total',
- 'i_testing_group_code_coverage_visit_total',
- 'i_testing_load_performance_widget_total',
- 'i_testing_metrics_report_widget_total'
- ]
diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb
index d28fd17a989..4ab310a2519 100644
--- a/lib/gitlab/usage_data_counters/base_counter.rb
+++ b/lib/gitlab/usage_data_counters/base_counter.rb
@@ -22,11 +22,11 @@ module Gitlab::UsageDataCounters
end
def totals
- known_events.map { |event| [counter_key(event), read(event)] }.to_h
+ known_events.to_h { |event| [counter_key(event), read(event)] }
end
def fallback_totals
- known_events.map { |event| [counter_key(event), -1] }.to_h
+ known_events.to_h { |event| [counter_key(event), -1] }
end
def fetch_supported_event(event_name)
diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
index 772a4623280..c9106d7c6b8 100644
--- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb
@@ -2,7 +2,7 @@
module Gitlab::UsageDataCounters
class CiTemplateUniqueCounter
- REDIS_SLOT = 'ci_templates'.freeze
+ REDIS_SLOT = 'ci_templates'
# NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_`
TEMPLATE_TO_EVENT = {
@@ -20,8 +20,6 @@ module Gitlab::UsageDataCounters
class << self
def track_unique_project_event(project_id:, template:, config_source:)
- return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml)
-
if event = unique_project_event(template, config_source)
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id)
end
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
index 336bef081a6..a8691169fb8 100644
--- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -151,13 +151,16 @@ module Gitlab
aggregation = events.first[:aggregation]
keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context)
+
+ return FALLBACK unless keys.any?
+
redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) }
end
def feature_enabled?(event)
return true if event[:feature_flag].blank?
- Feature.enabled?(event[:feature_flag], default_enabled: :yaml)
+ Feature.enabled?(event[:feature_flag], default_enabled: :yaml) && Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml)
end
# Allow to add totals for events that are in the same redis slot, category and have the same aggregation level
diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
index c2662a74432..6f5f878501f 100644
--- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb
@@ -34,120 +34,120 @@ module Gitlab
ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed'
class << self
- def track_issue_created_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_CREATED, author, time)
+ def track_issue_created_action(author:)
+ track_unique_action(ISSUE_CREATED, author)
end
- def track_issue_title_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_TITLE_CHANGED, author, time)
+ def track_issue_title_changed_action(author:)
+ track_unique_action(ISSUE_TITLE_CHANGED, author)
end
- def track_issue_description_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_DESCRIPTION_CHANGED, author, time)
+ def track_issue_description_changed_action(author:)
+ track_unique_action(ISSUE_DESCRIPTION_CHANGED, author)
end
- def track_issue_assignee_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_ASSIGNEE_CHANGED, author, time)
+ def track_issue_assignee_changed_action(author:)
+ track_unique_action(ISSUE_ASSIGNEE_CHANGED, author)
end
- def track_issue_made_confidential_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_MADE_CONFIDENTIAL, author, time)
+ def track_issue_made_confidential_action(author:)
+ track_unique_action(ISSUE_MADE_CONFIDENTIAL, author)
end
- def track_issue_made_visible_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_MADE_VISIBLE, author, time)
+ def track_issue_made_visible_action(author:)
+ track_unique_action(ISSUE_MADE_VISIBLE, author)
end
- def track_issue_closed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_CLOSED, author, time)
+ def track_issue_closed_action(author:)
+ track_unique_action(ISSUE_CLOSED, author)
end
- def track_issue_reopened_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_REOPENED, author, time)
+ def track_issue_reopened_action(author:)
+ track_unique_action(ISSUE_REOPENED, author)
end
- def track_issue_label_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_LABEL_CHANGED, author, time)
+ def track_issue_label_changed_action(author:)
+ track_unique_action(ISSUE_LABEL_CHANGED, author)
end
- def track_issue_milestone_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_MILESTONE_CHANGED, author, time)
+ def track_issue_milestone_changed_action(author:)
+ track_unique_action(ISSUE_MILESTONE_CHANGED, author)
end
- def track_issue_cross_referenced_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_CROSS_REFERENCED, author, time)
+ def track_issue_cross_referenced_action(author:)
+ track_unique_action(ISSUE_CROSS_REFERENCED, author)
end
- def track_issue_moved_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_MOVED, author, time)
+ def track_issue_moved_action(author:)
+ track_unique_action(ISSUE_MOVED, author)
end
- def track_issue_related_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_RELATED, author, time)
+ def track_issue_related_action(author:)
+ track_unique_action(ISSUE_RELATED, author)
end
- def track_issue_unrelated_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_UNRELATED, author, time)
+ def track_issue_unrelated_action(author:)
+ track_unique_action(ISSUE_UNRELATED, author)
end
- def track_issue_marked_as_duplicate_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author, time)
+ def track_issue_marked_as_duplicate_action(author:)
+ track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author)
end
- def track_issue_locked_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_LOCKED, author, time)
+ def track_issue_locked_action(author:)
+ track_unique_action(ISSUE_LOCKED, author)
end
- def track_issue_unlocked_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_UNLOCKED, author, time)
+ def track_issue_unlocked_action(author:)
+ track_unique_action(ISSUE_UNLOCKED, author)
end
- def track_issue_designs_added_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_DESIGNS_ADDED, author, time)
+ def track_issue_designs_added_action(author:)
+ track_unique_action(ISSUE_DESIGNS_ADDED, author)
end
- def track_issue_designs_modified_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_DESIGNS_MODIFIED, author, time)
+ def track_issue_designs_modified_action(author:)
+ track_unique_action(ISSUE_DESIGNS_MODIFIED, author)
end
- def track_issue_designs_removed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_DESIGNS_REMOVED, author, time)
+ def track_issue_designs_removed_action(author:)
+ track_unique_action(ISSUE_DESIGNS_REMOVED, author)
end
- def track_issue_due_date_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_DUE_DATE_CHANGED, author, time)
+ def track_issue_due_date_changed_action(author:)
+ track_unique_action(ISSUE_DUE_DATE_CHANGED, author)
end
- def track_issue_time_estimate_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author, time)
+ def track_issue_time_estimate_changed_action(author:)
+ track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author)
end
- def track_issue_time_spent_changed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time)
+ def track_issue_time_spent_changed_action(author:)
+ track_unique_action(ISSUE_TIME_SPENT_CHANGED, author)
end
- def track_issue_comment_added_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_COMMENT_ADDED, author, time)
+ def track_issue_comment_added_action(author:)
+ track_unique_action(ISSUE_COMMENT_ADDED, author)
end
- def track_issue_comment_edited_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_COMMENT_EDITED, author, time)
+ def track_issue_comment_edited_action(author:)
+ track_unique_action(ISSUE_COMMENT_EDITED, author)
end
- def track_issue_comment_removed_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_COMMENT_REMOVED, author, time)
+ def track_issue_comment_removed_action(author:)
+ track_unique_action(ISSUE_COMMENT_REMOVED, author)
end
- def track_issue_cloned_action(author:, time: Time.zone.now)
- track_unique_action(ISSUE_CLONED, author, time)
+ def track_issue_cloned_action(author:)
+ track_unique_action(ISSUE_CLONED, author)
end
private
- def track_unique_action(action, author, time)
+ def track_unique_action(action, author)
return unless author
- Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time)
+ Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id)
end
end
end
diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
index 9c19c9e8b8c..3c692f2b1af 100644
--- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
+++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml
@@ -3,89 +3,74 @@
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_implicit_auto_devops_build
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_implicit_auto_devops_deploy
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_implicit_security_sast
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_implicit_security_secret_detection
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
# Explicit include:template pipeline events
- name: p_ci_templates_5_min_production_app
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_auto_devops
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_aws_cf_deploy_ec2
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_aws_deploy_ecs
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_auto_devops_build
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_auto_devops_deploy
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_auto_devops_deploy_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_security_sast
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_security_secret_detection
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
- name: p_ci_templates_terraform_base_latest
category: ci_templates
redis_slot: ci_templates
aggregation: weekly
- feature_flag: usage_data_track_ci_templates_unique_projects
diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml
index 80a79682338..077864032e8 100644
--- a/lib/gitlab/usage_data_counters/known_events/common.yml
+++ b/lib/gitlab/usage_data_counters/known_events/common.yml
@@ -91,6 +91,11 @@
redis_slot: analytics
aggregation: weekly
feature_flag: track_unique_visits
+- name: i_analytics_dev_ops_adoption
+ category: analytics
+ redis_slot: analytics
+ aggregation: weekly
+ feature_flag: track_unique_visits
- name: g_analytics_merge_request
category: analytics
redis_slot: analytics
@@ -242,6 +247,12 @@
category: incident_management_alerts
aggregation: weekly
feature_flag: usage_data_incident_management_alert_create_incident
+# Incident management on-call
+- name: i_incident_management_oncall_notification_sent
+ redis_slot: incident_management
+ category: incident_management_oncall
+ aggregation: weekly
+ feature_flag: usage_data_i_incident_management_oncall_notification_sent
# Testing category
- name: i_testing_test_case_parsed
category: testing
@@ -283,6 +294,11 @@
redis_slot: testing
aggregation: weekly
feature_flag: usage_data_i_testing_metrics_report_artifact_uploaders
+- name: i_testing_summary_widget_total
+ category: testing
+ redis_slot: testing
+ aggregation: weekly
+ feature_flag: usage_data_i_testing_summary_widget_total
# Project Management group
- name: g_project_management_issue_title_changed
category: issues_edit
@@ -444,13 +460,19 @@
redis_slot: pipeline_authoring
aggregation: weekly
feature_flag: usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile
-# Epic events
-#
-# We are using the same slot of issue events 'project_management' for
-# epic events to allow data aggregation.
-# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405
-- name: g_project_management_epic_created
- category: epics_usage
- redis_slot: project_management
- aggregation: daily
- feature_flag: track_epics_activity
+# Merge request widgets
+- name: users_expanding_secure_security_report
+ redis_slot: secure
+ category: secure
+ aggregation: weekly
+ feature_flag: users_expanding_widgets_usage_data
+- name: users_expanding_testing_code_quality_report
+ redis_slot: testing
+ category: testing
+ aggregation: weekly
+ feature_flag: users_expanding_widgets_usage_data
+- name: users_expanding_testing_accessibility_report
+ redis_slot: testing
+ category: testing
+ aggregation: weekly
+ feature_flag: users_expanding_widgets_usage_data
diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml
new file mode 100644
index 00000000000..80460dbe4d2
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml
@@ -0,0 +1,142 @@
+# Epic events
+#
+# We are using the same slot of issue events 'project_management' for
+# epic events to allow data aggregation.
+# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405
+- name: g_project_management_epic_created
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_epic_titles
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_epic_descriptions
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+# epic notes
+
+- name: g_project_management_users_creating_epic_notes
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_epic_notes
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_destroying_epic_notes
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+# start date events
+
+- name: g_project_management_users_setting_epic_start_date_as_fixed
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_fixed_epic_start_date
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_setting_epic_start_date_as_inherited
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+# due date events
+
+- name: g_project_management_users_setting_epic_due_date_as_fixed
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_updating_fixed_epic_due_date
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_setting_epic_due_date_as_inherited
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_epic_issue_added
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_epic_issue_removed
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_epic_issue_moved_from_project
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_epic_closed
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_epic_reopened
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: 'g_project_management_issue_promoted_to_epic'
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_setting_epic_confidential
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_users_setting_epic_visible
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_epic_users_changing_labels
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
+
+- name: g_project_management_epic_destroyed
+ category: epics_usage
+ redis_slot: project_management
+ aggregation: daily
+ feature_flag: track_epics_activity
diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb
index 7a76180cb08..aae2d144c5b 100644
--- a/lib/gitlab/usage_data_counters/note_counter.rb
+++ b/lib/gitlab/usage_data_counters/note_counter.rb
@@ -24,13 +24,13 @@ module Gitlab::UsageDataCounters
end
def totals
- COUNTABLE_TYPES.map do |countable_type|
+ COUNTABLE_TYPES.to_h do |countable_type|
[counter_key(countable_type), read(:create, countable_type)]
- end.to_h
+ end
end
def fallback_totals
- COUNTABLE_TYPES.map { |counter_key| [counter_key(counter_key), -1] }.to_h
+ COUNTABLE_TYPES.to_h { |counter_key| [counter_key(counter_key), -1] }
end
private
diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb
index 15c68fb3945..ed3df7dcf75 100644
--- a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb
+++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb
@@ -28,7 +28,7 @@ module Gitlab
'unassign_reviewer'
when 'request_review', 'reviewer'
'assign_reviewer'
- when 'spend'
+ when 'spend', 'spent'
event_name_for_spend(args)
when 'unassign'
event_name_for_unassign(args)
diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb
new file mode 100644
index 00000000000..1f72bf4ce26
--- /dev/null
+++ b/lib/gitlab/usage_data_non_sql_metrics.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ class UsageDataNonSqlMetrics < UsageData
+ SQL_METRIC_DEFAULT = -3
+
+ class << self
+ def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
+ SQL_METRIC_DEFAULT
+ end
+
+ def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
+ SQL_METRIC_DEFAULT
+ end
+
+ def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil)
+ SQL_METRIC_DEFAULT
+ end
+
+ def sum(relation, column, batch_size: nil, start: nil, finish: nil)
+ SQL_METRIC_DEFAULT
+ end
+
+ def histogram(relation, column, buckets:, bucket_size: buckets.size)
+ SQL_METRIC_DEFAULT
+ end
+
+ def maximum_id(model)
+ end
+
+ def minimum_id(model)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb
index c00e7a2aa13..c0dfae88fc7 100644
--- a/lib/gitlab/usage_data_queries.rb
+++ b/lib/gitlab/usage_data_queries.rb
@@ -5,11 +5,11 @@ module Gitlab
# See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091
class UsageDataQueries < UsageData
class << self
- def count(relation, column = nil, *rest)
+ def count(relation, column = nil, *args, **kwargs)
raw_sql(relation, column)
end
- def distinct_count(relation, column = nil, *rest)
+ def distinct_count(relation, column = nil, *args, **kwargs)
raw_sql(relation, column, :distinct)
end
@@ -21,14 +21,14 @@ module Gitlab
end
end
- def sum(relation, column, *rest)
+ def sum(relation, column, *args, **kwargs)
relation.select(relation.all.table[column].sum).to_sql
end
# For estimated distinct count use exact query instead of hll
# buckets query, because it can't be used to obtain estimations without
# supplementary ruby code present in Gitlab::Database::PostgresHll::BatchDistinctCounter
- def estimate_batch_distinct_count(relation, column = nil, *rest)
+ def estimate_batch_distinct_count(relation, column = nil, *args, **kwargs)
raw_sql(relation, column, :distinct)
end
@@ -36,6 +36,12 @@ module Gitlab
'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ')
end
+ def maximum_id(model)
+ end
+
+ def minimum_id(model)
+ end
+
private
def raw_sql(relation, column, distinct = nil)
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index 29f02a5912a..c1a57566640 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -99,6 +99,8 @@ module Gitlab
end
def to_boolean(value, default: nil)
+ value = value.to_s if [0, 1].include?(value)
+
return value if [true, false].include?(value)
return true if value =~ /^(true|t|yes|y|1|on)$/i
return false if value =~ /^(false|f|no|n|0|off)$/i
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb
index 854fc5c917d..efa2f7a943f 100644
--- a/lib/gitlab/utils/usage_data.rb
+++ b/lib/gitlab/utils/usage_data.rb
@@ -36,6 +36,7 @@
module Gitlab
module Utils
module UsageData
+ include Gitlab::Utils::StrongMemoize
extend self
FALLBACK = -1
@@ -209,6 +210,20 @@ module Gitlab
Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values)
end
+ def maximum_id(model)
+ key = :"#{model.name.downcase}_maximum_id"
+ strong_memoize(key) do
+ model.maximum(:id)
+ end
+ end
+
+ def minimum_id(model)
+ key = :"#{model.name.downcase}_minimum_id"
+ strong_memoize(key) do
+ model.minimum(:id)
+ end
+ end
+
private
def prometheus_client(verify:)
diff --git a/lib/gitlab/uuid.rb b/lib/gitlab/uuid.rb
index 80caf2c6788..016c25eb94b 100644
--- a/lib/gitlab/uuid.rb
+++ b/lib/gitlab/uuid.rb
@@ -9,9 +9,9 @@ module Gitlab
production: "58dc0f06-936c-43b3-93bb-71693f1b6570"
}.freeze
- UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{4}\h{8}/.freeze
+ UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{12}/.freeze
NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze
- PACK_PATTERN = "NnnnnN".freeze
+ PACK_PATTERN = "NnnnnN"
class << self
def v5(name, namespace_id: default_namespace_id)
diff --git a/lib/gitlab/web_ide/config/entry/terminal.rb b/lib/gitlab/web_ide/config/entry/terminal.rb
index 403e308d45b..ec07023461f 100644
--- a/lib/gitlab/web_ide/config/entry/terminal.rb
+++ b/lib/gitlab/web_ide/config/entry/terminal.rb
@@ -10,6 +10,7 @@ module Gitlab
class Terminal < ::Gitlab::Config::Entry::Node
include ::Gitlab::Config::Entry::Configurable
include ::Gitlab::Config::Entry::Attributable
+ include Gitlab::Utils::StrongMemoize
# By default the build will finish in a few seconds, not giving the webide
# enough time to connect to the terminal. This default script provides
@@ -51,21 +52,26 @@ module Gitlab
private
def to_hash
- { tag_list: tags || [],
- yaml_variables: yaml_variables,
+ {
+ tag_list: tags || [],
+ yaml_variables: yaml_variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+ job_variables: yaml_variables,
options: {
image: image_value,
services: services_value,
before_script: before_script_value,
script: script_value || DEFAULT_SCRIPT
- }.compact }
+ }.compact
+ }.compact
end
def yaml_variables
- return unless variables_value
+ strong_memoize(:yaml_variables) do
+ next unless variables_value
- variables_value.map do |key, value|
- { key: key.to_s, value: value, public: true }
+ variables_value.map do |key, value|
+ { key: key.to_s, value: value, public: true }
+ end
end
end
end
diff --git a/lib/gitlab/word_diff/chunk_collection.rb b/lib/gitlab/word_diff/chunk_collection.rb
index dd388f75302..d5c3e59d405 100644
--- a/lib/gitlab/word_diff/chunk_collection.rb
+++ b/lib/gitlab/word_diff/chunk_collection.rb
@@ -18,6 +18,27 @@ module Gitlab
def reset
@chunks = []
end
+
+ def marker_ranges
+ start = 0
+
+ @chunks.each_with_object([]) do |element, ranges|
+ mode = mode_for_element(element)
+
+ ranges << Gitlab::MarkerRange.new(start, start + element.length - 1, mode: mode) if mode
+
+ start += element.length
+ end
+ end
+
+ private
+
+ def mode_for_element(element)
+ return Gitlab::MarkerRange::DELETION if element.removed?
+ return Gitlab::MarkerRange::ADDITION if element.added?
+
+ nil
+ end
end
end
end
diff --git a/lib/gitlab/word_diff/parser.rb b/lib/gitlab/word_diff/parser.rb
index 3b6d4d4d384..e611abb5692 100644
--- a/lib/gitlab/word_diff/parser.rb
+++ b/lib/gitlab/word_diff/parser.rb
@@ -31,7 +31,7 @@ module Gitlab
@chunks.add(segment)
when Segments::Newline
- yielder << build_line(@chunks.content, nil, parent_file: diff_file)
+ yielder << build_line(@chunks.content, nil, parent_file: diff_file).tap { |line| line.set_marker_ranges(@chunks.marker_ranges) }
@chunks.reset
counter.increase_pos_num
diff --git a/lib/learn_gitlab.rb b/lib/learn_gitlab.rb
index 771083193d1..abceb80bd30 100644
--- a/lib/learn_gitlab.rb
+++ b/lib/learn_gitlab.rb
@@ -1,9 +1,9 @@
# frozen_string_literal: true
class LearnGitlab
- PROJECT_NAME = 'Learn GitLab'.freeze
- BOARD_NAME = 'GitLab onboarding'.freeze
- LABEL_NAME = 'Novice'.freeze
+ PROJECT_NAME = 'Learn GitLab'
+ BOARD_NAME = 'GitLab onboarding'
+ LABEL_NAME = 'Novice'
def initialize(current_user)
@current_user = current_user
diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb
index 523e673e9e1..4040bed50a9 100644
--- a/lib/peek/views/active_record.rb
+++ b/lib/peek/views/active_record.rb
@@ -17,22 +17,32 @@ module Peek
}
}.freeze
- def results
- super.merge(calls: detailed_calls)
- end
-
def self.thresholds
@thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS)
end
+ def results
+ super.merge(summary: summary)
+ end
+
private
- def detailed_calls
- "#{calls} (#{cached_calls} cached)"
+ def summary
+ detail_store.each_with_object({}) do |item, count|
+ count_summary(item, count)
+ end
end
- def cached_calls
- detail_store.count { |item| item[:cached] == 'cached' }
+ def count_summary(item, count)
+ if item[:cached].present?
+ count[item[:cached]] ||= 0
+ count[item[:cached]] += 1
+ end
+
+ if item[:transaction].present?
+ count[item[:transaction]] ||= 0
+ count[item[:transaction]] += 1
+ end
end
def setup_subscribers
@@ -45,10 +55,12 @@ module Peek
def generate_detail(start, finish, data)
{
+ start: start,
duration: finish - start,
sql: data[:sql].strip,
backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller),
- cached: data[:cached] ? 'cached' : ''
+ cached: data[:cached] ? 'Cached' : '',
+ transaction: data[:connection].transaction_open? ? 'In a transaction' : ''
}
end
end
diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb
deleted file mode 100644
index ad9de067375..00000000000
--- a/lib/quality/test_level.rb
+++ /dev/null
@@ -1,142 +0,0 @@
-# frozen_string_literal: true
-
-module Quality
- class TestLevel
- UnknownTestLevelError = Class.new(StandardError)
-
- TEST_LEVEL_FOLDERS = {
- migration: %w[
- migrations
- ],
- background_migration: %w[
- lib/gitlab/background_migration
- lib/ee/gitlab/background_migration
- ],
- frontend_fixture: %w[
- frontend/fixtures
- ],
- unit: %w[
- bin
- channels
- config
- db
- dependencies
- elastic
- elastic_integration
- experiments
- factories
- finders
- frontend
- graphql
- haml_lint
- helpers
- initializers
- javascripts
- lib
- models
- policies
- presenters
- rack_servers
- replicators
- routing
- rubocop
- serializers
- services
- sidekiq
- spam
- support_specs
- tasks
- uploaders
- validators
- views
- workers
- tooling
- ],
- integration: %w[
- controllers
- mailers
- requests
- ],
- system: ['features']
- }.freeze
-
- attr_reader :prefix
-
- def initialize(prefix = nil)
- @prefix = prefix
- @patterns = {}
- @regexps = {}
- end
-
- def pattern(level)
- @patterns[level] ||= "#{prefix}spec/#{folders_pattern(level)}{,/**/}*#{suffix(level)}"
- end
-
- def regexp(level)
- @regexps[level] ||= Regexp.new("#{prefix}spec/#{folders_regex(level)}").freeze
- end
-
- def level_for(file_path)
- case file_path
- # Detect migration first since some background migration tests are under
- # spec/lib/gitlab/background_migration and tests under spec/lib are unit by default
- when regexp(:migration), regexp(:background_migration)
- :migration
- # Detect frontend fixture before matching other unit tests
- when regexp(:frontend_fixture)
- :frontend_fixture
- when regexp(:unit)
- :unit
- when regexp(:integration)
- :integration
- when regexp(:system)
- :system
- else
- raise UnknownTestLevelError, "Test level for #{file_path} couldn't be set. Please rename the file properly or change the test level detection regexes in #{__FILE__}."
- end
- end
-
- def background_migration?(file_path)
- !!(file_path =~ regexp(:background_migration))
- end
-
- private
-
- def suffix(level)
- case level
- when :frontend_fixture
- ".rb"
- else
- "_spec.rb"
- end
- end
-
- def migration_and_background_migration_folders
- TEST_LEVEL_FOLDERS.fetch(:migration) + TEST_LEVEL_FOLDERS.fetch(:background_migration)
- end
-
- def folders_pattern(level)
- case level
- when :migration
- "{#{migration_and_background_migration_folders.join(',')}}"
- # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
- when :all, :geo
- '**'
- else
- "{#{TEST_LEVEL_FOLDERS.fetch(level).join(',')}}"
- end
- end
-
- def folders_regex(level)
- case level
- when :migration
- "(#{migration_and_background_migration_folders.join('|')})"
- # Geo specs aren't in a specific folder, but they all have the :geo tag, so we must search for them globally
- when :all, :geo
- ''
- else
- "(#{TEST_LEVEL_FOLDERS.fetch(level).join('|')})"
- end
- end
- end
-end
diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb
index 8f18d6433e0..e0e9677fac7 100644
--- a/lib/rouge/formatters/html_gitlab.rb
+++ b/lib/rouge/formatters/html_gitlab.rb
@@ -7,10 +7,11 @@ module Rouge
# Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance.
#
- # [+tag+] The tag (language) of the lexer used to generate the formatted tokens
+ # [+tag+] The tag (language) of the lexer used to generate the formatted tokens
+ # [+line_number+] The line number used to populate line IDs
def initialize(options = {})
- @line_number = 1
@tag = options[:tag]
+ @line_number = options[:line_number] || 1
end
def stream(tokens)
diff --git a/lib/spam/concerns/has_spam_action_response_fields.rb b/lib/spam/concerns/has_spam_action_response_fields.rb
index d49f5cd6454..6688ae56cb0 100644
--- a/lib/spam/concerns/has_spam_action_response_fields.rb
+++ b/lib/spam/concerns/has_spam_action_response_fields.rb
@@ -23,15 +23,6 @@ module Spam
captcha_site_key: Gitlab::CurrentSettings.recaptcha_site_key
}
end
-
- # with_spam_action_response_fields(spammable) { {other_fields: true} } -> hash
- #
- # Takes a Spammable and a block as arguments.
- #
- # The block passed should be a hash, which the spam_action_fields will be merged into.
- def with_spam_action_response_fields(spammable)
- yield.merge(spam_action_response_fields(spammable))
- end
end
end
end
diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab
index 9eafa5ef008..74eb8634d58 100644
--- a/lib/support/nginx/gitlab
+++ b/lib/support/nginx/gitlab
@@ -7,8 +7,8 @@
## CONTRIBUTING ##
##################################
##
-## If you change this file in a Merge Request, please also create
-## a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+## If you change this file in a merge request, please also create
+## a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
##
###################################
## configuration ##
diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl
index ae5c88455e4..576c13d8d10 100644
--- a/lib/support/nginx/gitlab-ssl
+++ b/lib/support/nginx/gitlab-ssl
@@ -11,8 +11,8 @@
## CONTRIBUTING ##
##################################
##
-## If you change this file in a Merge Request, please also create
-## a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+## If you change this file in a merge request, please also create
+## a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
##
###################################
## configuration ##
diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb
index 0f4fbe4fba5..31456dc096b 100644
--- a/lib/system_check/app/git_version_check.rb
+++ b/lib/system_check/app/git_version_check.rb
@@ -7,7 +7,7 @@ module SystemCheck
set_check_pass -> { "yes (#{self.current_version})" }
def self.required_version
- @required_version ||= Gitlab::VersionInfo.parse('2.29.0')
+ @required_version ||= Gitlab::VersionInfo.parse('2.31.0')
end
def self.current_version
diff --git a/lib/system_check/app/redis_version_check.rb b/lib/system_check/app/redis_version_check.rb
index a205861b9a9..e72d8b6b04d 100644
--- a/lib/system_check/app/redis_version_check.rb
+++ b/lib/system_check/app/redis_version_check.rb
@@ -5,8 +5,10 @@ require 'redis'
module SystemCheck
module App
class RedisVersionCheck < SystemCheck::BaseCheck
+ # Redis 4.x will be deprecated
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/327197
MIN_REDIS_VERSION = '4.0.0'
- RECOMMENDED_REDIS_VERSION = '4.0.0' # In future we may deprecate but still support Redis 4
+ RECOMMENDED_REDIS_VERSION = '5.0.0'
set_name "Redis version >= #{RECOMMENDED_REDIS_VERSION}?"
@custom_error_message = ''
diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake
deleted file mode 100644
index 44d2071751f..00000000000
--- a/lib/tasks/brakeman.rake
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-desc 'Security check via brakeman'
-task :brakeman do
- # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge
- # requests are welcome!
- if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z))
- puts 'Security check succeed'
- else
- puts 'Security check failed'
- exit 1
- end
-end
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index 4d698e56444..13365b9ec07 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -3,7 +3,7 @@
namespace :cache do
namespace :clear do
REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000
- REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan
+ REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan
desc "GitLab | Cache | Clear redis cache"
task redis: :environment do
diff --git a/lib/tasks/db_obsolete_ignored_columns.rake b/lib/tasks/db_obsolete_ignored_columns.rake
index cf35a355ce9..a689a9bf2d8 100644
--- a/lib/tasks/db_obsolete_ignored_columns.rake
+++ b/lib/tasks/db_obsolete_ignored_columns.rake
@@ -20,7 +20,7 @@ task 'db:obsolete_ignored_columns' => :environment do
WARNING: Removing columns is tricky because running GitLab processes may still be using the columns.
- See also https://docs.gitlab.com/ee/development/what_requires_downtime.html#dropping-columns
+ See also https://docs.gitlab.com/ee/development/avoiding_downtime_in_migrations.html#dropping-columns
TEXT
end
end
diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake
deleted file mode 100644
index 3428e3f8f53..00000000000
--- a/lib/tasks/downtime_check.rake
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-desc 'Checks if migrations in a branch require downtime'
-task downtime_check: :environment do
- repo = if defined?(Gitlab::License)
- 'gitlab'
- else
- 'gitlab-foss'
- end
-
- `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1`
-
- Rake::Task['gitlab:db:downtime_check'].invoke('FETCH_HEAD')
-end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 541a4fc62af..3baf4e7b7c6 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -80,22 +80,6 @@ namespace :gitlab do
end
end
- desc 'GitLab | DB | Checks if migrations require downtime or not'
- task :downtime_check, [:ref] => :environment do |_, args|
- abort 'You must specify a Git reference to compare with' unless args[:ref]
-
- require 'shellwords'
-
- ref = Shellwords.escape(args[:ref])
-
- migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines
- .map { |file| Rails.root.join(file.strip).to_s }
- .select { |file| File.file?(file) }
- .select { |file| /\A[0-9]+.*\.rb\z/ =~ File.basename(file) }
-
- Gitlab::DowntimeCheck.new.check_and_print(migrations)
- end
-
desc 'GitLab | DB | Sets up EE specific database functionality'
if Gitlab.ee?
@@ -237,7 +221,8 @@ namespace :gitlab do
result_file = args[:result_file] || raise("Please specify result_file argument")
raise "File exists already, won't overwrite: #{result_file}" if File.exist?(result_file)
- verbose_was, ActiveRecord::Migration.verbose = ActiveRecord::Migration.verbose, true
+ verbose_was = ActiveRecord::Migration.verbose
+ ActiveRecord::Migration.verbose = true
ctx = ActiveRecord::Base.connection.migration_context
existing_versions = ctx.get_all_versions.to_set
diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake
new file mode 100644
index 00000000000..0c8e0755348
--- /dev/null
+++ b/lib/tasks/gitlab/docs/redirect.rake
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+require 'date'
+require 'pathname'
+
+# https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page
+namespace :gitlab do
+ namespace :docs do
+ desc 'GitLab | Docs | Create a doc redirect'
+ task :redirect, [:old_path, :new_path] do |_, args|
+ if args.old_path
+ old_path = args.old_path
+ else
+ puts '=> Enter the path of the OLD file:'
+ old_path = STDIN.gets.chomp
+ end
+
+ if args.new_path
+ new_path = args.new_path
+ else
+ puts '=> Enter the path of the NEW file:'
+ new_path = STDIN.gets.chomp
+ end
+
+ #
+ # If the new path is a relative URL, find the relative path between
+ # the old and new paths.
+ # The returned path is one level deeper, so remove the leading '../'.
+ #
+ unless new_path.start_with?('http')
+ old_pathname = Pathname.new(old_path)
+ new_pathname = Pathname.new(new_path)
+ relative_path = new_pathname.relative_path_from(old_pathname).to_s
+ (_, *last) = relative_path.split('/')
+ new_path = last.join('/')
+ end
+
+ #
+ # - If this is an external URL, move the date 1 year later.
+ # - If this is a relative URL, move the date 3 months later.
+ #
+ date = Time.now.utc.strftime('%Y-%m-%d')
+ date = new_path.start_with?('http') ? Date.parse(date) >> 12 : Date.parse(date) >> 3
+
+ puts "=> Creating new redirect from #{old_path} to #{new_path}"
+ File.open(old_path, 'w') do |post|
+ post.puts '---'
+ post.puts "redirect_to: '#{new_path}'"
+ post.puts '---'
+ post.puts
+ post.puts "This file was moved to [another location](#{new_path})."
+ post.puts
+ post.puts "<!-- This redirect file can be deleted after <#{date}>. -->"
+ post.puts "<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page -->"
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index 9e474b00ba7..df75b3cf716 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -17,24 +17,29 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
- command = []
- _, status = Gitlab::Popen.popen(%w[which gmake])
- command << (status == 0 ? 'gmake' : 'make')
-
- if Rails.env.test?
- command.push(
- 'BUNDLE_FLAGS=--no-deployment',
- "GEM_HOME=#{Bundler.bundle_path}")
- end
-
storage_paths = { 'default' => args.storage_path }
Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths)
+
+ # In CI we run scripts/gitaly-test-build
+ next if ENV['CI'].present?
+
Dir.chdir(args.dir) do
- # In CI we run scripts/gitaly-test-build instead of this command
- unless ENV['CI'].present?
- Bundler.with_original_env { Gitlab::Popen.popen(command, nil, { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }) }
+ Bundler.with_original_env do
+ env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }
+
+ if Rails.env.test?
+ env["GEM_HOME"] = Bundler.bundle_path.to_s
+ env["BUNDLE_DEPLOYMENT"] = 'false'
+ end
+
+ Gitlab::Popen.popen([make_cmd], nil, env)
end
end
end
+
+ def make_cmd
+ _, status = Gitlab::Popen.popen(%w[which gmake])
+ status == 0 ? 'gmake' : 'make'
+ end
end
end
diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake
index 77377a7e0fd..27bba6aa307 100644
--- a/lib/tasks/gitlab/graphql.rake
+++ b/lib/tasks/gitlab/graphql.rake
@@ -110,7 +110,7 @@ namespace :gitlab do
desc 'GitLab | GraphQL | Generate GraphQL docs'
task compile_docs: [:environment, :enable_feature_flags] do
- renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options)
+ renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema, render_options)
renderer.write
@@ -119,7 +119,7 @@ namespace :gitlab do
desc 'GitLab | GraphQL | Check if GraphQL docs are up to date'
task check_docs: [:environment, :enable_feature_flags] do
- renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options)
+ renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema, render_options)
doc = File.read(Rails.root.join(OUTPUT_DIR, 'index.md'))
diff --git a/lib/tasks/gitlab/pages.rake b/lib/tasks/gitlab/pages.rake
index b598dab901d..ee2931f0c4f 100644
--- a/lib/tasks/gitlab/pages.rake
+++ b/lib/tasks/gitlab/pages.rake
@@ -9,9 +9,9 @@ namespace :gitlab do
logger.info('Starting to migrate legacy pages storage to zip deployments')
result = ::Pages::MigrateFromLegacyStorageService.new(logger,
- migration_threads: migration_threads,
- batch_size: batch_size,
- ignore_invalid_entries: ignore_invalid_entries).execute
+ ignore_invalid_entries: ignore_invalid_entries,
+ mark_projects_as_not_deployed: mark_projects_as_not_deployed)
+ .execute_with_threads(threads: migration_threads, batch_size: batch_size)
logger.info("A total of #{result[:migrated] + result[:errored]} projects were processed.")
logger.info("- The #{result[:migrated]} projects migrated successfully")
@@ -51,5 +51,39 @@ namespace :gitlab do
ENV.fetch('PAGES_MIGRATION_IGNORE_INVALID_ENTRIES', 'false')
)
end
+
+ def mark_projects_as_not_deployed
+ Gitlab::Utils.to_boolean(
+ ENV.fetch('PAGES_MIGRATION_MARK_PROJECTS_AS_NOT_DEPLOYED', 'false')
+ )
+ end
+
+ namespace :deployments do
+ task migrate_to_object_storage: :gitlab_environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Starting transfer of pages deployments to remote storage')
+
+ helper = Gitlab::Pages::MigrationHelper.new(logger)
+
+ begin
+ helper.migrate_to_remote_storage
+ rescue => e
+ logger.error(e.message)
+ end
+ end
+
+ task migrate_to_local: :gitlab_environment do
+ logger = Logger.new(STDOUT)
+ logger.info('Starting transfer of Pages deployments to local storage')
+
+ helper = Gitlab::Pages::MigrationHelper.new(logger)
+
+ begin
+ helper.migrate_to_local_storage
+ rescue => e
+ logger.error(e.message)
+ end
+ end
+ end
end
end
diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake
deleted file mode 100644
index a83ba69bc75..00000000000
--- a/lib/tasks/gitlab/test.rake
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-namespace :gitlab do
- desc "GitLab | Run all tests"
- task :test do
- cmds = [
- %w(rake brakeman),
- %w(rake rubocop),
- %w(rake spec),
- %w(rake karma)
- ]
-
- cmds.each do |cmd|
- system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!")
- end
- end
-end
diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake
index bf18332a8eb..5eed5d4dce4 100644
--- a/lib/tasks/spec.rake
+++ b/lib/tasks/spec.rake
@@ -2,30 +2,44 @@
return if Rails.env.production?
+require_relative '../../tooling/merge_request_rspec_failure_rake_task'
+
namespace :spec do
desc 'GitLab | RSpec | Run unit tests'
RSpec::Core::RakeTask.new(:unit, :rspec_opts) do |t, args|
- require_dependency 'quality/test_level'
+ require_test_level
t.pattern = Quality::TestLevel.new.pattern(:unit)
t.rspec_opts = args[:rspec_opts]
end
desc 'GitLab | RSpec | Run integration tests'
RSpec::Core::RakeTask.new(:integration, :rspec_opts) do |t, args|
- require_dependency 'quality/test_level'
+ require_test_level
t.pattern = Quality::TestLevel.new.pattern(:integration)
t.rspec_opts = args[:rspec_opts]
end
desc 'GitLab | RSpec | Run system tests'
RSpec::Core::RakeTask.new(:system, :rspec_opts) do |t, args|
- require_dependency 'quality/test_level'
+ require_test_level
t.pattern = Quality::TestLevel.new.pattern(:system)
t.rspec_opts = args[:rspec_opts]
end
+ desc 'GitLab | RSpec | Run merge request RSpec failures'
+ Tooling::MergeRequestRspecFailureRakeTask.new(:merge_request_rspec_failure, :rspec_opts) do |t, args|
+ t.pattern = t.rspec_failures_on_merge_request
+ t.rspec_opts = args[:rspec_opts]
+ end
+
desc 'Run the code examples in spec/requests/api'
RSpec::Core::RakeTask.new(:api) do |t|
t.pattern = 'spec/requests/api/**/*_spec.rb'
end
+
+ private
+
+ def require_test_level
+ require_relative '../../tooling/quality/test_level'
+ end
end
diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake
index b24817468c6..c4eb9450b31 100644
--- a/lib/tasks/test.rake
+++ b/lib/tasks/test.rake
@@ -2,7 +2,16 @@
Rake::Task["test"].clear
-desc "GitLab | Run all tests"
+desc "GitLab | List rake tasks for tests"
task :test do
- Rake::Task["gitlab:test"].invoke
+ puts "Running the full GitLab test suite takes significant time to pass. We recommend using one of the following spec tasks:\n\n"
+
+ spec_tasks = Rake::Task.tasks.select { |t| t.name.start_with?('spec:') }
+ longest_task_name = spec_tasks.map { |t| t.name.size }.max
+
+ spec_tasks.each do |task|
+ puts "#{"%-#{longest_task_name}s" % task.name} | #{task.full_comment}"
+ end
+
+ puts "\nLearn more at https://docs.gitlab.com/ee/development/rake_tasks.html#run-tests."
end