From b76ae638462ab0f673e5915986070518dd3f9ad3 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 19 Aug 2021 09:08:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@14-2-stable-ee --- lib/after_commit_queue.rb | 2 +- lib/api/api.rb | 9 +- lib/api/appearance.rb | 2 + lib/api/badges.rb | 2 +- lib/api/bulk_imports.rb | 53 +- lib/api/ci/helpers/runner.rb | 123 ++++ lib/api/ci/job_artifacts.rb | 143 ++++ lib/api/ci/jobs.rb | 206 ++++++ lib/api/ci/pipelines.rb | 27 +- lib/api/ci/runner.rb | 22 +- lib/api/ci/triggers.rb | 148 +++++ lib/api/ci/variables.rb | 126 ++++ lib/api/commits.rb | 2 +- .../packages/debian_distribution_endpoints.rb | 4 + .../concerns/packages/debian_package_endpoints.rb | 165 +++-- lib/api/debian_group_packages.rb | 47 +- lib/api/debian_project_packages.rb | 50 +- lib/api/entities/ci/job_request/dependency.rb | 2 +- lib/api/entities/ci/pipeline_basic.rb | 2 + lib/api/entities/error_tracking.rb | 1 + lib/api/entities/issue_basic.rb | 2 +- lib/api/entities/project.rb | 1 + lib/api/entities/project_with_access.rb | 6 +- lib/api/environments.rb | 6 +- lib/api/error_tracking.rb | 5 + lib/api/error_tracking_collector.rb | 26 +- lib/api/group_debian_distributions.rb | 35 + lib/api/group_variables.rb | 2 +- lib/api/groups.rb | 10 +- lib/api/helpers.rb | 4 + lib/api/helpers/groups_helpers.rb | 2 +- lib/api/helpers/members_helpers.rb | 8 + .../helpers/packages/dependency_proxy_helpers.rb | 23 +- lib/api/helpers/packages/npm.rb | 14 +- lib/api/helpers/projects_helpers.rb | 11 +- lib/api/helpers/runner.rb | 121 ---- lib/api/internal/base.rb | 4 +- lib/api/invitations.rb | 11 +- lib/api/issues.rb | 4 +- lib/api/job_artifacts.rb | 141 ---- lib/api/jobs.rb | 204 ------ lib/api/members.rb | 8 +- lib/api/merge_requests.rb | 1 + lib/api/namespaces.rb | 5 +- lib/api/project_debian_distributions.rb | 2 - lib/api/project_templates.rb | 2 +- lib/api/projects.rb | 48 +- lib/api/pypi_packages.rb | 43 +- lib/api/repositories.rb | 12 +- lib/api/rubygem_packages.rb | 2 +- lib/api/settings.rb | 2 +- lib/api/tags.rb | 2 - lib/api/templates.rb | 19 +- lib/api/time_tracking_endpoints.rb | 1 + lib/api/triggers.rb | 146 ----- lib/api/user_counts.rb | 6 +- lib/api/v3/github.rb | 2 + lib/api/variables.rb | 124 ---- lib/atlassian/jira_connect/client.rb | 2 +- lib/backup.rb | 39 ++ lib/backup/gitaly_backup.rb | 19 +- lib/backup/manager.rb | 13 +- lib/banzai/filter/references/reference_cache.rb | 57 +- lib/banzai/filter/table_of_contents_tag_filter.rb | 44 +- lib/error_tracking/collector/sentry_auth_parser.rb | 25 + lib/extracts_path.rb | 10 - lib/feature.rb | 6 +- lib/feature/gitaly.rb | 2 +- lib/gem_extensions/active_record/association.rb | 37 ++ .../active_record/associations/builder/has_many.rb | 21 + .../active_record/associations/builder/has_one.rb | 21 + .../associations/has_many_through_association.rb | 18 + .../associations/has_one_through_association.rb | 17 + .../associations/preloader/through_association.rb | 22 + .../active_record/configurable_disable_joins.rb | 17 + lib/gem_extensions/active_record/delegate_cache.rb | 34 + .../associations/association_scope.rb | 78 +++ .../active_record/disable_joins/relation.rb | 43 ++ .../database_instrumentation_class.rb.template | 17 + .../generic_instrumentation_class.rb.template | 15 + .../templates/instrumentation_class.rb.template | 14 - .../gitlab/usage_metric/usage_metric_generator.rb | 71 -- lib/generators/gitlab/usage_metric_generator.rb | 81 +++ .../post_deployment_migration_generator.rb | 17 + .../post_deployment_migration_generator.rb | 17 - lib/gitlab.rb | 1 - .../analytics/cycle_analytics/records_fetcher.rb | 59 +- .../analytics/cycle_analytics/request_params.rb | 186 ++++++ .../cycle_analytics/stage_query_helpers.rb | 2 +- lib/gitlab/auth.rb | 20 +- lib/gitlab/auth/auth_finders.rb | 2 +- .../auth/otp/strategies/forti_token_cloud.rb | 2 +- lib/gitlab/auth/result.rb | 36 +- lib/gitlab/background_migration.rb | 21 +- .../backfill_draft_status_on_merge_requests.rb | 11 + .../backfill_integrations_type_new.rb | 86 +++ .../backfill_project_repositories.rb | 2 +- .../backfill_snippet_repositories.rb | 2 +- .../copy_ci_builds_columns_to_security_scans.rb | 44 ++ .../create_security_setting.rb | 14 + .../migrate_fingerprint_sha256_within_keys.rb | 2 +- .../migrate_issue_trackers_sensitive_data.rb | 2 +- .../populate_issue_email_participants.rb | 2 +- ...recalculate_vulnerabilities_occurrences_uuid.rb | 5 + lib/gitlab/bare_repository_import/importer.rb | 2 +- lib/gitlab/bitbucket_server_import/importer.rb | 3 +- lib/gitlab/cache/import/caching.rb | 20 +- lib/gitlab/chaos.rb | 2 +- lib/gitlab/chat/command.rb | 4 +- lib/gitlab/checks/branch_check.rb | 2 +- lib/gitlab/checks/changes_access.rb | 52 +- lib/gitlab/checks/single_change_access.rb | 3 +- lib/gitlab/ci/ansi2html.rb | 3 + lib/gitlab/ci/ansi2json/line.rb | 9 +- lib/gitlab/ci/config.rb | 36 +- lib/gitlab/ci/config/entry/include.rb | 18 +- lib/gitlab/ci/config/entry/include/rules.rb | 28 + lib/gitlab/ci/config/entry/include/rules/rule.rb | 30 + lib/gitlab/ci/config/entry/inherit/variables.rb | 11 - lib/gitlab/ci/config/entry/job.rb | 3 - lib/gitlab/ci/config/entry/processable.rb | 21 +- lib/gitlab/ci/config/entry/rules.rb | 2 +- lib/gitlab/ci/config/external/file/remote.rb | 2 +- lib/gitlab/ci/config/external/mapper.rb | 12 +- lib/gitlab/ci/config/external/rules.rb | 31 + lib/gitlab/ci/config/normalizer/matrix_strategy.rb | 1 - lib/gitlab/ci/features.rb | 2 +- lib/gitlab/ci/limit.rb | 11 +- lib/gitlab/ci/lint.rb | 1 + lib/gitlab/ci/model.rb | 15 - lib/gitlab/ci/parsers.rb | 4 +- lib/gitlab/ci/parsers/security/common.rb | 266 ++++++++ .../parsers/security/concerns/deprecated_syntax.rb | 36 + lib/gitlab/ci/parsers/security/sast.rb | 26 + lib/gitlab/ci/parsers/security/secret_detection.rb | 27 + .../security/validators/schema_validator.rb | 68 ++ .../parsers/security/validators/schemas/sast.json | 706 ++++++++++++++++++++ .../validators/schemas/secret_detection.json | 729 +++++++++++++++++++++ lib/gitlab/ci/pipeline/chain/command.rb | 9 +- lib/gitlab/ci/pipeline/chain/config/process.rb | 2 +- lib/gitlab/ci/pipeline/chain/sequence.rb | 1 + lib/gitlab/ci/pipeline/chain/skip.rb | 8 +- .../ci/pipeline/expression/lexeme/pattern.rb | 2 +- lib/gitlab/ci/pipeline/metrics.rb | 11 +- lib/gitlab/ci/pipeline/seed/build.rb | 4 +- .../ci/reports/security/aggregated_report.rb | 24 + lib/gitlab/ci/reports/security/finding.rb | 150 +++++ lib/gitlab/ci/reports/security/finding_key.rb | 36 + .../ci/reports/security/finding_signature.rb | 46 ++ lib/gitlab/ci/reports/security/locations/base.rb | 41 ++ lib/gitlab/ci/reports/security/locations/sast.rb | 33 + .../reports/security/locations/secret_detection.rb | 33 + lib/gitlab/ci/reports/security/report.rb | 76 +++ lib/gitlab/ci/reports/security/reports.rb | 42 ++ .../security/vulnerability_reports_comparer.rb | 163 +++++ .../5-Minute-Production-App.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Bash.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Django.gitlab-ci.yml | 2 +- .../ci/templates/Getting-Started.gitlab-ci.yml | 2 +- .../ci/templates/Jobs/Code-Quality.gitlab-ci.yml | 2 +- .../Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml | 1 + .../templates/Jobs/Secret-Detection.gitlab-ci.yml | 4 +- lib/gitlab/ci/templates/Laravel.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml | 2 +- lib/gitlab/ci/templates/Ruby.gitlab-ci.yml | 3 +- .../Security/Cluster-Image-Scanning.gitlab-ci.yml | 2 +- .../Security/DAST-Runner-Validation.gitlab-ci.yml | 23 + .../Security/Secure-Binaries.gitlab-ci.yml | 12 +- lib/gitlab/ci/templates/Terraform.gitlab-ci.yml | 2 +- .../ci/templates/Terraform.latest.gitlab-ci.yml | 15 +- .../ci/templates/Terraform/Base.gitlab-ci.yml | 64 ++ .../templates/Terraform/Base.latest.gitlab-ci.yml | 16 +- lib/gitlab/ci/yaml_processor/dag.rb | 2 +- lib/gitlab/ci/yaml_processor/result.rb | 2 +- lib/gitlab/config/entry/validators.rb | 18 +- .../config_checker/external_database_checker.rb | 4 +- lib/gitlab/conflict/file.rb | 22 +- lib/gitlab/conflict/file_collection.rb | 4 +- .../content_security_policy/config_loader.rb | 83 +-- lib/gitlab/current_settings.rb | 2 +- lib/gitlab/data_builder/deployment.rb | 1 + lib/gitlab/data_builder/pipeline.rb | 51 +- lib/gitlab/database.rb | 337 +++------- lib/gitlab/database/as_with_materialized.rb | 2 +- lib/gitlab/database/async_indexes.rb | 15 + lib/gitlab/database/async_indexes/index_creator.rb | 63 ++ .../database/async_indexes/migration_helpers.rb | 80 +++ .../database/async_indexes/postgres_async_index.rb | 22 + lib/gitlab/database/batch_counter.rb | 2 +- lib/gitlab/database/connection.rb | 249 +++++++ .../database/count/reltuples_count_strategy.rb | 2 +- .../database/count/tablesample_count_strategy.rb | 2 +- lib/gitlab/database/grant.rb | 2 +- lib/gitlab/database/load_balancing.rb | 35 +- .../database/load_balancing/active_record_proxy.rb | 2 +- .../database/load_balancing/connection_proxy.rb | 34 +- lib/gitlab/database/load_balancing/host.rb | 20 +- lib/gitlab/database/load_balancing/host_list.rb | 17 +- .../database/load_balancing/load_balancer.rb | 142 ++-- .../database/load_balancing/rack_middleware.rb | 12 +- .../database/load_balancing/service_discovery.rb | 53 +- lib/gitlab/database/load_balancing/sticking.rb | 12 +- lib/gitlab/database/metrics.rb | 26 + lib/gitlab/database/migration_helpers.rb | 37 +- .../migrations/background_migration_helpers.rb | 34 + lib/gitlab/database/migrations/instrumentation.rb | 20 +- lib/gitlab/database/migrations/observation.rb | 3 +- lib/gitlab/database/migrations/observers.rb | 8 +- .../migrations/observers/migration_observer.rb | 7 +- .../database/migrations/observers/query_details.rb | 8 +- .../database/migrations/observers/query_log.rb | 8 +- .../migrations/observers/query_statistics.rb | 2 +- .../observers/total_database_size_change.rb | 2 +- lib/gitlab/database/multi_threaded_migration.rb | 52 -- .../partitioning/detached_partition_dropper.rb | 56 ++ .../database/partitioning/monthly_strategy.rb | 2 +- .../database/partitioning/partition_manager.rb | 34 +- .../database/partitioning/partition_monitoring.rb | 5 + lib/gitlab/database/partitioning/time_partition.rb | 7 + lib/gitlab/database/postgres_foreign_key.rb | 15 + .../postgres_hll/batch_distinct_counter.rb | 2 +- lib/gitlab/database/postgres_index.rb | 7 +- lib/gitlab/database/postgres_partition.rb | 8 +- lib/gitlab/database/reindexing.rb | 27 +- .../database/reindexing/reindex_concurrently.rb | 7 - lib/gitlab/database/schema_migrations/context.rb | 13 +- lib/gitlab/database/similarity_score.rb | 4 +- lib/gitlab/database/transaction/context.rb | 125 ++++ lib/gitlab/database/transaction/observer.rb | 66 ++ lib/gitlab/deprecation_json_logger.rb | 9 + lib/gitlab/diff/file_collection/base.rb | 6 + lib/gitlab/email/handler/create_note_handler.rb | 11 +- lib/gitlab/email/handler/reply_processing.rb | 31 +- .../message/in_product_marketing/admin_verify.rb | 43 ++ .../email/message/in_product_marketing/base.rb | 8 +- .../email/message/in_product_marketing/create.rb | 2 +- .../email/message/in_product_marketing/team.rb | 4 + .../message/in_product_marketing/team_short.rb | 47 ++ .../email/message/in_product_marketing/trial.rb | 4 + .../message/in_product_marketing/trial_short.rb | 47 ++ .../email/message/in_product_marketing/verify.rb | 2 +- lib/gitlab/email/reply_parser.rb | 14 +- lib/gitlab/email/smtp_config.rb | 29 + lib/gitlab/encoding_helper.rb | 9 + lib/gitlab/encrypted_command_base.rb | 105 +++ lib/gitlab/encrypted_ldap_command.rb | 92 +-- lib/gitlab/encrypted_smtp_command.rb | 23 + lib/gitlab/etag_caching/router/restful.rb | 2 +- lib/gitlab/experimentation.rb | 18 +- lib/gitlab/fake_application_settings.rb | 52 +- lib/gitlab/form_builders/gitlab_ui_form_builder.rb | 55 ++ lib/gitlab/git/blob.rb | 4 +- lib/gitlab/git/commit.rb | 12 +- lib/gitlab/git/commit_stats.rb | 24 +- lib/gitlab/git/conflict/file.rb | 12 +- lib/gitlab/git/conflict/resolver.rb | 5 +- lib/gitlab/git/remote_mirror.rb | 6 +- lib/gitlab/git/repository.rb | 53 +- lib/gitlab/git/rugged_impl/tree.rb | 7 +- lib/gitlab/git/tag.rb | 16 +- lib/gitlab/git/tree.rb | 8 +- lib/gitlab/git_access.rb | 17 +- lib/gitlab/gitaly_client/commit_service.rb | 43 +- .../gitaly_client/conflict_files_stitcher.rb | 1 + lib/gitlab/gitaly_client/conflicts_service.rb | 5 +- lib/gitlab/gitaly_client/ref_service.rb | 39 +- lib/gitlab/gitaly_client/remote_service.rb | 29 +- lib/gitlab/gitaly_client/repository_service.rb | 48 +- lib/gitlab/github_import/bulk_importing.rb | 43 +- .../github_import/importer/diff_note_importer.rb | 2 +- .../github_import/importer/issue_importer.rb | 2 +- .../github_import/importer/label_links_importer.rb | 2 +- .../github_import/importer/labels_importer.rb | 14 +- .../github_import/importer/lfs_objects_importer.rb | 6 +- .../github_import/importer/milestones_importer.rb | 14 +- lib/gitlab/github_import/importer/note_importer.rb | 2 +- .../importer/pull_requests_importer.rb | 6 +- .../importer/pull_requests_reviews_importer.rb | 43 +- .../github_import/importer/releases_importer.rb | 14 +- .../github_import/importer/repository_importer.rb | 12 +- lib/gitlab/github_import/logger.rb | 11 + lib/gitlab/github_import/object_counter.rb | 31 +- lib/gitlab/github_import/parallel_scheduling.rb | 35 +- lib/gitlab/github_import/user_finder.rb | 14 +- lib/gitlab/graphql/copy_field_description.rb | 2 +- lib/gitlab/graphql/markdown_field.rb | 2 +- lib/gitlab/highlight.rb | 6 +- lib/gitlab/http.rb | 19 +- lib/gitlab/i18n.rb | 28 +- lib/gitlab/import/database_helpers.rb | 2 +- lib/gitlab/import/import_failure_service.rb | 76 +++ lib/gitlab/import/logger.rb | 4 + lib/gitlab/import_export/json/legacy_reader.rb | 2 +- lib/gitlab/import_export/json/ndjson_reader.rb | 4 +- .../import_export/json/streaming_serializer.rb | 15 +- lib/gitlab/import_export/lfs_restorer.rb | 2 +- lib/gitlab/import_export/project/import_export.yml | 3 + lib/gitlab/import_export/project/object_builder.rb | 2 +- lib/gitlab/instrumentation/redis_interceptor.rb | 20 + lib/gitlab/instrumentation_helper.rb | 5 + lib/gitlab/integrations/sti_type.rb | 12 +- lib/gitlab/jira/http_client.rb | 8 + lib/gitlab/jira_import/issue_serializer.rb | 5 +- lib/gitlab/json_cache.rb | 4 +- lib/gitlab/json_logger.rb | 8 +- lib/gitlab/kas.rb | 7 + lib/gitlab/kubernetes/default_namespace.rb | 17 +- lib/gitlab/kubernetes/kubeconfig/entry/cluster.rb | 43 ++ lib/gitlab/kubernetes/kubeconfig/entry/context.rb | 39 ++ lib/gitlab/kubernetes/kubeconfig/entry/user.rb | 29 + lib/gitlab/kubernetes/kubeconfig/template.rb | 59 ++ lib/gitlab/language_detection.rb | 2 +- lib/gitlab/markdown_cache.rb | 10 +- .../markdown_cache/active_record/extension.rb | 5 +- lib/gitlab/metrics/requests_rack_middleware.rb | 16 +- lib/gitlab/metrics/samplers/base_sampler.rb | 2 +- lib/gitlab/metrics/subscribers/action_cable.rb | 2 +- lib/gitlab/metrics/subscribers/action_view.rb | 2 +- lib/gitlab/metrics/subscribers/active_record.rb | 130 +++- lib/gitlab/metrics/subscribers/rails_cache.rb | 2 +- lib/gitlab/middleware/go.rb | 16 +- lib/gitlab/middleware/multipart.rb | 2 +- lib/gitlab/optimistic_locking.rb | 2 +- lib/gitlab/otp_key_rotator.rb | 4 +- .../pagination/keyset/column_condition_builder.rb | 206 ++++++ lib/gitlab/pagination/keyset/order.rb | 48 +- .../pagination/keyset/simple_order_builder.rb | 1 + lib/gitlab/profiler.rb | 6 +- lib/gitlab/project_search_results.rb | 5 +- .../query_limiting/active_support_subscriber.rb | 6 +- lib/gitlab/query_limiting/middleware.rb | 2 +- lib/gitlab/quick_actions/issuable_actions.rb | 41 ++ lib/gitlab/reactive_cache_set_cache.rb | 5 - lib/gitlab/redis/wrapper.rb | 4 + lib/gitlab/regex.rb | 10 +- lib/gitlab/repository_set_cache.rb | 5 - lib/gitlab/search_results.rb | 2 +- lib/gitlab/set_cache.rb | 6 - lib/gitlab/setup_helper.rb | 8 +- lib/gitlab/sidekiq_cluster/cli.rb | 16 + lib/gitlab/sidekiq_config/dummy_worker.rb | 5 +- lib/gitlab/sidekiq_config/worker.rb | 14 +- lib/gitlab/signed_tag.rb | 47 ++ lib/gitlab/slash_commands/presenters/help.rb | 2 +- lib/gitlab/sql/glob.rb | 2 +- lib/gitlab/sql/set_operator.rb | 31 +- lib/gitlab/tracking/docs/helper.rb | 2 +- lib/gitlab/usage/docs/helper.rb | 64 -- lib/gitlab/usage/docs/renderer.rb | 32 - lib/gitlab/usage/docs/templates/default.md.haml | 48 -- lib/gitlab/usage/docs/value_formatter.rb | 28 - lib/gitlab/usage/metric.rb | 49 +- lib/gitlab/usage/metric_definition.rb | 22 +- lib/gitlab/usage/metrics/aggregates.rb | 26 + lib/gitlab/usage/metrics/aggregates/aggregate.rb | 17 - lib/gitlab/usage/metrics/aggregates/sources.rb | 13 + .../usage/metrics/aggregates/sources/redis_hll.rb | 2 - .../usage/metrics/instrumentations/base_metric.rb | 4 + .../collected_data_categories_metric.rb | 4 +- .../metrics/instrumentations/database_metric.rb | 12 +- .../metrics/instrumentations/generic_metric.rb | 22 +- .../metrics/instrumentations/redis_hll_metric.rb | 9 +- .../usage/metrics/instrumentations/redis_metric.rb | 49 ++ .../usage/metrics/names_suggestions/generator.rb | 6 + lib/gitlab/usage_data.rb | 27 +- lib/gitlab/usage_data_counters.rb | 3 +- lib/gitlab/usage_data_counters/diffs_counter.rb | 10 + .../usage_data_counters/hll_redis_counter.rb | 7 +- .../known_events/code_review_events.yml | 5 + .../usage_data_counters/known_events/common.yml | 1 - .../known_events/quickactions.yml | 4 + lib/gitlab/usage_data_counters/redis_counter.rb | 4 +- lib/gitlab/usage_data_metrics.rb | 21 +- lib/gitlab/usage_data_non_sql_metrics.rb | 16 + lib/gitlab/usage_data_queries.rb | 16 + lib/gitlab/utils.rb | 2 +- lib/gitlab/utils/usage_data.rb | 6 + lib/gitlab/visibility_level.rb | 1 + lib/gitlab/web_ide/config/entry/terminal.rb | 1 - lib/gitlab/x509/tag.rb | 33 +- lib/peek/views/active_record.rb | 13 +- lib/product_analytics/tracker.rb | 2 +- lib/sidebars/concerns/has_partial.rb | 21 + lib/sidebars/concerns/has_pill.rb | 13 + lib/sidebars/groups/menus/ci_cd_menu.rb | 51 ++ .../groups/menus/group_information_menu.rb | 79 +++ lib/sidebars/groups/menus/issues_menu.rb | 101 +++ lib/sidebars/groups/menus/kubernetes_menu.rb | 41 ++ lib/sidebars/groups/menus/merge_requests_menu.rb | 58 ++ .../groups/menus/packages_registries_menu.rb | 74 +++ lib/sidebars/groups/menus/settings_menu.rb | 117 ++++ lib/sidebars/groups/panel.rb | 10 + lib/sidebars/menu.rb | 3 +- .../projects/menus/packages_registries_menu.rb | 8 +- lib/support/init.d/gitlab | 2 +- lib/support/init.d/gitlab.default.example | 2 +- lib/support/nginx/gitlab-pages-ssl | 15 +- lib/support/nginx/gitlab-ssl | 18 +- lib/support/nginx/registry-ssl | 21 +- lib/tasks/gitlab/backup.rake | 4 +- lib/tasks/gitlab/db.rake | 10 +- lib/tasks/gitlab/docs/redirect.rake | 63 -- lib/tasks/gitlab/gitaly.rake | 55 ++ lib/tasks/gitlab/graphql.rake | 4 +- lib/tasks/gitlab/info.rake | 4 +- lib/tasks/gitlab/product_intelligence.rake | 24 + lib/tasks/gitlab/smtp.rake | 23 + lib/tasks/gitlab/storage.rake | 2 +- lib/tasks/gitlab/usage_data.rake | 6 - 412 files changed, 9101 insertions(+), 2955 deletions(-) create mode 100644 lib/api/ci/helpers/runner.rb create mode 100644 lib/api/ci/job_artifacts.rb create mode 100644 lib/api/ci/jobs.rb create mode 100644 lib/api/ci/triggers.rb create mode 100644 lib/api/ci/variables.rb create mode 100644 lib/api/group_debian_distributions.rb delete mode 100644 lib/api/helpers/runner.rb delete mode 100644 lib/api/job_artifacts.rb delete mode 100644 lib/api/jobs.rb delete mode 100644 lib/api/triggers.rb delete mode 100644 lib/api/variables.rb create mode 100644 lib/error_tracking/collector/sentry_auth_parser.rb create mode 100644 lib/gem_extensions/active_record/association.rb create mode 100644 lib/gem_extensions/active_record/associations/builder/has_many.rb create mode 100644 lib/gem_extensions/active_record/associations/builder/has_one.rb create mode 100644 lib/gem_extensions/active_record/associations/has_many_through_association.rb create mode 100644 lib/gem_extensions/active_record/associations/has_one_through_association.rb create mode 100644 lib/gem_extensions/active_record/associations/preloader/through_association.rb create mode 100644 lib/gem_extensions/active_record/configurable_disable_joins.rb create mode 100644 lib/gem_extensions/active_record/delegate_cache.rb create mode 100644 lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb create mode 100644 lib/gem_extensions/active_record/disable_joins/relation.rb create mode 100644 lib/generators/gitlab/usage_metric/templates/database_instrumentation_class.rb.template create mode 100644 lib/generators/gitlab/usage_metric/templates/generic_instrumentation_class.rb.template delete mode 100644 lib/generators/gitlab/usage_metric/templates/instrumentation_class.rb.template delete mode 100644 lib/generators/gitlab/usage_metric/usage_metric_generator.rb create mode 100644 lib/generators/gitlab/usage_metric_generator.rb create mode 100644 lib/generators/post_deployment_migration/post_deployment_migration_generator.rb delete mode 100644 lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb create mode 100644 lib/gitlab/analytics/cycle_analytics/request_params.rb create mode 100644 lib/gitlab/background_migration/backfill_integrations_type_new.rb create mode 100644 lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb create mode 100644 lib/gitlab/background_migration/create_security_setting.rb create mode 100644 lib/gitlab/ci/config/entry/include/rules.rb create mode 100644 lib/gitlab/ci/config/entry/include/rules/rule.rb create mode 100644 lib/gitlab/ci/config/external/rules.rb delete mode 100644 lib/gitlab/ci/model.rb create mode 100644 lib/gitlab/ci/parsers/security/common.rb create mode 100644 lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb create mode 100644 lib/gitlab/ci/parsers/security/sast.rb create mode 100644 lib/gitlab/ci/parsers/security/secret_detection.rb create mode 100644 lib/gitlab/ci/parsers/security/validators/schema_validator.rb create mode 100644 lib/gitlab/ci/parsers/security/validators/schemas/sast.json create mode 100644 lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json create mode 100644 lib/gitlab/ci/reports/security/aggregated_report.rb create mode 100644 lib/gitlab/ci/reports/security/finding.rb create mode 100644 lib/gitlab/ci/reports/security/finding_key.rb create mode 100644 lib/gitlab/ci/reports/security/finding_signature.rb create mode 100644 lib/gitlab/ci/reports/security/locations/base.rb create mode 100644 lib/gitlab/ci/reports/security/locations/sast.rb create mode 100644 lib/gitlab/ci/reports/security/locations/secret_detection.rb create mode 100644 lib/gitlab/ci/reports/security/report.rb create mode 100644 lib/gitlab/ci/reports/security/reports.rb create mode 100644 lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb create mode 100644 lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml create mode 100644 lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml create mode 100644 lib/gitlab/database/async_indexes.rb create mode 100644 lib/gitlab/database/async_indexes/index_creator.rb create mode 100644 lib/gitlab/database/async_indexes/migration_helpers.rb create mode 100644 lib/gitlab/database/async_indexes/postgres_async_index.rb create mode 100644 lib/gitlab/database/connection.rb create mode 100644 lib/gitlab/database/metrics.rb delete mode 100644 lib/gitlab/database/multi_threaded_migration.rb create mode 100644 lib/gitlab/database/partitioning/detached_partition_dropper.rb create mode 100644 lib/gitlab/database/postgres_foreign_key.rb create mode 100644 lib/gitlab/database/transaction/context.rb create mode 100644 lib/gitlab/database/transaction/observer.rb create mode 100644 lib/gitlab/deprecation_json_logger.rb create mode 100644 lib/gitlab/email/message/in_product_marketing/admin_verify.rb create mode 100644 lib/gitlab/email/message/in_product_marketing/team_short.rb create mode 100644 lib/gitlab/email/message/in_product_marketing/trial_short.rb create mode 100644 lib/gitlab/email/smtp_config.rb create mode 100644 lib/gitlab/encrypted_command_base.rb create mode 100644 lib/gitlab/encrypted_smtp_command.rb create mode 100644 lib/gitlab/form_builders/gitlab_ui_form_builder.rb create mode 100644 lib/gitlab/github_import/logger.rb create mode 100644 lib/gitlab/import/import_failure_service.rb create mode 100644 lib/gitlab/kubernetes/kubeconfig/entry/cluster.rb create mode 100644 lib/gitlab/kubernetes/kubeconfig/entry/context.rb create mode 100644 lib/gitlab/kubernetes/kubeconfig/entry/user.rb create mode 100644 lib/gitlab/kubernetes/kubeconfig/template.rb create mode 100644 lib/gitlab/pagination/keyset/column_condition_builder.rb create mode 100644 lib/gitlab/signed_tag.rb delete mode 100644 lib/gitlab/usage/docs/helper.rb delete mode 100644 lib/gitlab/usage/docs/renderer.rb delete mode 100644 lib/gitlab/usage/docs/templates/default.md.haml delete mode 100644 lib/gitlab/usage/docs/value_formatter.rb create mode 100644 lib/gitlab/usage/metrics/aggregates.rb create mode 100644 lib/gitlab/usage/metrics/aggregates/sources.rb create mode 100644 lib/gitlab/usage/metrics/instrumentations/redis_metric.rb create mode 100644 lib/gitlab/usage_data_counters/diffs_counter.rb create mode 100644 lib/sidebars/concerns/has_partial.rb create mode 100644 lib/sidebars/groups/menus/ci_cd_menu.rb create mode 100644 lib/sidebars/groups/menus/group_information_menu.rb create mode 100644 lib/sidebars/groups/menus/issues_menu.rb create mode 100644 lib/sidebars/groups/menus/kubernetes_menu.rb create mode 100644 lib/sidebars/groups/menus/merge_requests_menu.rb create mode 100644 lib/sidebars/groups/menus/packages_registries_menu.rb create mode 100644 lib/sidebars/groups/menus/settings_menu.rb create mode 100644 lib/tasks/gitlab/product_intelligence.rake create mode 100644 lib/tasks/gitlab/smtp.rake (limited to 'lib') diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index aea4231205d..2698d7adbd7 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -15,7 +15,7 @@ module AfterCommitQueue end def run_after_commit_or_now(&block) - if Gitlab::Database.inside_transaction? + if Gitlab::Database.main.inside_transaction? if ActiveRecord::Base.connection.current_transaction.records&.include?(self) run_after_commit(&block) else diff --git a/lib/api/api.rb b/lib/api/api.rb index f9e89191a36..40f1b2fa9d3 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -153,10 +153,14 @@ module API mount ::API::Branches mount ::API::BroadcastMessages mount ::API::BulkImports + mount ::API::Ci::JobArtifacts + mount ::API::Ci::Jobs mount ::API::Ci::Pipelines mount ::API::Ci::PipelineSchedules mount ::API::Ci::Runner mount ::API::Ci::Runners + mount ::API::Ci::Triggers + mount ::API::Ci::Variables mount ::API::Commits mount ::API::CommitStatuses mount ::API::ContainerRegistryEvent @@ -184,14 +188,13 @@ module API mount ::API::GroupMilestones mount ::API::Groups mount ::API::GroupContainerRepositories + mount ::API::GroupDebianDistributions mount ::API::GroupVariables mount ::API::ImportBitbucketServer mount ::API::ImportGithub mount ::API::IssueLinks mount ::API::Invitations mount ::API::Issues - mount ::API::JobArtifacts - mount ::API::Jobs mount ::API::Keys mount ::API::Labels mount ::API::Lint @@ -268,14 +271,12 @@ module API mount ::API::Tags mount ::API::Templates mount ::API::Todos - mount ::API::Triggers mount ::API::Unleash mount ::API::UsageData mount ::API::UsageDataQueries mount ::API::UsageDataNonSqlMetrics mount ::API::UserCounts mount ::API::Users - mount ::API::Variables mount ::API::Version mount ::API::Wikis end diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb index fe498bf611b..1eaa4167a7d 100644 --- a/lib/api/appearance.rb +++ b/lib/api/appearance.rb @@ -48,3 +48,5 @@ module API end end end + +API::Appearance.prepend_mod diff --git a/lib/api/badges.rb b/lib/api/badges.rb index 04f155be4e1..d7c850c2f40 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -8,7 +8,7 @@ module API helpers ::API::Helpers::BadgesHelpers - feature_category :continuous_integration + feature_category :projects helpers do def find_source_if_admin(source_type) diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index 189851cee65..0705a8285c1 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -8,7 +8,10 @@ module API helpers do def bulk_imports - @bulk_imports ||= ::BulkImports::ImportsFinder.new(user: current_user, status: params[:status]).execute + @bulk_imports ||= ::BulkImports::ImportsFinder.new( + user: current_user, + status: params[:status] + ).execute end def bulk_import @@ -16,7 +19,11 @@ module API end def bulk_import_entities - @bulk_import_entities ||= ::BulkImports::EntitiesFinder.new(user: current_user, bulk_import: bulk_import, status: params[:status]).execute + @bulk_import_entities ||= ::BulkImports::EntitiesFinder.new( + user: current_user, + bulk_import: bulk_import, + status: params[:status] + ).execute end def bulk_import_entity @@ -27,13 +34,44 @@ module API before { authenticate! } resource :bulk_imports do + desc 'Start a new GitLab Migration' do + detail 'This feature was introduced in GitLab 14.2.' + end + params do + requires :configuration, type: Hash, desc: 'The source GitLab instance configuration' do + requires :url, type: String, desc: 'Source GitLab instance URL' + requires :access_token, type: String, desc: 'Access token to the source GitLab instance' + end + requires :entities, type: Array, desc: 'List of entities to import' do + requires :source_type, type: String, desc: 'Source entity type (only `group_entity` is supported)', + values: %w[group_entity] + requires :source_full_path, type: String, desc: 'Source full path of the entity to import' + requires :destination_name, type: String, desc: 'Destination name for the entity' + requires :destination_namespace, type: String, desc: 'Destination namespace for the entity' + end + end + post do + response = BulkImportService.new( + current_user, + params[:entities], + url: params[:configuration][:url], + access_token: params[:configuration][:access_token] + ).execute + + if response.success? + present response.payload, with: Entities::BulkImport + else + render_api_error!(response.message, response.http_status) + end + end + desc 'List all GitLab Migrations' do detail 'This feature was introduced in GitLab 14.1.' end params do use :pagination optional :status, type: String, values: BulkImport.all_human_statuses, - desc: 'Return GitLab Migrations with specified status' + desc: 'Return GitLab Migrations with specified status' end get do present paginate(bulk_imports), with: Entities::BulkImport @@ -45,10 +83,13 @@ module API params do use :pagination optional :status, type: String, values: ::BulkImports::Entity.all_human_statuses, - desc: "Return all GitLab Migrations' entities with specified status" + desc: "Return all GitLab Migrations' entities with specified status" end get :entities do - entities = ::BulkImports::EntitiesFinder.new(user: current_user, status: params[:status]).execute + entities = ::BulkImports::EntitiesFinder.new( + user: current_user, + status: params[:status] + ).execute present paginate(entities), with: Entities::BulkImports::Entity end @@ -69,7 +110,7 @@ module API params do requires :import_id, type: Integer, desc: "The ID of user's GitLab Migration" optional :status, type: String, values: ::BulkImports::Entity.all_human_statuses, - desc: 'Return import entities with specified status' + desc: 'Return import entities with specified status' use :pagination end get ':import_id/entities' do diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb new file mode 100644 index 00000000000..b9662b822fb --- /dev/null +++ b/lib/api/ci/helpers/runner.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module API + module Ci + module Helpers + module Runner + include Gitlab::Utils::StrongMemoize + + prepend_mod_with('API::Ci::Helpers::Runner') # rubocop: disable Cop/InjectEnterpriseEditionModule + + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN' + JOB_TOKEN_PARAM = :token + + def runner_registration_token_valid? + ActiveSupport::SecurityUtils.secure_compare(params[:token], Gitlab::CurrentSettings.runners_registration_token) + end + + def runner_registrar_valid?(type) + Feature.disabled?(:runner_registration_control) || Gitlab::CurrentSettings.valid_runner_registrars.include?(type) + end + + def authenticate_runner! + forbidden! unless current_runner + + current_runner + .heartbeat(get_runner_details_from_request) + end + + def get_runner_details_from_request + return get_runner_ip unless params['info'].present? + + attributes_for_keys(%w(name version revision platform architecture), params['info']) + .merge(get_runner_config_from_request) + .merge(get_runner_ip) + end + + def get_runner_ip + { ip_address: ip_address } + end + + def current_runner + token = params[:token] + + if token + ::Gitlab::Database::LoadBalancing::RackMiddleware + .stick_or_unstick(env, :runner, token) + end + + strong_memoize(:current_runner) do + ::Ci::Runner.find_by_token(token.to_s) + end + end + + # HTTP status codes to terminate the job on GitLab Runner: + # - 403 + def authenticate_job!(require_running: true) + job = current_job + + # 404 is not returned here because we want to terminate the job if it's + # running. A 404 can be returned from anywhere in the networking stack which is why + # we are explicit about a 403, we should improve this in + # https://gitlab.com/gitlab-org/gitlab/-/issues/327703 + forbidden! unless job + + forbidden! unless job_token_valid?(job) + + forbidden!('Project has been deleted!') if job.project.nil? || job.project.pending_delete? + forbidden!('Job has been erased!') if job.erased? + + if require_running + job_forbidden!(job, 'Job is not running') unless job.running? + end + + job.runner&.heartbeat(get_runner_ip) + + job + end + + def current_job + id = params[:id] + + if id + ::Gitlab::Database::LoadBalancing::RackMiddleware + .stick_or_unstick(env, :build, id) + end + + strong_memoize(:current_job) do + ::Ci::Build.find_by_id(id) + end + end + + def job_token_valid?(job) + token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s + token && job.valid_token?(token) + end + + def job_forbidden!(job, reason) + header 'Job-Status', job.status + forbidden!(reason) + end + + def set_application_context + return unless current_job + + Gitlab::ApplicationContext.push( + user: -> { current_job.user }, + project: -> { current_job.project } + ) + end + + def track_ci_minutes_usage!(_build, _runner) + # noop: overridden in EE + end + + private + + def get_runner_config_from_request + { config: attributes_for_keys(%w(gpus), params.dig('info', 'config')) } + end + end + end + end +end diff --git a/lib/api/ci/job_artifacts.rb b/lib/api/ci/job_artifacts.rb new file mode 100644 index 00000000000..6431436b50d --- /dev/null +++ b/lib/api/ci/job_artifacts.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +module API + module Ci + class JobArtifacts < ::API::Base + before { authenticate_non_get! } + + feature_category :build_artifacts + + # EE::API::Ci::JobArtifacts would override the following helpers + helpers do + def authorize_download_artifacts! + authorize_read_builds! + end + end + + prepend_mod_with('API::Ci::JobArtifacts') # rubocop: disable Cop/InjectEnterpriseEditionModule + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Download the artifacts archive from a job' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the job' + end + route_setting :authentication, job_token_allowed: true + get ':id/jobs/artifacts/:ref_name/download', + requirements: { ref_name: /.+/ } do + authorize_download_artifacts! + + latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) + authorize_read_job_artifacts!(latest_build) + + present_carrierwave_file!(latest_build.artifacts_file) + end + + desc 'Download a specific file from artifacts archive from a ref' do + detail 'This feature was introduced in GitLab 11.5' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the job' + requires :artifact_path, type: String, desc: 'Artifact path' + end + route_setting :authentication, job_token_allowed: true + get ':id/jobs/artifacts/:ref_name/raw/*artifact_path', + format: false, + requirements: { ref_name: /.+/ } do + authorize_download_artifacts! + + build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) + authorize_read_job_artifacts!(build) + + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:artifact_path]) + + bad_request! unless path.valid? + + send_artifacts_entry(build.artifacts_file, path) + end + + desc 'Download the artifacts archive from a job' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + route_setting :authentication, job_token_allowed: true + get ':id/jobs/:job_id/artifacts' do + authorize_download_artifacts! + + build = find_build!(params[:job_id]) + authorize_read_job_artifacts!(build) + + present_carrierwave_file!(build.artifacts_file) + end + + desc 'Download a specific file from artifacts archive' do + detail 'This feature was introduced in GitLab 10.0' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + requires :artifact_path, type: String, desc: 'Artifact path' + end + route_setting :authentication, job_token_allowed: true + get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do + authorize_download_artifacts! + + build = find_build!(params[:job_id]) + authorize_read_job_artifacts!(build) + + not_found! unless build.available_artifacts? + + path = Gitlab::Ci::Build::Artifacts::Path + .new(params[:artifact_path]) + + bad_request! unless path.valid? + + send_artifacts_entry(build.artifacts_file, path) + end + + desc 'Keep the artifacts to prevent them from being deleted' do + success ::API::Entities::Ci::Job + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + post ':id/jobs/:job_id/artifacts/keep' do + authorize_update_builds! + + build = find_build!(params[:job_id]) + authorize!(:update_build, build) + break not_found!(build) unless build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: ::API::Entities::Ci::Job + end + + desc 'Delete the artifacts files from a job' do + detail 'This feature was introduced in GitLab 11.9' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + delete ':id/jobs/:job_id/artifacts' do + authorize_destroy_artifacts! + build = find_build!(params[:job_id]) + authorize!(:destroy_artifacts, build) + + build.erase_erasable_artifacts! + + status :no_content + end + end + end + end +end diff --git a/lib/api/ci/jobs.rb b/lib/api/ci/jobs.rb new file mode 100644 index 00000000000..eea1637c32a --- /dev/null +++ b/lib/api/ci/jobs.rb @@ -0,0 +1,206 @@ +# frozen_string_literal: true + +module API + module Ci + class Jobs < ::API::Base + include PaginationParams + before { authenticate! } + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + params do + requires :id, type: String, desc: 'The ID of a project' + end + + helpers do + params :optional_scope do + optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + values: ::CommitStatus::AVAILABLE_STATUSES, + coerce_with: ->(scope) { + case scope + when String + [scope] + when ::Hash + scope.values + when ::Array + scope + else + ['unknown'] + end + } + end + end + + desc 'Get a projects jobs' do + success Entities::Ci::Job + end + params do + use :optional_scope + use :pagination + end + # rubocop: disable CodeReuse/ActiveRecord + get ':id/jobs', feature_category: :continuous_integration do + authorize_read_builds! + + builds = user_project.builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) + + builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, :tags, pipeline: :project) + present paginate(builds), with: Entities::Ci::Job + end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'Get a specific job of a project' do + success Entities::Ci::Job + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + get ':id/jobs/:job_id', feature_category: :continuous_integration do + authorize_read_builds! + + build = find_build!(params[:job_id]) + + present build, with: Entities::Ci::Job + end + + # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace + # is saved in the DB instead of file). But before that, we need to consider how to replace the value of + # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + desc 'Get a trace of a specific job of a project' + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + get ':id/jobs/:job_id/trace', feature_category: :continuous_integration do + authorize_read_builds! + + build = find_build!(params[:job_id]) + + authorize_read_build_trace!(build) if build + + header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" + content_type 'text/plain' + env['api.format'] = :binary + + # The trace can be nil bu body method expects a string as an argument. + trace = build.trace.raw || '' + body trace + end + + desc 'Cancel a specific job of a project' do + success Entities::Ci::Job + end + params do + requires :job_id, type: Integer, desc: 'The ID of a job' + end + post ':id/jobs/:job_id/cancel', feature_category: :continuous_integration do + authorize_update_builds! + + build = find_build!(params[:job_id]) + authorize!(:update_build, build) + + build.cancel + + present build, with: Entities::Ci::Job + end + + desc 'Retry a specific build of a project' do + success Entities::Ci::Job + end + params do + requires :job_id, type: Integer, desc: 'The ID of a build' + end + post ':id/jobs/:job_id/retry', feature_category: :continuous_integration do + authorize_update_builds! + + build = find_build!(params[:job_id]) + authorize!(:update_build, build) + break forbidden!('Job is not retryable') unless build.retryable? + + build = ::Ci::Build.retry(build, current_user) + + present build, with: Entities::Ci::Job + end + + desc 'Erase job (remove artifacts and the trace)' do + success Entities::Ci::Job + end + params do + requires :job_id, type: Integer, desc: 'The ID of a build' + end + post ':id/jobs/:job_id/erase', feature_category: :continuous_integration do + authorize_update_builds! + + build = find_build!(params[:job_id]) + authorize!(:erase_build, build) + break forbidden!('Job is not erasable!') unless build.erasable? + + build.erase(erased_by: current_user) + present build, with: Entities::Ci::Job + end + + desc 'Trigger an actionable job (manual, delayed, etc)' do + success Entities::Ci::JobBasic + detail 'This feature was added in GitLab 8.11' + end + params do + requires :job_id, type: Integer, desc: 'The ID of a Job' + end + + post ":id/jobs/:job_id/play", feature_category: :continuous_integration do + authorize_read_builds! + + job = find_job!(params[:job_id]) + + authorize!(:play_job, job) + + bad_request!("Unplayable Job") unless job.playable? + + job.play(current_user) + + status 200 + + if job.is_a?(::Ci::Build) + present job, with: Entities::Ci::Job + else + present job, with: Entities::Ci::Bridge + end + end + end + + resource :job do + desc 'Get current project using job token' do + success Entities::Ci::Job + end + route_setting :authentication, job_token_allowed: true + get '', feature_category: :continuous_integration do + validate_current_authenticated_job + + present current_authenticated_job, with: Entities::Ci::Job + end + end + + helpers do + # rubocop: disable CodeReuse/ActiveRecord + def filter_builds(builds, scope) + return builds if scope.nil? || scope.empty? + + available_statuses = ::CommitStatus::AVAILABLE_STATUSES + + unknown = scope - available_statuses + render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? + + builds.where(status: available_statuses && scope) + end + # rubocop: enable CodeReuse/ActiveRecord + + def validate_current_authenticated_job + # current_authenticated_job will be nil if user is using + # a valid authentication (like PRIVATE-TOKEN) that is not CI_JOB_TOKEN + not_found!('Job') unless current_authenticated_job + end + end + end + end +end + +API::Ci::Jobs.prepend_mod_with('API::Ci::Jobs') diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 339c0e779f9..4d6d38f2dce 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -44,7 +44,7 @@ module API optional :ref, type: String, desc: 'The ref of pipelines' optional :sha, type: String, desc: 'The sha of pipelines' optional :yaml_errors, type: Boolean, desc: 'Returns pipelines with invalid configurations' - optional :name, type: String, desc: 'The name of the user who triggered pipelines' + optional :name, type: String, desc: '(deprecated) The name of the user who triggered pipelines' optional :username, type: String, desc: 'The username of the user who triggered pipelines' optional :updated_before, type: DateTime, desc: 'Return pipelines updated before the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' optional :updated_after, type: DateTime, desc: 'Return pipelines updated after the specified datetime. Format: ISO 8601 YYYY-MM-DDTHH:MM:SSZ' @@ -52,13 +52,14 @@ module API desc: 'Order pipelines' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Sort pipelines' + optional :source, type: String, values: ::Ci::Pipeline.sources.keys end get ':id/pipelines' do authorize! :read_pipeline, user_project authorize! :read_build, user_project pipelines = ::Ci::PipelinesFinder.new(user_project, current_user, params).execute - present paginate(pipelines), with: Entities::Ci::PipelineBasic + present paginate(pipelines), with: Entities::Ci::PipelineBasic, project: user_project end desc 'Create a new pipeline' do @@ -78,12 +79,11 @@ module API .merge(variables_attributes: params[:variables]) .except(:variables) - new_pipeline = ::Ci::CreatePipelineService.new(user_project, - current_user, - pipeline_params) - .execute(:api, ignore_skip_ci: true, save_on_errors: false) + response = ::Ci::CreatePipelineService.new(user_project, current_user, pipeline_params) + .execute(:api, ignore_skip_ci: true, save_on_errors: false) + new_pipeline = response.payload - if new_pipeline.persisted? + if response.success? present new_pipeline, with: Entities::Ci::Pipeline else render_validation_error!(new_pipeline) @@ -188,6 +188,19 @@ module API present pipeline.test_reports, with: TestReportEntity, details: true end + desc 'Gets the test report summary for a given pipeline' do + detail 'This feature was introduced in GitLab 14.2' + success TestReportSummaryEntity + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + get ':id/pipelines/:pipeline_id/test_report_summary' do + authorize! :read_build, pipeline + + present pipeline.test_report_summary, with: TestReportSummaryEntity + end + desc 'Deletes a pipeline' do detail 'This feature was introduced in GitLab 11.6' http_codes [[204, 'Pipeline was deleted'], [403, 'Forbidden']] diff --git a/lib/api/ci/runner.rb b/lib/api/ci/runner.rb index 0bac6fe2054..aabcf34952c 100644 --- a/lib/api/ci/runner.rb +++ b/lib/api/ci/runner.rb @@ -3,12 +3,10 @@ module API module Ci class Runner < ::API::Base - helpers ::API::Helpers::Runner + helpers ::API::Ci::Helpers::Runner content_type :txt, 'text/plain' - feature_category :runner - resource :runners do desc 'Registers a new Runner' do success Entities::Ci::RunnerRegistrationDetails @@ -26,7 +24,7 @@ module API optional :tag_list, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: %q(List of Runner's tags) optional :maximum_timeout, type: Integer, desc: 'Maximum timeout set when this Runner will handle the job' end - post '/' do + post '/', feature_category: :runner do attributes = attributes_for_keys([:description, :active, :locked, :run_untagged, :tag_list, :access_level, :maximum_timeout]) .merge(get_runner_details_from_request) @@ -59,7 +57,7 @@ module API params do requires :token, type: String, desc: %q(Runner's authentication token) end - delete '/' do + delete '/', feature_category: :runner do authenticate_runner! destroy_conditionally!(current_runner) @@ -71,7 +69,7 @@ module API params do requires :token, type: String, desc: %q(Runner's authentication token) end - post '/verify' do + post '/verify', feature_category: :runner do authenticate_runner! status 200 body "200" @@ -123,7 +121,7 @@ module API formatter :build_json, ->(object, _) { object } parser :build_json, ::Grape::Parser::Json - post '/request' do + post '/request', feature_category: :continuous_integration do authenticate_runner! unless current_runner.active? @@ -177,7 +175,7 @@ module API end optional :exit_code, type: Integer, desc: %q(Job's exit code) end - put '/:id' do + put '/:id', feature_category: :continuous_integration do job = authenticate_job! Gitlab::Metrics.add_event(:update_build) @@ -204,7 +202,7 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :token, type: String, desc: %q(Job's authentication token) end - patch '/:id/trace' do + patch '/:id/trace', feature_category: :continuous_integration do job = authenticate_job! error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range') @@ -249,7 +247,7 @@ module API optional :artifact_type, type: String, desc: %q(The type of artifact), default: 'archive', values: ::Ci::JobArtifact.file_types.keys end - post '/:id/artifacts/authorize' do + post '/:id/artifacts/authorize', feature_category: :build_artifacts do not_allowed! unless Gitlab.config.artifacts.enabled require_gitlab_workhorse! @@ -285,7 +283,7 @@ module API default: 'zip', values: ::Ci::JobArtifact.file_formats.keys optional :metadata, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact metadata to store (generated by Multipart middleware)) end - post '/:id/artifacts' do + post '/:id/artifacts', feature_category: :build_artifacts do not_allowed! unless Gitlab.config.artifacts.enabled require_gitlab_workhorse! @@ -314,7 +312,7 @@ module API optional :token, type: String, desc: %q(Job's authentication token) optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts) end - get '/:id/artifacts' do + get '/:id/artifacts', feature_category: :build_artifacts do job = authenticate_job!(require_running: false) present_carrierwave_file!(job.artifacts_file, supports_direct_download: params[:direct_download]) diff --git a/lib/api/ci/triggers.rb b/lib/api/ci/triggers.rb new file mode 100644 index 00000000000..6a2b16e1568 --- /dev/null +++ b/lib/api/ci/triggers.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +module API + module Ci + class Triggers < ::API::Base + include PaginationParams + + HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase + + feature_category :continuous_integration + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Trigger a GitLab project pipeline' do + success Entities::Ci::Pipeline + end + params do + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false + requires :token, type: String, desc: 'The unique token of trigger or job token' + optional :variables, type: Hash, desc: 'The list of variables to be injected into build' + end + post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do + Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20758') + + forbidden! if gitlab_pipeline_hook_request? + + # validate variables + params[:variables] = params[:variables].to_h + unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) + end + + project = find_project(params[:id]) + not_found! unless project + + result = ::Ci::PipelineTriggerService.new(project, nil, params).execute + not_found! unless result + + if result.error? + render_api_error!(result[:message], result[:http_status]) + else + present result[:pipeline], with: Entities::Ci::Pipeline + end + end + + desc 'Get triggers list' do + success Entities::Trigger + end + params do + use :pagination + end + # rubocop: disable CodeReuse/ActiveRecord + get ':id/triggers' do + authenticate! + authorize! :admin_build, user_project + + triggers = user_project.triggers.includes(:trigger_requests) + + present paginate(triggers), with: Entities::Trigger, current_user: current_user + end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'Get specific trigger of a project' do + success Entities::Trigger + end + params do + requires :trigger_id, type: Integer, desc: 'The trigger ID' + end + get ':id/triggers/:trigger_id' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find(params.delete(:trigger_id)) + break not_found!('Trigger') unless trigger + + present trigger, with: Entities::Trigger, current_user: current_user + end + + desc 'Create a trigger' do + success Entities::Trigger + end + params do + requires :description, type: String, desc: 'The trigger description' + end + post ':id/triggers' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.create( + declared_params(include_missing: false).merge(owner: current_user)) + + if trigger.valid? + present trigger, with: Entities::Trigger, current_user: current_user + else + render_validation_error!(trigger) + end + end + + desc 'Update a trigger' do + success Entities::Trigger + end + params do + requires :trigger_id, type: Integer, desc: 'The trigger ID' + optional :description, type: String, desc: 'The trigger description' + end + put ':id/triggers/:trigger_id' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find(params.delete(:trigger_id)) + break not_found!('Trigger') unless trigger + + authorize! :admin_trigger, trigger + + if trigger.update(declared_params(include_missing: false)) + present trigger, with: Entities::Trigger, current_user: current_user + else + render_validation_error!(trigger) + end + end + + desc 'Delete a trigger' do + success Entities::Trigger + end + params do + requires :trigger_id, type: Integer, desc: 'The trigger ID' + end + delete ':id/triggers/:trigger_id' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find(params.delete(:trigger_id)) + break not_found!('Trigger') unless trigger + + destroy_conditionally!(trigger) + end + end + + helpers do + def gitlab_pipeline_hook_request? + request.get_header(HTTP_GITLAB_EVENT_HEADER) == WebHookService.hook_to_event(:pipeline_hooks) + end + end + end + end +end diff --git a/lib/api/ci/variables.rb b/lib/api/ci/variables.rb new file mode 100644 index 00000000000..9c04d5e9923 --- /dev/null +++ b/lib/api/ci/variables.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +module API + module Ci + class Variables < ::API::Base + include PaginationParams + + before { authenticate! } + before { authorize! :admin_build, user_project } + + feature_category :pipeline_authoring + + helpers ::API::Helpers::VariablesHelpers + + params do + requires :id, type: String, desc: 'The ID of a project' + end + + resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Get project variables' do + success Entities::Ci::Variable + end + params do + use :pagination + end + get ':id/variables' do + variables = user_project.variables + present paginate(variables), with: Entities::Ci::Variable + end + + desc 'Get a specific variable from a project' do + success Entities::Ci::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end + # rubocop: disable CodeReuse/ActiveRecord + get ':id/variables/:key' do + variable = find_variable(user_project, params) + not_found!('Variable') unless variable + + present variable, with: Entities::Ci::Variable + end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'Create a new variable in a project' do + success Entities::Ci::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + optional :protected, type: Boolean, desc: 'Whether the variable is protected' + optional :masked, type: Boolean, desc: 'Whether the variable is masked' + optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' + end + post ':id/variables' do + variable = ::Ci::ChangeVariableService.new( + container: user_project, + current_user: current_user, + params: { action: :create, variable_params: declared_params(include_missing: false) } + ).execute + + if variable.valid? + present variable, with: Entities::Ci::Variable + else + render_validation_error!(variable) + end + end + + desc 'Update an existing variable from a project' do + success Entities::Ci::Variable + end + params do + optional :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + optional :protected, type: Boolean, desc: 'Whether the variable is protected' + optional :masked, type: Boolean, desc: 'Whether the variable is masked' + optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' + optional :environment_scope, type: String, desc: 'The environment_scope of the variable' + optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' + end + # rubocop: disable CodeReuse/ActiveRecord + put ':id/variables/:key' do + variable = find_variable(user_project, params) + not_found!('Variable') unless variable + + variable = ::Ci::ChangeVariableService.new( + container: user_project, + current_user: current_user, + params: { action: :update, variable: variable, variable_params: declared_params(include_missing: false).except(:key, :filter) } + ).execute + + if variable.valid? + present variable, with: Entities::Ci::Variable + else + render_validation_error!(variable) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + desc 'Delete an existing variable from a project' do + success Entities::Ci::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' + end + # rubocop: disable CodeReuse/ActiveRecord + delete ':id/variables/:key' do + variable = find_variable(user_project, params) + not_found!('Variable') unless variable + + ::Ci::ChangeVariableService.new( + container: user_project, + current_user: current_user, + params: { action: :destroy, variable: variable } + ).execute + + no_content! + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 541a37b0abe..5d8985455ad 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -47,7 +47,7 @@ module API path = params[:path] before = params[:until] after = params[:since] - ref = params[:ref_name].presence || user_project.try(:default_branch) || 'master' unless params[:all] + ref = params[:ref_name].presence || user_project.default_branch unless params[:all] offset = (params[:page] - 1) * params[:per_page] all = params[:all] with_stats = params[:with_stats] diff --git a/lib/api/concerns/packages/debian_distribution_endpoints.rb b/lib/api/concerns/packages/debian_distribution_endpoints.rb index 4670c3e3521..798e583b87a 100644 --- a/lib/api/concerns/packages/debian_distribution_endpoints.rb +++ b/lib/api/concerns/packages/debian_distribution_endpoints.rb @@ -80,6 +80,8 @@ module API use :optional_distribution_params end get '/' do + authorize_read_package!(project_or_group) + distribution_params = declared_params(include_missing: false) distributions = ::Packages::Debian::DistributionsFinder.new(project_or_group, distribution_params).execute @@ -96,6 +98,8 @@ module API requires :codename, type: String, regexp: Gitlab::Regex.debian_distribution_regex, desc: 'The Debian Codename' end get '/:codename' do + authorize_read_package!(project_or_group) + distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename: params[:codename]).execute.last! present distribution, with: ::API::Entities::Packages::Debian::Distribution diff --git a/lib/api/concerns/packages/debian_package_endpoints.rb b/lib/api/concerns/packages/debian_package_endpoints.rb index 7740ba6bfa6..0acc015f366 100644 --- a/lib/api/concerns/packages/debian_package_endpoints.rb +++ b/lib/api/concerns/packages/debian_package_endpoints.rb @@ -6,8 +6,6 @@ module API module DebianPackageEndpoints extend ActiveSupport::Concern - LETTER_REGEX = %r{(lib)?[a-z0-9]}.freeze - PACKAGE_REGEX = API::NO_SLASH_URL_PART_REGEX DISTRIBUTION_REQUIREMENTS = { distribution: ::Packages::Debian::DISTRIBUTION_REGEX }.freeze @@ -15,14 +13,6 @@ module API component: ::Packages::Debian::COMPONENT_REGEX, architecture: ::Packages::Debian::ARCHITECTURE_REGEX }.freeze - COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS = { - component: ::Packages::Debian::COMPONENT_REGEX, - letter: LETTER_REGEX, - source_package: PACKAGE_REGEX - }.freeze - FILE_NAME_REQUIREMENTS = { - file_name: API::NO_SLASH_URL_PART_REGEX - }.freeze included do feature_category :package_registry @@ -31,109 +21,106 @@ module API helpers ::API::Helpers::Packages::BasicAuthHelpers include ::API::Helpers::Authentication - namespace 'packages/debian' do - authenticate_with do |accept| - accept.token_types(:personal_access_token, :deploy_token, :job_token) - .sent_through(:http_basic_auth) + helpers do + params :shared_package_file_params do + requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex + requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)' + requires :package_name, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex + requires :package_version, type: String, desc: 'The Debian Source Package Version', regexp: Gitlab::Regex.debian_version_regex + requires :file_name, type: String, desc: 'The Debian File Name' end - helpers do - def present_release_file - distribution = ::Packages::Debian::DistributionsFinder.new(project_or_group, codename_or_suite: params[:distribution]).execute.last! - - present_carrierwave_file!(distribution.file) - end + def distribution_from!(container) + ::Packages::Debian::DistributionsFinder.new(container, codename_or_suite: params[:distribution]).execute.last! end - format :txt - content_type :txt, 'text/plain' + def present_package_file! + not_found! unless params[:package_name].start_with?(params[:letter]) - params do - requires :distribution, type: String, desc: 'The Debian Codename', regexp: Gitlab::Regex.debian_distribution_regex + package_file = distribution_from!(user_project).package_files.with_file_name(params[:file_name]).last! + + present_carrierwave_file!(package_file.file) end + end - namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg - desc 'The Release file signature' do - detail 'This feature was introduced in GitLab 13.5' - end + authenticate_with do |accept| + accept.token_types(:personal_access_token, :deploy_token, :job_token) + .sent_through(:http_basic_auth) + end - route_setting :authentication, authenticate_non_public: true - get 'Release.gpg' do - not_found! - end + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end - # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release - desc 'The unsigned Release file' do - detail 'This feature was introduced in GitLab 13.5' - end + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end - route_setting :authentication, authenticate_non_public: true - get 'Release' do - present_release_file - end + format :txt + content_type :txt, 'text/plain' - # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease - desc 'The signed Release file' do - detail 'This feature was introduced in GitLab 13.5' - end + params do + requires :distribution, type: String, desc: 'The Debian Codename or Suite', regexp: Gitlab::Regex.debian_distribution_regex + end - route_setting :authentication, authenticate_non_public: true - get 'InRelease' do - # Signature to be added in 7.3 of https://gitlab.com/groups/gitlab-org/-/epics/6057#note_582697034 - present_release_file - end + namespace 'dists/*distribution', requirements: DISTRIBUTION_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release.gpg + desc 'The Release file signature' do + detail 'This feature was introduced in GitLab 13.5' + end - params do - requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex - requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex - end + route_setting :authentication, authenticate_non_public: true + get 'Release.gpg' do + distribution_from!(project_or_group).file_signature + end - namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages - desc 'The binary files index' do - detail 'This feature was introduced in GitLab 13.5' - end - - route_setting :authentication, authenticate_non_public: true - get 'Packages' do - relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize - - component_file = relation - .preload_distribution - .with_container(project_or_group) - .with_codename_or_suite(params[:distribution]) - .with_component_name(params[:component]) - .with_file_type(:packages) - .with_architecture_name(params[:architecture]) - .with_compression_type(nil) - .order_created_asc - .last! - - present_carrierwave_file!(component_file.file) - end - end + # GET {projects|groups}/:id/packages/debian/dists/*distribution/Release + desc 'The unsigned Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, authenticate_non_public: true + get 'Release' do + present_carrierwave_file!(distribution_from!(project_or_group).file) + end + + # GET {projects|groups}/:id/packages/debian/dists/*distribution/InRelease + desc 'The signed Release file' do + detail 'This feature was introduced in GitLab 13.5' + end + + route_setting :authentication, authenticate_non_public: true + get 'InRelease' do + present_carrierwave_file!(distribution_from!(project_or_group).signed_file) end params do requires :component, type: String, desc: 'The Debian Component', regexp: Gitlab::Regex.debian_component_regex - requires :letter, type: String, desc: 'The Debian Classification (first-letter or lib-first-letter)' - requires :source_package, type: String, desc: 'The Debian Source Package Name', regexp: Gitlab::Regex.debian_package_name_regex + requires :architecture, type: String, desc: 'The Debian Architecture', regexp: Gitlab::Regex.debian_architecture_regex end - namespace 'pool/:component/:letter/:source_package', requirements: COMPONENT_LETTER_SOURCE_PACKAGE_REQUIREMENTS do - # GET {projects|groups}/:id/packages/debian/pool/:component/:letter/:source_package/:file_name - params do - requires :file_name, type: String, desc: 'The Debian File Name' - end - desc 'The package' do + namespace ':component/binary-:architecture', requirements: COMPONENT_ARCHITECTURE_REQUIREMENTS do + # GET {projects|groups}/:id/packages/debian/dists/*distribution/:component/binary-:architecture/Packages + desc 'The binary files index' do detail 'This feature was introduced in GitLab 13.5' end route_setting :authentication, authenticate_non_public: true - get ':file_name', requirements: FILE_NAME_REQUIREMENTS do - # https://gitlab.com/gitlab-org/gitlab/-/issues/5835#note_414103286 - 'TODO File' + get 'Packages' do + relation = "::Packages::Debian::#{project_or_group.class.name}ComponentFile".constantize + + component_file = relation + .preload_distribution + .with_container(project_or_group) + .with_codename_or_suite(params[:distribution]) + .with_component_name(params[:component]) + .with_file_type(:packages) + .with_architecture_name(params[:architecture]) + .with_compression_type(nil) + .order_created_asc + .last! + + present_carrierwave_file!(component_file.file) end end end diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index 191ed42a5b8..29f5047230a 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -2,35 +2,50 @@ module API class DebianGroupPackages < ::API::Base - params do - requires :id, type: String, desc: 'The ID of a group' - end + PACKAGE_FILE_REQUIREMENTS = ::API::DebianProjectPackages::PACKAGE_FILE_REQUIREMENTS.merge( + project_id: %r{[0-9]+}.freeze + ).freeze resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - rescue_from ArgumentError do |e| - render_api_error!(e.message, 400) - end + helpers do + def user_project + @project ||= find_project!(params[:project_id]) + end - rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) + def project_or_group + user_group + end end - before do + after_validation do require_packages_enabled! - not_found! unless ::Feature.enabled?(:debian_packages, user_group) + not_found! unless ::Feature.enabled?(:debian_group_packages, user_group) authorize_read_package!(user_group) end - namespace ':id/-' do - helpers do - def project_or_group - user_group - end - end + params do + requires :id, type: String, desc: 'The ID of a group' + end + namespace ':id/-/packages/debian' do include ::API::Concerns::Packages::DebianPackageEndpoints + + # GET groups/:id/packages/debian/pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name + params do + requires :project_id, type: Integer, desc: 'The Project Id' + use :shared_package_file_params + end + + desc 'The package' do + detail 'This feature was introduced in GitLab 14.2' + end + + route_setting :authentication, authenticate_non_public: true + get 'pool/:distribution/:project_id/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do + present_package_file! + end end end end diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index 70ddf9dea37..497ce2f4356 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -2,17 +2,23 @@ module API class DebianProjectPackages < ::API::Base - params do - requires :id, type: String, desc: 'The ID of a project' - end + PACKAGE_FILE_REQUIREMENTS = { + id: API::NO_SLASH_URL_PART_REGEX, + distribution: ::Packages::Debian::DISTRIBUTION_REGEX, + letter: ::Packages::Debian::LETTER_REGEX, + package_name: API::NO_SLASH_URL_PART_REGEX, + package_version: API::NO_SLASH_URL_PART_REGEX, + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + FILE_NAME_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - rescue_from ArgumentError do |e| - render_api_error!(e.message, 400) - end - - rescue_from ActiveRecord::RecordInvalid do |e| - render_api_error!(e.message, 400) + helpers do + def project_or_group + user_project + end end after_validation do @@ -23,20 +29,32 @@ module API authorize_read_package! end - namespace ':id' do - helpers do - def project_or_group - user_project - end - end + params do + requires :id, type: String, desc: 'The ID of a project' + end + namespace ':id/packages/debian' do include ::API::Concerns::Packages::DebianPackageEndpoints + # GET projects/:id/packages/debian/pool/:distribution/:letter/:package_name/:package_version/:file_name + params do + use :shared_package_file_params + end + + desc 'The package' do + detail 'This feature was introduced in GitLab 14.2' + end + + route_setting :authentication, authenticate_non_public: true + get 'pool/:distribution/:letter/:package_name/:package_version/:file_name', requirements: PACKAGE_FILE_REQUIREMENTS do + present_package_file! + end + params do requires :file_name, type: String, desc: 'The file name' end - namespace 'packages/debian/:file_name', requirements: FILE_NAME_REQUIREMENTS do + namespace ':file_name', requirements: FILE_NAME_REQUIREMENTS do format :txt content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE diff --git a/lib/api/entities/ci/job_request/dependency.rb b/lib/api/entities/ci/job_request/dependency.rb index 2c6ed417714..2672a4a245b 100644 --- a/lib/api/entities/ci/job_request/dependency.rb +++ b/lib/api/entities/ci/job_request/dependency.rb @@ -6,7 +6,7 @@ module API module JobRequest class Dependency < Grape::Entity expose :id, :name, :token - expose :artifacts_file, using: Entities::Ci::JobArtifactFile, if: ->(job, _) { job.artifacts? } + expose :artifacts_file, using: Entities::Ci::JobArtifactFile, if: ->(job, _) { job.available_artifacts? } end end end diff --git a/lib/api/entities/ci/pipeline_basic.rb b/lib/api/entities/ci/pipeline_basic.rb index f4f2356c812..8086062dc9b 100644 --- a/lib/api/entities/ci/pipeline_basic.rb +++ b/lib/api/entities/ci/pipeline_basic.rb @@ -7,6 +7,8 @@ module API expose :id, :project_id, :sha, :ref, :status expose :created_at, :updated_at + expose :source, if: ->(pipeline, options) { ::Feature.enabled?(:pipeline_source_filter, options[:project], default_enabled: :yaml) } + expose :web_url do |pipeline, _options| Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline) end diff --git a/lib/api/entities/error_tracking.rb b/lib/api/entities/error_tracking.rb index c762c274486..a38e00ca295 100644 --- a/lib/api/entities/error_tracking.rb +++ b/lib/api/entities/error_tracking.rb @@ -8,6 +8,7 @@ module API expose :project_name expose :sentry_external_url expose :api_url + expose :integrated end end end diff --git a/lib/api/entities/issue_basic.rb b/lib/api/entities/issue_basic.rb index 6c332870228..ab248523028 100644 --- a/lib/api/entities/issue_basic.rb +++ b/lib/api/entities/issue_basic.rb @@ -23,7 +23,7 @@ module API expose :issue_type, as: :type, format_with: :upcase, - documentation: { type: "String", desc: "One of #{::Issue.issue_types.keys.map(&:upcase)}" } + documentation: { type: "String", desc: "One of #{::WorkItem::Type.base_types.keys.map(&:upcase)}" } expose :assignee, using: ::API::Entities::UserBasic do |issue| issue.assignees.first diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index f5f565e5b07..890b42ed8c8 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -71,6 +71,7 @@ module API expose(:pages_access_level) { |project, options| project.project_feature.string_access_level(:pages) } expose(:operations_access_level) { |project, options| project.project_feature.string_access_level(:operations) } expose(:analytics_access_level) { |project, options| project.project_feature.string_access_level(:analytics) } + expose(:container_registry_access_level) { |project, options| project.project_feature.string_access_level(:container_registry) } expose :emails_disabled expose :shared_runners_enabled diff --git a/lib/api/entities/project_with_access.rb b/lib/api/entities/project_with_access.rb index c53a712a879..ac89cb52e43 100644 --- a/lib/api/entities/project_with_access.rb +++ b/lib/api/entities/project_with_access.rb @@ -26,8 +26,10 @@ module API # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) relation = super(projects_relation, options) - project_ids = relation.select('projects.id') - namespace_ids = relation.select(:namespace_id) + # use reselect to override the existing select and + # prevent an error `subquery has too many columns` + project_ids = relation.reselect('projects.id') + namespace_ids = relation.reselect(:namespace_id) options[:project_members] = options[:current_user] .project_members diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 57e548183b0..e50da4264b5 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -77,7 +77,7 @@ module API desc "Delete multiple stopped review apps" do detail "Remove multiple stopped review environments older than a specific age" - success Entities::Environment + success Entities::EnvironmentBasic end params do optional :before, type: Time, desc: "The timestamp before which environments can be deleted. Defaults to 30 days ago.", default: -> { 30.days.ago } @@ -90,8 +90,8 @@ module API result = ::Environments::ScheduleToDeleteReviewAppsService.new(user_project, current_user, params).execute response = { - scheduled_entries: Entities::Environment.represent(result.scheduled_entries), - unprocessable_entries: Entities::Environment.represent(result.unprocessable_entries) + scheduled_entries: Entities::EnvironmentBasic.represent(result.scheduled_entries), + unprocessable_entries: Entities::EnvironmentBasic.represent(result.unprocessable_entries) } if result.success? diff --git a/lib/api/error_tracking.rb b/lib/api/error_tracking.rb index 0e44c8b1081..3abf2831bd3 100644 --- a/lib/api/error_tracking.rb +++ b/lib/api/error_tracking.rb @@ -32,6 +32,7 @@ module API end params do requires :active, type: Boolean, desc: 'Specifying whether to enable or disable error tracking settings', allow_blank: false + optional :integrated, type: Boolean, desc: 'Specifying whether to enable or disable integrated error tracking' end patch ':id/error_tracking/settings/' do @@ -45,6 +46,10 @@ module API error_tracking_setting_attributes: { enabled: params[:active] } } + unless params[:integrated].nil? + update_params[:error_tracking_setting_attributes][:integrated] = params[:integrated] + end + result = ::Projects::Operations::UpdateService.new(user_project, current_user, update_params).execute if result[:status] == :success diff --git a/lib/api/error_tracking_collector.rb b/lib/api/error_tracking_collector.rb index 08ff8d2e4d1..13e8e476808 100644 --- a/lib/api/error_tracking_collector.rb +++ b/lib/api/error_tracking_collector.rb @@ -13,6 +13,7 @@ module API before do not_found!('Project') unless project not_found! unless feature_enabled? + not_found! unless active_client_key? end helpers do @@ -21,8 +22,24 @@ module API end def feature_enabled? - ::Feature.enabled?(:integrated_error_tracking, project) && - project.error_tracking_setting&.enabled? + project.error_tracking_setting&.enabled? && + project.error_tracking_setting&.integrated_client? + end + + def find_client_key(public_key) + return unless public_key.present? + + project.error_tracking_client_keys.active.find_by_public_key(public_key) + end + + def active_client_key? + begin + public_key = ::ErrorTracking::Collector::SentryAuthParser.parse(request)[:public_key] + rescue StandardError + bad_request!('Failed to parse sentry request') + end + + find_client_key(public_key) end end @@ -46,7 +63,7 @@ module API begin parsed_request = ::ErrorTracking::Collector::SentryRequestParser.parse(request) rescue StandardError - render_api_error!('Failed to parse sentry request', 400) + bad_request!('Failed to parse sentry request') end type = parsed_request[:request_type] @@ -67,6 +84,9 @@ module API .execute end + # Collector should never return any information back. + # Because DSN and public key are designed for public use, + # it is safe only for submission of new events. no_content! end end diff --git a/lib/api/group_debian_distributions.rb b/lib/api/group_debian_distributions.rb new file mode 100644 index 00000000000..01a8774bd97 --- /dev/null +++ b/lib/api/group_debian_distributions.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module API + class GroupDebianDistributions < ::API::Base + params do + requires :id, type: String, desc: 'The ID of a group' + end + + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + rescue_from ArgumentError do |e| + render_api_error!(e.message, 400) + end + + rescue_from ActiveRecord::RecordInvalid do |e| + render_api_error!(e.message, 400) + end + + after_validation do + require_packages_enabled! + + not_found! unless ::Feature.enabled?(:debian_group_packages, user_group) + end + + namespace ':id/-' do + helpers do + def project_or_group + user_group + end + end + + include ::API::Concerns::Packages::DebianDistributionEndpoints + end + end + end +end diff --git a/lib/api/group_variables.rb b/lib/api/group_variables.rb index 8d52a0a5b4e..13daf05fc78 100644 --- a/lib/api/group_variables.rb +++ b/lib/api/group_variables.rb @@ -8,7 +8,7 @@ module API before { authorize! :admin_group, user_group } feature_category :continuous_integration - helpers Helpers::VariablesHelpers + helpers ::API::Helpers::VariablesHelpers params do requires :id, type: String, desc: 'The ID of a group' diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 9b6b28733ff..0896357cc73 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -35,7 +35,8 @@ module API :all_available, :custom_attributes, :owned, :min_access_level, - :include_parent_descendants + :include_parent_descendants, + :search ) find_params[:parent] = if params[:top_level_only] @@ -48,7 +49,6 @@ module API find_params.fetch(:all_available, current_user&.can_read_all_resources?) groups = GroupsFinder.new(current_user, find_params).execute - groups = groups.search(params[:search], include_parents: true) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? order_groups(groups) @@ -128,10 +128,6 @@ module API groups.reorder(group_without_similarity_options) # rubocop: disable CodeReuse/ActiveRecord end - def order_by_similarity? - params[:order_by] == 'similarity' && params[:search].present? - end - def group_without_similarity_options order_options = { params[:order_by] => params[:sort] } order_options['name'] = order_options.delete('similarity') if order_options.has_key?('similarity') @@ -141,7 +137,7 @@ module API # rubocop: disable CodeReuse/ActiveRecord def handle_similarity_order(group, projects) - if params[:search].present? && Feature.enabled?(:similarity_search, group, default_enabled: true) + if params[:search].present? projects.sorted_by_similarity_desc(params[:search]) else order_options = { name: :asc } diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3398d5da7f5..9c347148fd0 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -577,6 +577,10 @@ module API Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") end + def order_by_similarity?(allow_unauthorized: true) + params[:order_by] == 'similarity' && params[:search].present? && (allow_unauthorized || current_user.present?) + end + protected def project_finder_params_visibility_ce diff --git a/lib/api/helpers/groups_helpers.rb b/lib/api/helpers/groups_helpers.rb index e38213532ba..72bdb32d38c 100644 --- a/lib/api/helpers/groups_helpers.rb +++ b/lib/api/helpers/groups_helpers.rb @@ -23,7 +23,7 @@ module API optional :mentions_disabled, type: Boolean, desc: 'Disable a group from getting mentioned' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to master' + optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to default branch' optional :shared_runners_setting, type: String, values: ::Namespace::SHARED_RUNNERS_SETTINGS, desc: 'Enable/disable shared runners for the group and its subgroups and projects' end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index bd0c2501220..e72bbb931f0 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -54,6 +54,14 @@ module API source.add_user(user, params[:access_level], current_user: current_user, expires_at: params[:expires_at]) end + def track_areas_of_focus(member, areas_of_focus) + return unless areas_of_focus + + areas_of_focus.each do |area_of_focus| + Gitlab::Tracking.event(::Members::CreateService.name, 'area_of_focus', label: area_of_focus, property: member.id.to_s) + end + end + def present_members(members) present members, with: Entities::Member, current_user: current_user, show_seat_info: params[:show_seat_info] end diff --git a/lib/api/helpers/packages/dependency_proxy_helpers.rb b/lib/api/helpers/packages/dependency_proxy_helpers.rb index 989c4e1761b..b8ae1dddd7e 100644 --- a/lib/api/helpers/packages/dependency_proxy_helpers.rb +++ b/lib/api/helpers/packages/dependency_proxy_helpers.rb @@ -5,11 +5,17 @@ module API module Packages module DependencyProxyHelpers REGISTRY_BASE_URLS = { - npm: 'https://registry.npmjs.org/' + npm: 'https://registry.npmjs.org/', + pypi: 'https://pypi.org/simple/' + }.freeze + + APPLICATION_SETTING_NAMES = { + npm: 'npm_package_requests_forwarding', + pypi: 'pypi_package_requests_forwarding' }.freeze def redirect_registry_request(forward_to_registry, package_type, options) - if forward_to_registry && redirect_registry_request_available? + if forward_to_registry && redirect_registry_request_available?(package_type) ::Gitlab::Tracking.event(self.options[:for].name, "#{package_type}_request_forward") redirect(registry_url(package_type, options)) else @@ -25,11 +31,20 @@ module API case package_type when :npm "#{base_url}#{options[:package_name]}" + when :pypi + "#{base_url}#{options[:package_name]}/" end end - def redirect_registry_request_available? - ::Gitlab::CurrentSettings.current_application_settings.npm_package_requests_forwarding + def redirect_registry_request_available?(package_type) + application_setting_name = APPLICATION_SETTING_NAMES[package_type] + + raise ArgumentError, "Can't find application setting for package_type #{package_type}" unless application_setting_name + + ::Gitlab::CurrentSettings + .current_application_settings + .attributes + .fetch(application_setting_name, false) end end end diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index 2d556f889bf..ce5db52fdbc 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -49,28 +49,20 @@ module API when :project params[:id] when :instance - namespace_path = namespace_path_from_package_name + package_name = params[:package_name] + namespace_path = ::Packages::Npm.scope_of(package_name) next unless namespace_path namespace = Namespace.top_most .by_path(namespace_path) next unless namespace - finder = ::Packages::Npm::PackageFinder.new(params[:package_name], namespace: namespace) + finder = ::Packages::Npm::PackageFinder.new(package_name, namespace: namespace) finder.last&.project_id end end end - - # from "@scope/package-name" return "scope" or nil - def namespace_path_from_package_name - package_name = params[:package_name] - return unless package_name.starts_with?('@') - return unless package_name.include?('/') - - package_name.match(Gitlab::Regex.npm_package_name_regex)&.captures&.first - end end end end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 272452bd8db..becd25595a6 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -35,13 +35,14 @@ module API optional :pages_access_level, type: String, values: %w(disabled private enabled public), desc: 'Pages access level. One of `disabled`, `private`, `enabled` or `public`' optional :operations_access_level, type: String, values: %w(disabled private enabled), desc: 'Operations access level. One of `disabled`, `private` or `enabled`' optional :analytics_access_level, type: String, values: %w(disabled private enabled), desc: 'Analytics access level. One of `disabled`, `private` or `enabled`' + optional :container_registry_access_level, type: String, values: %w(disabled private enabled), desc: 'Controls visibility of the container registry. One of `disabled`, `private` or `enabled`. `private` will make the container registry accessible only to project members (reporter role and above). `enabled` will make the container registry accessible to everyone who has access to the project. `disabled` will disable the container registry' optional :emails_disabled, type: Boolean, desc: 'Disable email notifications' optional :show_default_award_emojis, type: Boolean, desc: 'Show default award emojis' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push' optional :remove_source_branch_after_merge, type: Boolean, desc: 'Remove the source branch by default after merge' - optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :container_registry_enabled, type: Boolean, desc: 'Deprecated: Use :container_registry_access_level instead. Flag indication if the container registry is enabled for that project' optional :container_expiration_policy_attributes, type: Hash do use :optional_container_expiration_policy_params end @@ -124,7 +125,7 @@ module API :ci_config_path, :ci_default_git_depth, :ci_forward_deployment_enabled, - :container_registry_enabled, + :container_registry_access_level, :container_expiration_policy_attributes, :default_branch, :description, @@ -132,7 +133,10 @@ module API :forking_access_level, :issues_access_level, :lfs_enabled, + :merge_pipelines_enabled, :merge_requests_access_level, + :merge_requests_template, + :merge_trains_enabled, :merge_method, :name, :only_allow_merge_if_all_discussions_are_resolved, @@ -166,7 +170,8 @@ module API :jobs_enabled, :merge_requests_enabled, :wiki_enabled, - :snippets_enabled + :snippets_enabled, + :container_registry_enabled ] end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb deleted file mode 100644 index a022d1a56ac..00000000000 --- a/lib/api/helpers/runner.rb +++ /dev/null @@ -1,121 +0,0 @@ -# frozen_string_literal: true - -module API - module Helpers - module Runner - include Gitlab::Utils::StrongMemoize - - prepend_mod_with('API::Helpers::Runner') # rubocop: disable Cop/InjectEnterpriseEditionModule - - JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN' - JOB_TOKEN_PARAM = :token - - def runner_registration_token_valid? - ActiveSupport::SecurityUtils.secure_compare(params[:token], Gitlab::CurrentSettings.runners_registration_token) - end - - def runner_registrar_valid?(type) - Feature.disabled?(:runner_registration_control) || Gitlab::CurrentSettings.valid_runner_registrars.include?(type) - end - - def authenticate_runner! - forbidden! unless current_runner - - current_runner - .heartbeat(get_runner_details_from_request) - end - - def get_runner_details_from_request - return get_runner_ip unless params['info'].present? - - attributes_for_keys(%w(name version revision platform architecture), params['info']) - .merge(get_runner_config_from_request) - .merge(get_runner_ip) - end - - def get_runner_ip - { ip_address: ip_address } - end - - def current_runner - token = params[:token] - - if token - ::Gitlab::Database::LoadBalancing::RackMiddleware - .stick_or_unstick(env, :runner, token) - end - - strong_memoize(:current_runner) do - ::Ci::Runner.find_by_token(token.to_s) - end - end - - # HTTP status codes to terminate the job on GitLab Runner: - # - 403 - def authenticate_job!(require_running: true) - job = current_job - - # 404 is not returned here because we want to terminate the job if it's - # running. A 404 can be returned from anywhere in the networking stack which is why - # we are explicit about a 403, we should improve this in - # https://gitlab.com/gitlab-org/gitlab/-/issues/327703 - forbidden! unless job - - forbidden! unless job_token_valid?(job) - - forbidden!('Project has been deleted!') if job.project.nil? || job.project.pending_delete? - forbidden!('Job has been erased!') if job.erased? - - if require_running - job_forbidden!(job, 'Job is not running') unless job.running? - end - - job.runner&.heartbeat(get_runner_ip) - - job - end - - def current_job - id = params[:id] - - if id - ::Gitlab::Database::LoadBalancing::RackMiddleware - .stick_or_unstick(env, :build, id) - end - - strong_memoize(:current_job) do - ::Ci::Build.find_by_id(id) - end - end - - def job_token_valid?(job) - token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s - token && job.valid_token?(token) - end - - def job_forbidden!(job, reason) - header 'Job-Status', job.status - forbidden!(reason) - end - - def set_application_context - return unless current_job - - Gitlab::ApplicationContext.push( - user: -> { current_job.user }, - project: -> { current_job.project } - ) - end - - def track_ci_minutes_usage!(_build, _runner) - # noop: overridden in EE - end - - private - - def get_runner_config_from_request - { config: attributes_for_keys(%w(gpus), params.dig('info', 'config')) } - end - end - end -end diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index a06b052847d..d740c626557 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -165,9 +165,9 @@ module API # Check whether an SSH key is known to GitLab # get '/authorized_keys', feature_category: :source_code_management do - fingerprint = Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint + fingerprint = Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint_sha256 - key = Key.find_by_fingerprint(fingerprint) + key = Key.find_by_fingerprint_sha256(fingerprint) not_found!('Key') if key.nil? present key, with: Entities::SSHKey end diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 46d8c0c958d..1f437ad5bd3 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -24,6 +24,7 @@ module API requires :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'invitations-api' + optional :areas_of_focus, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Areas the inviter wants the member to focus upon' end post ":id/invitations" do params[:source] = find_source(source_type, params[:id]) @@ -54,11 +55,11 @@ module API success Entities::Member end params do - requires :email, type: String, desc: 'The email address of the invitation.' - optional :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level).' - optional :expires_at, type: DateTime, desc: 'Date string in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`).' + requires :email, type: String, desc: 'The email address of the invitation' + optional :access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :expires_at, type: DateTime, desc: 'Date string in ISO 8601 format (`YYYY-MM-DDTHH:MM:SSZ`)' end - put ":id/invitations/:email", requirements: { email: /[^\/]+/ } do + put ":id/invitations/:email", requirements: { email: %r{[^/]+} } do source = find_source(source_type, params.delete(:id)) invite_email = params[:email] authorize_admin_source!(source_type, source) @@ -87,7 +88,7 @@ module API params do requires :email, type: String, desc: 'The email address of the invitation' end - delete ":id/invitations/:email", requirements: { email: /[^\/]+/ } do + delete ":id/invitations/:email", requirements: { email: %r{[^/]+} } do source = find_source(source_type, params[:id]) invite_email = params[:email] authorize_admin_source!(source_type, source) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 54013d0e7b4..a6565f913e3 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -74,7 +74,7 @@ module API desc: 'Return issues sorted in `asc` or `desc` order.' optional :due_date, type: String, values: %w[0 overdue week month next_month_and_previous_two_weeks] << '', desc: 'Return issues that have no due date (`0`), or whose due date is this week, this month, between two weeks ago and next month, or which are overdue. Accepts: `overdue`, `week`, `month`, `next_month_and_previous_two_weeks`, `0`' - optional :issue_type, type: String, values: Issue.issue_types.keys, desc: "The type of the issue. Accepts: #{Issue.issue_types.keys.join(', ')}" + optional :issue_type, type: String, values: WorkItem::Type.base_types.keys, desc: "The type of the issue. Accepts: #{WorkItem::Type.base_types.keys.join(', ')}" use :issues_stats_params use :pagination @@ -91,7 +91,7 @@ module API optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" - optional :issue_type, type: String, values: Issue.issue_types.keys, desc: "The type of the issue. Accepts: #{Issue.issue_types.keys.join(', ')}" + optional :issue_type, type: String, values: WorkItem::Type.base_types.keys, desc: "The type of the issue. Accepts: #{WorkItem::Type.base_types.keys.join(', ')}" use :optional_issue_params_ee end diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb deleted file mode 100644 index beda4433e4f..00000000000 --- a/lib/api/job_artifacts.rb +++ /dev/null @@ -1,141 +0,0 @@ -# frozen_string_literal: true - -module API - class JobArtifacts < ::API::Base - before { authenticate_non_get! } - - feature_category :build_artifacts - - # EE::API::JobArtifacts would override the following helpers - helpers do - def authorize_download_artifacts! - authorize_read_builds! - end - end - - prepend_mod_with('API::JobArtifacts') # rubocop: disable Cop/InjectEnterpriseEditionModule - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Download the artifacts archive from a job' do - detail 'This feature was introduced in GitLab 8.10' - end - params do - requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the job' - end - route_setting :authentication, job_token_allowed: true - get ':id/jobs/artifacts/:ref_name/download', - requirements: { ref_name: /.+/ } do - authorize_download_artifacts! - - latest_build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) - authorize_read_job_artifacts!(latest_build) - - present_carrierwave_file!(latest_build.artifacts_file) - end - - desc 'Download a specific file from artifacts archive from a ref' do - detail 'This feature was introduced in GitLab 11.5' - end - params do - requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the job' - requires :artifact_path, type: String, desc: 'Artifact path' - end - route_setting :authentication, job_token_allowed: true - get ':id/jobs/artifacts/:ref_name/raw/*artifact_path', - format: false, - requirements: { ref_name: /.+/ } do - authorize_download_artifacts! - - build = user_project.latest_successful_build_for_ref!(params[:job], params[:ref_name]) - authorize_read_job_artifacts!(build) - - path = Gitlab::Ci::Build::Artifacts::Path - .new(params[:artifact_path]) - - bad_request! unless path.valid? - - send_artifacts_entry(build.artifacts_file, path) - end - - desc 'Download the artifacts archive from a job' do - detail 'This feature was introduced in GitLab 8.5' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - route_setting :authentication, job_token_allowed: true - get ':id/jobs/:job_id/artifacts' do - authorize_download_artifacts! - - build = find_build!(params[:job_id]) - authorize_read_job_artifacts!(build) - - present_carrierwave_file!(build.artifacts_file) - end - - desc 'Download a specific file from artifacts archive' do - detail 'This feature was introduced in GitLab 10.0' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - requires :artifact_path, type: String, desc: 'Artifact path' - end - route_setting :authentication, job_token_allowed: true - get ':id/jobs/:job_id/artifacts/*artifact_path', format: false do - authorize_download_artifacts! - - build = find_build!(params[:job_id]) - authorize_read_job_artifacts!(build) - - not_found! unless build.available_artifacts? - - path = Gitlab::Ci::Build::Artifacts::Path - .new(params[:artifact_path]) - - bad_request! unless path.valid? - - send_artifacts_entry(build.artifacts_file, path) - end - - desc 'Keep the artifacts to prevent them from being deleted' do - success ::API::Entities::Ci::Job - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - post ':id/jobs/:job_id/artifacts/keep' do - authorize_update_builds! - - build = find_build!(params[:job_id]) - authorize!(:update_build, build) - break not_found!(build) unless build.artifacts? - - build.keep_artifacts! - - status 200 - present build, with: ::API::Entities::Ci::Job - end - - desc 'Delete the artifacts files from a job' do - detail 'This feature was introduced in GitLab 11.9' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - delete ':id/jobs/:job_id/artifacts' do - authorize_destroy_artifacts! - build = find_build!(params[:job_id]) - authorize!(:destroy_artifacts, build) - - build.erase_erasable_artifacts! - - status :no_content - end - end - end -end diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb deleted file mode 100644 index 723a5b0fa3a..00000000000 --- a/lib/api/jobs.rb +++ /dev/null @@ -1,204 +0,0 @@ -# frozen_string_literal: true - -module API - class Jobs < ::API::Base - include PaginationParams - before { authenticate! } - - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - params do - requires :id, type: String, desc: 'The ID of a project' - end - - helpers do - params :optional_scope do - optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', - values: ::CommitStatus::AVAILABLE_STATUSES, - coerce_with: ->(scope) { - case scope - when String - [scope] - when ::Hash - scope.values - when ::Array - scope - else - ['unknown'] - end - } - end - end - - desc 'Get a projects jobs' do - success Entities::Ci::Job - end - params do - use :optional_scope - use :pagination - end - # rubocop: disable CodeReuse/ActiveRecord - get ':id/jobs', feature_category: :continuous_integration do - authorize_read_builds! - - builds = user_project.builds.order('id DESC') - builds = filter_builds(builds, params[:scope]) - - builds = builds.preload(:user, :job_artifacts_archive, :job_artifacts, :runner, :tags, pipeline: :project) - present paginate(builds), with: Entities::Ci::Job - end - # rubocop: enable CodeReuse/ActiveRecord - - desc 'Get a specific job of a project' do - success Entities::Ci::Job - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - get ':id/jobs/:job_id', feature_category: :continuous_integration do - authorize_read_builds! - - build = find_build!(params[:job_id]) - - present build, with: Entities::Ci::Job - end - - # TODO: We should use `present_disk_file!` and leave this implementation for backward compatibility (when build trace - # is saved in the DB instead of file). But before that, we need to consider how to replace the value of - # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. - desc 'Get a trace of a specific job of a project' - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - get ':id/jobs/:job_id/trace', feature_category: :continuous_integration do - authorize_read_builds! - - build = find_build!(params[:job_id]) - - authorize_read_build_trace!(build) if build - - header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" - content_type 'text/plain' - env['api.format'] = :binary - - # The trace can be nil bu body method expects a string as an argument. - trace = build.trace.raw || '' - body trace - end - - desc 'Cancel a specific job of a project' do - success Entities::Ci::Job - end - params do - requires :job_id, type: Integer, desc: 'The ID of a job' - end - post ':id/jobs/:job_id/cancel', feature_category: :continuous_integration do - authorize_update_builds! - - build = find_build!(params[:job_id]) - authorize!(:update_build, build) - - build.cancel - - present build, with: Entities::Ci::Job - end - - desc 'Retry a specific build of a project' do - success Entities::Ci::Job - end - params do - requires :job_id, type: Integer, desc: 'The ID of a build' - end - post ':id/jobs/:job_id/retry', feature_category: :continuous_integration do - authorize_update_builds! - - build = find_build!(params[:job_id]) - authorize!(:update_build, build) - break forbidden!('Job is not retryable') unless build.retryable? - - build = ::Ci::Build.retry(build, current_user) - - present build, with: Entities::Ci::Job - end - - desc 'Erase job (remove artifacts and the trace)' do - success Entities::Ci::Job - end - params do - requires :job_id, type: Integer, desc: 'The ID of a build' - end - post ':id/jobs/:job_id/erase', feature_category: :continuous_integration do - authorize_update_builds! - - build = find_build!(params[:job_id]) - authorize!(:erase_build, build) - break forbidden!('Job is not erasable!') unless build.erasable? - - build.erase(erased_by: current_user) - present build, with: Entities::Ci::Job - end - - desc 'Trigger an actionable job (manual, delayed, etc)' do - success Entities::Ci::JobBasic - detail 'This feature was added in GitLab 8.11' - end - params do - requires :job_id, type: Integer, desc: 'The ID of a Job' - end - - post ":id/jobs/:job_id/play", feature_category: :continuous_integration do - authorize_read_builds! - - job = find_job!(params[:job_id]) - - authorize!(:play_job, job) - - bad_request!("Unplayable Job") unless job.playable? - - job.play(current_user) - - status 200 - - if job.is_a?(::Ci::Build) - present job, with: Entities::Ci::Job - else - present job, with: Entities::Ci::Bridge - end - end - end - - resource :job do - desc 'Get current project using job token' do - success Entities::Ci::Job - end - route_setting :authentication, job_token_allowed: true - get '', feature_category: :continuous_integration do - validate_current_authenticated_job - - present current_authenticated_job, with: Entities::Ci::Job - end - end - - helpers do - # rubocop: disable CodeReuse/ActiveRecord - def filter_builds(builds, scope) - return builds if scope.nil? || scope.empty? - - available_statuses = ::CommitStatus::AVAILABLE_STATUSES - - unknown = scope - available_statuses - render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? - - builds.where(status: available_statuses && scope) - end - # rubocop: enable CodeReuse/ActiveRecord - - def validate_current_authenticated_job - # current_authenticated_job will be nil if user is using - # a valid authentication (like PRIVATE-TOKEN) that is not CI_JOB_TOKEN - not_found!('Job') unless current_authenticated_job - end - end - end -end - -API::Jobs.prepend_mod_with('API::Jobs') diff --git a/lib/api/members.rb b/lib/api/members.rb index 70e13e8d4ae..7130635281a 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -94,6 +94,7 @@ module API requires :user_id, types: [Integer, String], desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'members-api' + optional :areas_of_focus, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Areas the inviter wants the member to focus upon' end # rubocop: disable CodeReuse/ActiveRecord post ":id/members" do @@ -119,7 +120,12 @@ module API not_allowed! # This currently can only be reached in EE elsif member.valid? && member.persisted? present_members(member) - Gitlab::Tracking.event(::Members::CreateService.name, 'create_member', label: params[:invite_source], property: 'existing_user', user: current_user) + Gitlab::Tracking.event(::Members::CreateService.name, + 'create_member', + label: params[:invite_source], + property: 'existing_user', + user: current_user) + track_areas_of_focus(member, params[:areas_of_focus]) else render_validation_error!(member) end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index a9617482557..7ab57982907 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -404,6 +404,7 @@ module API pipeline = ::MergeRequests::CreatePipelineService .new(project: user_project, current_user: current_user, params: { allow_duplicate: true }) .execute(find_merge_request_with_access(params[:merge_request_iid])) + .payload if pipeline.nil? not_allowed! diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 9d41c2f148f..c2d839571a6 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -27,12 +27,15 @@ module API end params do optional :search, type: String, desc: "Search query for namespaces" + optional :owned_only, type: Boolean, desc: "Owned namespaces only" use :pagination use :optional_list_params_ee end get do - namespaces = current_user.admin ? Namespace.all : current_user.namespaces + owned_only = params[:owned_only] == true + + namespaces = current_user.admin ? Namespace.all : current_user.namespaces(owned_only: owned_only) namespaces = namespaces.include_route diff --git a/lib/api/project_debian_distributions.rb b/lib/api/project_debian_distributions.rb index 58edf51f4f7..f057251fb6b 100644 --- a/lib/api/project_debian_distributions.rb +++ b/lib/api/project_debian_distributions.rb @@ -19,8 +19,6 @@ module API require_packages_enabled! not_found! unless ::Feature.enabled?(:debian_packages, user_project) - - authorize_read_package! end namespace ':id' do diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb index acf9bfece65..fe0e837c596 100644 --- a/lib/api/project_templates.rb +++ b/lib/api/project_templates.rb @@ -12,7 +12,7 @@ module API before { authenticate_non_get! } - feature_category :templates + feature_category :source_code_management params do requires :id, type: String, desc: 'The ID of a project' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3b1d239398f..28bcb382ecf 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -45,6 +45,20 @@ module API end end + def support_order_by_similarity!(attrs) + return unless params[:order_by] == 'similarity' + + if order_by_similarity?(allow_unauthorized: false) + # Limit to projects the current user is a member of. + # Do not include all public projects because it + # could cause long running queries + attrs[:non_public] = true + attrs[:sort] = params['order_by'] + else + params[:order_by] = route.params['order_by'][:default] + end + end + def delete_project(user_project) destroy_conditionally!(user_project) do ::Projects::DestroyService.new(user_project, current_user, {}).async_execute @@ -93,8 +107,8 @@ module API params :sort_params do optional :order_by, type: String, - values: %w[id name path created_at updated_at last_activity_at] + Helpers::ProjectsHelpers::STATISTICS_SORT_PARAMS, - default: 'created_at', desc: "Return projects ordered by field. #{Helpers::ProjectsHelpers::STATISTICS_SORT_PARAMS.join(', ')} are only available to admins." + values: %w[id name path created_at updated_at last_activity_at similarity] + Helpers::ProjectsHelpers::STATISTICS_SORT_PARAMS, + default: 'created_at', desc: "Return projects ordered by field. #{Helpers::ProjectsHelpers::STATISTICS_SORT_PARAMS.join(', ')} are only available to admins. Similarity is available when searching and is limited to projects the user has access to." optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return projects sorted in ascending and descending order' end @@ -131,16 +145,17 @@ module API end def load_projects - params = project_finder_params - verify_project_filters!(params) + project_params = project_finder_params + support_order_by_similarity!(project_params) + verify_project_filters!(project_params) - ProjectsFinder.new(current_user: current_user, params: params).execute + ProjectsFinder.new(current_user: current_user, params: project_params).execute end def present_projects(projects, options = {}) verify_statistics_order_by_projects! - projects = reorder_projects(projects) + projects = reorder_projects(projects) unless order_by_similarity?(allow_unauthorized: false) projects = apply_filters(projects) records, options = paginate_with_strategies(projects, options[:request_scope]) do |projects| @@ -572,6 +587,27 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc 'Import members from another project' do + detail 'This feature was introduced in GitLab 14.2' + end + params do + requires :project_id, type: Integer, desc: 'The ID of the source project to import the members from.' + end + post ":id/import_project_members/:project_id", feature_category: :experimentation_expansion do + authorize! :admin_project, user_project + + source_project = Project.find_by_id(params[:project_id]) + not_found!('Project') unless source_project && can?(current_user, :read_project, source_project) + + result = ::Members::ImportProjectTeamService.new(current_user, params).execute + + if result + { status: result, message: 'Successfully imported' } + else + render_api_error!('Import failed', :unprocessable_entity) + end + end + desc 'Workhorse authorize the file upload' do detail 'This feature was introduced in GitLab 13.11' end diff --git a/lib/api/pypi_packages.rb b/lib/api/pypi_packages.rb index 7c5f8bb4d99..706c0702fce 100644 --- a/lib/api/pypi_packages.rb +++ b/lib/api/pypi_packages.rb @@ -10,6 +10,7 @@ module API helpers ::API::Helpers::PackagesManagerClientsHelpers helpers ::API::Helpers::RelatedResourcesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers + helpers ::API::Helpers::Packages::DependencyProxyHelpers include ::API::Helpers::Packages::BasicAuthHelpers::Constants feature_category :package_registry @@ -40,7 +41,7 @@ module API end params do - requires :id, type: Integer, desc: 'The ID of a group' + requires :id, type: String, desc: 'The ID of a group' end resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do after_validation do @@ -82,21 +83,26 @@ module API track_package_event('list_package', :pypi) - packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute! - presenter = ::Packages::Pypi::PackagePresenter.new(packages, group) + packages = Packages::Pypi::PackagesFinder.new(current_user, group, { package_name: params[:package_name] }).execute + empty_packages = packages.empty? - # Adjusts grape output format - # to be HTML - content_type "text/html; charset=utf-8" - env['api.format'] = :binary + redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do + not_found!('Package') if empty_packages + presenter = ::Packages::Pypi::PackagePresenter.new(packages, group) - body presenter.body + # Adjusts grape output format + # to be HTML + content_type "text/html; charset=utf-8" + env['api.format'] = :binary + + body presenter.body + end end end end params do - requires :id, type: Integer, desc: 'The ID of a project' + requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do @@ -142,15 +148,20 @@ module API track_package_event('list_package', :pypi, project: authorized_user_project, namespace: authorized_user_project.namespace) - packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute! - presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project) + packages = Packages::Pypi::PackagesFinder.new(current_user, authorized_user_project, { package_name: params[:package_name] }).execute + empty_packages = packages.empty? + + redirect_registry_request(empty_packages, :pypi, package_name: params[:package_name]) do + not_found!('Package') if empty_packages + presenter = ::Packages::Pypi::PackagePresenter.new(packages, authorized_user_project) - # Adjusts grape output format - # to be HTML - content_type "text/html; charset=utf-8" - env['api.format'] = :binary + # Adjusts grape output format + # to be HTML + content_type "text/html; charset=utf-8" + env['api.format'] = :binary - body presenter.body + body presenter.body + end end desc 'The PyPi Package upload endpoint' do diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index f274406e225..20320d1b7ae 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -29,14 +29,13 @@ module API not_found! end - def assign_blob_vars! + def assign_blob_vars!(limit:) authorize! :download_code, user_project @repo = user_project.repository begin - @blob = Gitlab::Git::Blob.raw(@repo, params[:sha]) - @blob.load_all_data!(@repo) + @blob = Gitlab::Git::Blob.raw(@repo, params[:sha], limit: limit) rescue StandardError not_found! 'Blob' end @@ -55,7 +54,7 @@ module API use :pagination end get ':id/repository/tree' do - ref = params[:ref] || user_project.try(:default_branch) || 'master' + ref = params[:ref] || user_project.default_branch path = params[:path] || nil commit = user_project.commit(ref) @@ -71,7 +70,8 @@ module API requires :sha, type: String, desc: 'The commit hash' end get ':id/repository/blobs/:sha/raw' do - assign_blob_vars! + # Load metadata enough to ask Workhorse to load the whole blob + assign_blob_vars!(limit: 0) no_cache_headers @@ -83,7 +83,7 @@ module API requires :sha, type: String, desc: 'The commit hash' end get ':id/repository/blobs/:sha' do - assign_blob_vars! + assign_blob_vars!(limit: -1) { size: @blob.size, diff --git a/lib/api/rubygem_packages.rb b/lib/api/rubygem_packages.rb index d7f9c584c67..9ef6ec03a41 100644 --- a/lib/api/rubygem_packages.rb +++ b/lib/api/rubygem_packages.rb @@ -101,7 +101,7 @@ module API package_file = nil - ActiveRecord::Base.transaction do + ApplicationRecord.transaction do package = ::Packages::CreateTemporaryPackageService.new( user_project, current_user, declared_params.merge(build: current_authenticated_job) ).execute(:rubygems, name: ::Packages::Rubygems::TEMPORARY_PACKAGE_NAME) diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 952bf09b1b1..aac195f0668 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -48,7 +48,7 @@ module API optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :default_ci_config_path, type: String, desc: 'The instance default CI/CD configuration file and path for new projects' optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group' - optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to master' + optional :default_branch_protection, type: Integer, values: ::Gitlab::Access.protection_values, desc: 'Determine if developers can push to default branch' optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 6c8e2c69a6d..395aacced78 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -59,8 +59,6 @@ module API optional :message, type: String, desc: 'Specifying a message creates an annotated tag' end post ':id/repository/tags', :release_orchestration do - deprecate_release_notes unless params[:release_description].blank? - authorize_admin_tag result = ::Tags::CreateService.new(user_project, current_user) diff --git a/lib/api/templates.rb b/lib/api/templates.rb index b7fb35eac03..a595129fd6a 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -4,17 +4,18 @@ module API class Templates < ::API::Base include PaginationParams - feature_category :templates - GLOBAL_TEMPLATE_TYPES = { gitignores: { - gitlab_version: 8.8 + gitlab_version: 8.8, + feature_category: :source_code_management }, gitlab_ci_ymls: { - gitlab_version: 8.9 + gitlab_version: 8.9, + feature_category: :continuous_integration }, dockerfiles: { - gitlab_version: 8.15 + gitlab_version: 8.15, + feature_category: :source_code_management } }.freeze @@ -33,7 +34,7 @@ module API optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' use :pagination end - get "templates/licenses" do + get "templates/licenses", feature_category: :source_code_management do popular = declared(params)[:popular] popular = to_boolean(popular) if popular.present? @@ -49,7 +50,7 @@ module API params do requires :name, type: String, desc: 'The name of the template' end - get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do + get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ }, feature_category: :source_code_management do template = TemplateFinder.build(:licenses, nil, name: params[:name]).execute not_found!('License') unless template.present? @@ -72,7 +73,7 @@ module API params do use :pagination end - get "templates/#{template_type}" do + get "templates/#{template_type}", feature_category: properties[:feature_category] do templates = ::Kaminari.paginate_array(TemplateFinder.build(template_type, nil).execute) present paginate(templates), with: Entities::TemplatesList end @@ -84,7 +85,7 @@ module API params do requires :name, type: String, desc: 'The name of the template' end - get "templates/#{template_type}/:name", requirements: { name: /[\w\.-]+/ } do + get "templates/#{template_type}/:name", requirements: { name: /[\w\.-]+/ }, feature_category: properties[:feature_category] do finder = TemplateFinder.build(template_type, nil, name: declared(params)[:name]) new_template = finder.execute diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 969122d7906..b8323304957 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -88,6 +88,7 @@ module API update_params = { spend_time: { duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)), + summary: params.delete(:summary), user_id: current_user.id } } diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb deleted file mode 100644 index a359083a9d2..00000000000 --- a/lib/api/triggers.rb +++ /dev/null @@ -1,146 +0,0 @@ -# frozen_string_literal: true - -module API - class Triggers < ::API::Base - include PaginationParams - - HTTP_GITLAB_EVENT_HEADER = "HTTP_#{WebHookService::GITLAB_EVENT_HEADER}".underscore.upcase - - feature_category :continuous_integration - - params do - requires :id, type: String, desc: 'The ID of a project' - end - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Trigger a GitLab project pipeline' do - success Entities::Ci::Pipeline - end - params do - requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false - requires :token, type: String, desc: 'The unique token of trigger or job token' - optional :variables, type: Hash, desc: 'The list of variables to be injected into build' - end - post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do - Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/20758') - - forbidden! if gitlab_pipeline_hook_request? - - # validate variables - params[:variables] = params[:variables].to_h - unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - project = find_project(params[:id]) - not_found! unless project - - result = ::Ci::PipelineTriggerService.new(project, nil, params).execute - not_found! unless result - - if result.error? - render_api_error!(result[:message], result[:http_status]) - else - present result[:pipeline], with: Entities::Ci::Pipeline - end - end - - desc 'Get triggers list' do - success Entities::Trigger - end - params do - use :pagination - end - # rubocop: disable CodeReuse/ActiveRecord - get ':id/triggers' do - authenticate! - authorize! :admin_build, user_project - - triggers = user_project.triggers.includes(:trigger_requests) - - present paginate(triggers), with: Entities::Trigger, current_user: current_user - end - # rubocop: enable CodeReuse/ActiveRecord - - desc 'Get specific trigger of a project' do - success Entities::Trigger - end - params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' - end - get ':id/triggers/:trigger_id' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.find(params.delete(:trigger_id)) - break not_found!('Trigger') unless trigger - - present trigger, with: Entities::Trigger, current_user: current_user - end - - desc 'Create a trigger' do - success Entities::Trigger - end - params do - requires :description, type: String, desc: 'The trigger description' - end - post ':id/triggers' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.create( - declared_params(include_missing: false).merge(owner: current_user)) - - if trigger.valid? - present trigger, with: Entities::Trigger, current_user: current_user - else - render_validation_error!(trigger) - end - end - - desc 'Update a trigger' do - success Entities::Trigger - end - params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' - optional :description, type: String, desc: 'The trigger description' - end - put ':id/triggers/:trigger_id' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.find(params.delete(:trigger_id)) - break not_found!('Trigger') unless trigger - - authorize! :admin_trigger, trigger - - if trigger.update(declared_params(include_missing: false)) - present trigger, with: Entities::Trigger, current_user: current_user - else - render_validation_error!(trigger) - end - end - - desc 'Delete a trigger' do - success Entities::Trigger - end - params do - requires :trigger_id, type: Integer, desc: 'The trigger ID' - end - delete ':id/triggers/:trigger_id' do - authenticate! - authorize! :admin_build, user_project - - trigger = user_project.triggers.find(params.delete(:trigger_id)) - break not_found!('Trigger') unless trigger - - destroy_conditionally!(trigger) - end - end - - helpers do - def gitlab_pipeline_hook_request? - request.get_header(HTTP_GITLAB_EVENT_HEADER) == WebHookService.hook_to_event(:pipeline_hooks) - end - end - end -end diff --git a/lib/api/user_counts.rb b/lib/api/user_counts.rb index 31c923a219a..634dd0f2179 100644 --- a/lib/api/user_counts.rb +++ b/lib/api/user_counts.rb @@ -6,15 +6,17 @@ module API resource :user_counts do desc 'Return the user specific counts' do - detail 'Open MR Count' + detail 'Assigned open issues, assigned MRs and pending todos count' end get do unauthorized! unless current_user { merge_requests: current_user.assigned_open_merge_requests_count, # @deprecated + assigned_issues: current_user.assigned_open_issues_count, assigned_merge_requests: current_user.assigned_open_merge_requests_count, - review_requested_merge_requests: current_user.review_requested_open_merge_requests_count + review_requested_merge_requests: current_user.review_requested_open_merge_requests_count, + todos: current_user.todos_pending_count } end end diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 29e4a79110f..310054c298a 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -214,6 +214,8 @@ module API update_project_feature_usage_for(user_project) + next [] unless user_project.repo_exists? + branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name)) present paginate(branches), with: ::API::Github::Entities::Branch, project: user_project diff --git a/lib/api/variables.rb b/lib/api/variables.rb deleted file mode 100644 index 75df0e050a6..00000000000 --- a/lib/api/variables.rb +++ /dev/null @@ -1,124 +0,0 @@ -# frozen_string_literal: true - -module API - class Variables < ::API::Base - include PaginationParams - - before { authenticate! } - before { authorize! :admin_build, user_project } - - feature_category :pipeline_authoring - - helpers Helpers::VariablesHelpers - - params do - requires :id, type: String, desc: 'The ID of a project' - end - - resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do - desc 'Get project variables' do - success Entities::Ci::Variable - end - params do - use :pagination - end - get ':id/variables' do - variables = user_project.variables - present paginate(variables), with: Entities::Ci::Variable - end - - desc 'Get a specific variable from a project' do - success Entities::Ci::Variable - end - params do - requires :key, type: String, desc: 'The key of the variable' - end - # rubocop: disable CodeReuse/ActiveRecord - get ':id/variables/:key' do - variable = find_variable(user_project, params) - not_found!('Variable') unless variable - - present variable, with: Entities::Ci::Variable - end - # rubocop: enable CodeReuse/ActiveRecord - - desc 'Create a new variable in a project' do - success Entities::Ci::Variable - end - params do - requires :key, type: String, desc: 'The key of the variable' - requires :value, type: String, desc: 'The value of the variable' - optional :protected, type: Boolean, desc: 'Whether the variable is protected' - optional :masked, type: Boolean, desc: 'Whether the variable is masked' - optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file. Defaults to env_var' - optional :environment_scope, type: String, desc: 'The environment_scope of the variable' - end - post ':id/variables' do - variable = ::Ci::ChangeVariableService.new( - container: user_project, - current_user: current_user, - params: { action: :create, variable_params: declared_params(include_missing: false) } - ).execute - - if variable.valid? - present variable, with: Entities::Ci::Variable - else - render_validation_error!(variable) - end - end - - desc 'Update an existing variable from a project' do - success Entities::Ci::Variable - end - params do - optional :key, type: String, desc: 'The key of the variable' - optional :value, type: String, desc: 'The value of the variable' - optional :protected, type: Boolean, desc: 'Whether the variable is protected' - optional :masked, type: Boolean, desc: 'Whether the variable is masked' - optional :variable_type, type: String, values: ::Ci::Variable.variable_types.keys, desc: 'The type of variable, must be one of env_var or file' - optional :environment_scope, type: String, desc: 'The environment_scope of the variable' - optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' - end - # rubocop: disable CodeReuse/ActiveRecord - put ':id/variables/:key' do - variable = find_variable(user_project, params) - not_found!('Variable') unless variable - - variable = ::Ci::ChangeVariableService.new( - container: user_project, - current_user: current_user, - params: { action: :update, variable: variable, variable_params: declared_params(include_missing: false).except(:key, :filter) } - ).execute - - if variable.valid? - present variable, with: Entities::Ci::Variable - else - render_validation_error!(variable) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - desc 'Delete an existing variable from a project' do - success Entities::Ci::Variable - end - params do - requires :key, type: String, desc: 'The key of the variable' - optional :filter, type: Hash, desc: 'Available filters: [environment_scope]. Example: filter[environment_scope]=production' - end - # rubocop: disable CodeReuse/ActiveRecord - delete ':id/variables/:key' do - variable = find_variable(user_project, params) - not_found!('Variable') unless variable - - ::Ci::ChangeVariableService.new( - container: user_project, - current_user: current_user, - params: { action: :destroy, variable: variable } - ).execute - - no_content! - end - # rubocop: enable CodeReuse/ActiveRecord - end - end -end diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index ea83076c49b..3e2e6f1b9ba 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -81,7 +81,7 @@ module Atlassian end def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil) - repo = Serializers::RepositoryEntity.represent( + repo = ::Atlassian::JiraConnect::Serializers::RepositoryEntity.represent( project, commits: commits, branches: branches, diff --git a/lib/backup.rb b/lib/backup.rb index 2712b33b4b4..91682645a9a 100644 --- a/lib/backup.rb +++ b/lib/backup.rb @@ -2,4 +2,43 @@ module Backup Error = Class.new(StandardError) + + class FileBackupError < Backup::Error + attr_reader :app_files_dir, :backup_tarball + + def initialize(app_files_dir, backup_tarball) + @app_files_dir = app_files_dir + @backup_tarball = backup_tarball + end + + def message + "Failed to create compressed file '#{backup_tarball}' when trying to backup the following paths: '#{app_files_dir}'" + end + end + + class RepositoryBackupError < Backup::Error + attr_reader :container, :backup_repos_path + + def initialize(container, backup_repos_path) + @container = container + @backup_repos_path = backup_repos_path + end + + def message + "Failed to create compressed file '#{backup_repos_path}' when trying to backup the following paths: '#{container.disk_path}'" + end + end + + class DatabaseBackupError < Backup::Error + attr_reader :config, :db_file_name + + def initialize(config, db_file_name) + @config = config + @db_file_name = db_file_name + end + + def message + "Failed to create compressed file '#{db_file_name}' when trying to backup the main database:\n - host: '#{config[:host]}'\n - port: '#{config[:port]}'\n - database: '#{config[:database]}'" + end + end end diff --git a/lib/backup/gitaly_backup.rb b/lib/backup/gitaly_backup.rb index c15b0ed6a1b..55fd68fd6e8 100644 --- a/lib/backup/gitaly_backup.rb +++ b/lib/backup/gitaly_backup.rb @@ -25,18 +25,21 @@ module Backup args += ['-parallel', @parallel.to_s] if type == :create && @parallel args += ['-parallel-storage', @parallel_storage.to_s] if type == :create && @parallel_storage - @read_io, @write_io = IO.pipe - @pid = Process.spawn(bin_path, command, '-path', backup_repos_path, *args, in: @read_io, out: @progress) + @stdin, stdout, @thread = Open3.popen2(ENV, bin_path, command, '-path', backup_repos_path, *args) + + @out_reader = Thread.new do + IO.copy_stream(stdout, @progress) + end end def wait return unless started? - @write_io.close - Process.wait(@pid) - status = $? + @stdin.close + [@thread, @out_reader].each(&:join) + status = @thread.value - @pid = nil + @thread = nil raise Error, "gitaly-backup exit status #{status.exitstatus}" if status.exitstatus != 0 end @@ -46,7 +49,7 @@ module Backup repository = repo_type.repository_for(container) - @write_io.puts({ + @stdin.puts({ storage_name: repository.storage, relative_path: repository.relative_path, gl_project_path: repository.gl_project_path, @@ -61,7 +64,7 @@ module Backup private def started? - @pid.present? + @thread.present? end def backup_repos_path diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 522a034a283..52810b0fb35 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -72,6 +72,17 @@ module Backup end end + def remove_tmp + # delete tmp inside backups + progress.print "Deleting backups/tmp ... " + + if FileUtils.rm_rf(File.join(backup_path, "tmp")) + progress.puts "done".color(:green) + else + puts "deleting backups/tmp failed".color(:red) + end + end + def remove_old # delete backups progress.print "Deleting old backups ... " @@ -232,7 +243,7 @@ module Backup end def folders_to_backup - FOLDERS_TO_BACKUP.reject { |name| skipped?(name) } + FOLDERS_TO_BACKUP.select { |name| !skipped?(name) && Dir.exist?(File.join(backup_path, name)) } end def disabled_features diff --git a/lib/banzai/filter/references/reference_cache.rb b/lib/banzai/filter/references/reference_cache.rb index 24b8b4984cd..816ce973cad 100644 --- a/lib/banzai/filter/references/reference_cache.rb +++ b/lib/banzai/filter/references/reference_cache.rb @@ -28,20 +28,11 @@ module Banzai @references_per_parent[parent_type] ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - nodes.each do |node| - prepare_node_for_scan(node).scan(regex) do - parent_path = if parent_type == :project - full_project_path($~[:namespace], $~[:project]) - else - full_group_path($~[:group]) - end - - ident = filter.identifier($~) - refs[parent_path] << ident if ident - end + if Feature.enabled?(:milestone_reference_pattern, default_enabled: :yaml) + doc_search(refs) + else + node_search(nodes, refs) end - - refs end end @@ -172,6 +163,39 @@ module Banzai delegate :project, :group, :parent, :parent_type, to: :filter + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/336268 + def node_search(nodes, refs) + nodes.each do |node| + prepare_node_for_scan(node).scan(regex) do + parent_path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + + ident = filter.identifier($~) + refs[parent_path] << ident if ident + end + end + + refs + end + + def doc_search(refs) + prepare_doc_for_scan(filter.doc).to_enum(:scan, regex).each do + parent_path = if parent_type == :project + full_project_path($~[:namespace], $~[:project]) + else + full_group_path($~[:group]) + end + + ident = filter.identifier($~) + refs[parent_path] << ident if ident + end + + refs + end + def regex strong_memoize(:regex) do [ @@ -185,6 +209,13 @@ module Banzai Gitlab::SafeRequestStore["banzai_#{parent_type}_refs".to_sym] ||= {} end + def prepare_doc_for_scan(doc) + html = doc.to_html + + filter.requires_unescaping? ? unescape_html_entities(html) : html + end + + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/336268 def prepare_node_for_scan(node) html = node.to_html diff --git a/lib/banzai/filter/table_of_contents_tag_filter.rb b/lib/banzai/filter/table_of_contents_tag_filter.rb index 13d0a6a4cc7..4e80b543e2d 100644 --- a/lib/banzai/filter/table_of_contents_tag_filter.rb +++ b/lib/banzai/filter/table_of_contents_tag_filter.rb @@ -2,26 +2,31 @@ module Banzai module Filter - # Using `[[_TOC_]]`, inserts a Table of Contents list. - # This syntax is based on the Gollum syntax. This way we have - # some consistency between with wiki and normal markdown. - # If there ever emerges a markdown standard, we can implement - # that here. + # Using `[[_TOC_]]` or `[TOC]` (both case insensitive), inserts a Table of Contents list. # + # `[[_TOC_]]` is based on the Gollum syntax. This way we have + # some consistency between with wiki and normal markdown. # The support for this has been removed from GollumTagsFilter # + # `[toc]` is a generally accepted form, used by Typora for example. + # # Based on Banzai::Filter::GollumTagsFilter class TableOfContentsTagFilter < HTML::Pipeline::Filter - TEXT_QUERY = %q(descendant-or-self::text()[ancestor::p and contains(., 'TOC')]) + TEXT_QUERY = %q(descendant-or-self::text()[ancestor::p and contains(translate(., 'TOC', 'toc'), 'toc')]) def call return doc if context[:no_header_anchors] doc.xpath(TEXT_QUERY).each do |node| - # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running - # before this one, it will be converted into `[[TOC]]`, so it - # needs special-case handling - process_toc_tag(node) if toc_tag?(node) + if toc_tag?(node) + # Support [TOC] / [toc] tags, which don't have a wrapping -tag + process_toc_tag(node) + elsif toc_tag_em?(node) + # Support Gollum like ToC tag (`[[_TOC_]]` / `[[_toc_]]`), which will be converted + # into `[[TOC]]` by the markdown filter, so it + # needs special-case handling + process_toc_tag_em(node) + end end doc @@ -31,14 +36,25 @@ module Banzai # Replace an entire `[[TOC]]` node with the result generated by # TableOfContentsFilter + def process_toc_tag_em(node) + process_toc_tag(node.parent) + end + + # Replace an entire `[TOC]` node with the result generated by + # TableOfContentsFilter def process_toc_tag(node) - node.parent.parent.replace(result[:toc].presence || '') + # we still need to go one step up to also replace the surrounding

+ node.parent.replace(result[:toc].presence || '') end - def toc_tag?(node) - node.content == 'TOC' && + def toc_tag_em?(node) + node.content.casecmp?('toc') && node.parent.name == 'em' && - node.parent.parent.text == '[[TOC]]' + node.parent.parent.text.casecmp?('[[toc]]') + end + + def toc_tag?(node) + node.parent.text.casecmp?('[toc]') end end end diff --git a/lib/error_tracking/collector/sentry_auth_parser.rb b/lib/error_tracking/collector/sentry_auth_parser.rb new file mode 100644 index 00000000000..4945b8f73e1 --- /dev/null +++ b/lib/error_tracking/collector/sentry_auth_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module ErrorTracking + module Collector + class SentryAuthParser + def self.parse(request) + # Sentry client sends auth in X-Sentry-Auth header + # + # Example of content: + # "Sentry sentry_version=7, sentry_client=sentry-ruby/4.5.1, sentry_timestamp=1623923398, + # sentry_key=afadk312..., sentry_secret=123456asd32131..." + auth = request.headers['X-Sentry-Auth'] + + # Sentry DSN contains key and secret. + # The key is required while secret is optional. + # We are going to use only the key since secret is deprecated. + public_key = auth[/sentry_key=(\w+)/, 1] + + { + public_key: public_key + } + end + end + end +end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 055a3a771c2..8f6576c2206 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -47,16 +47,6 @@ module ExtractsPath end # rubocop:enable Gitlab/ModuleWithInstanceVariables - def lfs_blob_ids - blob_ids = tree.blobs.map(&:id) - - # When current endpoint is a Blob then `tree.blobs` will be empty, it means we need to analyze - # the current Blob in order to determine if it's a LFS object - blob_ids = Array.wrap(@repo.blob_at(@commit.id, @path)&.id) if blob_ids.empty? # rubocop:disable Gitlab/ModuleWithInstanceVariables - - @lfs_blob_ids = Gitlab::Git::Blob.batch_lfs_pointers(repository_container.repository, blob_ids).map(&:id) # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - private # Override in controllers to determine which actions are subject to the redirect diff --git a/lib/feature.rb b/lib/feature.rb index 453ecc8255a..f8d34e9c386 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -36,7 +36,7 @@ class Feature end def persisted_names - return [] unless Gitlab::Database.exists? + return [] unless Gitlab::Database.main.exists? # This loads names of all stored feature flags # and returns a stable Set in the following order: @@ -56,7 +56,7 @@ class Feature # use `default_enabled: true` to default the flag to being `enabled` # unless set explicitly. The default is `disabled` - # TODO: remove the `default_enabled:` and read it from the `defintion_yaml` + # TODO: remove the `default_enabled:` and read it from the `definition_yaml` # check: https://gitlab.com/gitlab-org/gitlab/-/issues/30228 def enabled?(key, thing = nil, type: :development, default_enabled: false) if check_feature_flags_definition? @@ -73,7 +73,7 @@ class Feature # During setup the database does not exist yet. So we haven't stored a value # for the feature yet and return the default. - return default_enabled unless Gitlab::Database.exists? + return default_enabled unless Gitlab::Database.main.exists? feature = get(key) diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index e603a1dc8d2..a061a83e79c 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -15,7 +15,7 @@ class Feature def server_feature_flags(project = nil) # We need to check that both the DB connection and table exists - return {} unless ::Gitlab::Database.cached_table_exists?(FlipperFeature.table_name) + return {} unless ::Gitlab::Database.main.cached_table_exists?(FlipperFeature.table_name) Feature.persisted_names .select { |f| f.start_with?(PREFIX) } diff --git a/lib/gem_extensions/active_record/association.rb b/lib/gem_extensions/active_record/association.rb new file mode 100644 index 00000000000..91a9f45ce7e --- /dev/null +++ b/lib/gem_extensions/active_record/association.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module Association + extend ActiveSupport::Concern + + attr_reader :disable_joins + + def initialize(owner, reflection) + super + + @disable_joins = @reflection.options[:disable_joins] || false + end + + def scope + if disable_joins + DisableJoins::Associations::AssociationScope.create.scope(self) + else + super + end + end + + def association_scope + if klass + @association_scope ||= begin # rubocop:disable Gitlab/ModuleWithInstanceVariables + if disable_joins + DisableJoins::Associations::AssociationScope.scope(self) + else + super + end + end + end + end + end + end +end diff --git a/lib/gem_extensions/active_record/associations/builder/has_many.rb b/lib/gem_extensions/active_record/associations/builder/has_many.rb new file mode 100644 index 00000000000..7e51e632cc3 --- /dev/null +++ b/lib/gem_extensions/active_record/associations/builder/has_many.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module Associations + module Builder + module HasMany + extend ActiveSupport::Concern + + class_methods do + def valid_options(options) + valid = super + valid += [:disable_joins] if options[:disable_joins] && options[:through] + valid + end + end + end + end + end + end +end diff --git a/lib/gem_extensions/active_record/associations/builder/has_one.rb b/lib/gem_extensions/active_record/associations/builder/has_one.rb new file mode 100644 index 00000000000..91765db8a5a --- /dev/null +++ b/lib/gem_extensions/active_record/associations/builder/has_one.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module Associations + module Builder + module HasOne + extend ActiveSupport::Concern + + class_methods do + def valid_options(options) + valid = super + valid += [:disable_joins] if options[:disable_joins] && options[:through] + valid + end + end + end + end + end + end +end diff --git a/lib/gem_extensions/active_record/associations/has_many_through_association.rb b/lib/gem_extensions/active_record/associations/has_many_through_association.rb new file mode 100644 index 00000000000..e7051e4d9cb --- /dev/null +++ b/lib/gem_extensions/active_record/associations/has_many_through_association.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module Associations + module HasManyThroughAssociation + extend ActiveSupport::Concern + + def find_target + return [] unless target_reflection_has_associated_record? + return scope.to_a if disable_joins + + super + end + end + end + end +end diff --git a/lib/gem_extensions/active_record/associations/has_one_through_association.rb b/lib/gem_extensions/active_record/associations/has_one_through_association.rb new file mode 100644 index 00000000000..1487392a4ea --- /dev/null +++ b/lib/gem_extensions/active_record/associations/has_one_through_association.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module Associations + module HasOneThroughAssociation + extend ActiveSupport::Concern + + def find_target + return scope.first if disable_joins + + super + end + end + end + end +end diff --git a/lib/gem_extensions/active_record/associations/preloader/through_association.rb b/lib/gem_extensions/active_record/associations/preloader/through_association.rb new file mode 100644 index 00000000000..16b53846a58 --- /dev/null +++ b/lib/gem_extensions/active_record/associations/preloader/through_association.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module Associations + module Preloader + module ThroughAssociation + extend ActiveSupport::Concern + + def through_scope + scope = through_reflection.klass.unscoped + options = reflection.options + + return scope if options[:disable_joins] + + super + end + end + end + end + end +end diff --git a/lib/gem_extensions/active_record/configurable_disable_joins.rb b/lib/gem_extensions/active_record/configurable_disable_joins.rb new file mode 100644 index 00000000000..8e4c6bd6fc5 --- /dev/null +++ b/lib/gem_extensions/active_record/configurable_disable_joins.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module ConfigurableDisableJoins + extend ActiveSupport::Concern + + def disable_joins + # rubocop:disable Gitlab/ModuleWithInstanceVariables + return @disable_joins.call if @disable_joins.is_a?(Proc) + + @disable_joins + # rubocop:enable Gitlab/ModuleWithInstanceVariables + end + end + end +end diff --git a/lib/gem_extensions/active_record/delegate_cache.rb b/lib/gem_extensions/active_record/delegate_cache.rb new file mode 100644 index 00000000000..63c93f7a2d3 --- /dev/null +++ b/lib/gem_extensions/active_record/delegate_cache.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module DelegateCache + def relation_delegate_class(klass) + @relation_delegate_cache2[klass] || super # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def initialize_relation_delegate_cache_disable_joins + @relation_delegate_cache2 = {} # rubocop:disable Gitlab/ModuleWithInstanceVariables + + [ + ::GemExtensions::ActiveRecord::DisableJoins::Relation + ].each do |klass| + delegate = Class.new(klass) do + include ::ActiveRecord::Delegation::ClassSpecificRelation + end + include_relation_methods(delegate) + mangled_name = klass.name.gsub("::", "_") + const_set mangled_name, delegate + private_constant mangled_name + + @relation_delegate_cache2[klass] = delegate # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + + def inherited(child_class) + child_class.initialize_relation_delegate_cache_disable_joins + super + end + end + end +end diff --git a/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb b/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb new file mode 100644 index 00000000000..1e4476330a2 --- /dev/null +++ b/lib/gem_extensions/active_record/disable_joins/associations/association_scope.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module DisableJoins + module Associations + class AssociationScope < ::ActiveRecord::Associations::AssociationScope # :nodoc: + def scope(association) + source_reflection = association.reflection + owner = association.owner + unscoped = association.klass.unscoped + reverse_chain = get_chain(source_reflection, association, unscoped.alias_tracker).reverse + + previous_reflection, last_reflection, last_ordered, last_join_ids = last_scope_chain(reverse_chain, owner) + + add_constraints(last_reflection, last_reflection.join_primary_key, last_join_ids, owner, last_ordered, + previous_reflection: previous_reflection) + end + + private + + def last_scope_chain(reverse_chain, owner) + # Pulled from https://github.com/rails/rails/pull/42448 + # Fixes cases where the foreign key is not id + first_item = reverse_chain.shift + first_scope = [nil, first_item, false, [owner._read_attribute(first_item.join_foreign_key)]] + + reverse_chain.inject(first_scope) do |(previous_reflection, reflection, ordered, join_ids), next_reflection| + key = reflection.join_primary_key + records = add_constraints(reflection, key, join_ids, owner, ordered, previous_reflection: previous_reflection) + foreign_key = next_reflection.join_foreign_key + record_ids = records.pluck(foreign_key) # rubocop:disable CodeReuse/ActiveRecord + records_ordered = records && records.order_values.any? + + [reflection, next_reflection, records_ordered, record_ids] + end + end + + def add_constraints(reflection, key, join_ids, owner, ordered, previous_reflection: nil) + scope = reflection.build_scope(reflection.aliased_table).where(key => join_ids) # rubocop:disable CodeReuse/ActiveRecord + + # Pulled from https://github.com/rails/rails/pull/42590 + # Fixes cases where used with an STI type + relation = reflection.klass.scope_for_association + scope.merge!( + relation.except(:select, :create_with, :includes, :preload, :eager_load, :joins, :left_outer_joins) + ) + + # Attempt to fix use case where we have a polymorphic relationship + # Build on an additional scope to filter by the polymorphic type + if reflection.type + polymorphic_class = previous_reflection.try(:klass) || owner.class + + polymorphic_type = transform_value(polymorphic_class.polymorphic_name) + scope = apply_scope(scope, reflection.aliased_table, reflection.type, polymorphic_type) + end + + scope = reflection.constraints.inject(scope) do |memo, scope_chain_item| + item = eval_scope(reflection, scope_chain_item, owner) + scope.unscope!(*item.unscope_values) + scope.where_clause += item.where_clause + scope.order_values = item.order_values | scope.order_values + scope + end + + if scope.order_values.empty? && ordered + split_scope = DisableJoins::Relation.create(scope.klass, key, join_ids) + split_scope.where_clause += scope.where_clause + split_scope + else + scope + end + end + end + end + end + end +end diff --git a/lib/gem_extensions/active_record/disable_joins/relation.rb b/lib/gem_extensions/active_record/disable_joins/relation.rb new file mode 100644 index 00000000000..01eb6381a85 --- /dev/null +++ b/lib/gem_extensions/active_record/disable_joins/relation.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module GemExtensions + module ActiveRecord + module DisableJoins + class Relation < ::ActiveRecord::Relation + attr_reader :ids, :key + + def initialize(klass, key, ids) + @ids = ids.uniq + @key = key + super(klass) + end + + def limit(value) + records.take(value) # rubocop:disable CodeReuse/ActiveRecord + end + + def first(limit = nil) + if limit + records.limit(limit).first + else + records.first + end + end + + def load + super + records = @records + + records_by_id = records.group_by do |record| + record[key] + end + + records = ids.flat_map { |id| records_by_id[id.to_i] } + records.compact! + + @records = records + end + end + end + end +end diff --git a/lib/generators/gitlab/usage_metric/templates/database_instrumentation_class.rb.template b/lib/generators/gitlab/usage_metric/templates/database_instrumentation_class.rb.template new file mode 100644 index 00000000000..74b1ed69a5c --- /dev/null +++ b/lib/generators/gitlab/usage_metric/templates/database_instrumentation_class.rb.template @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class <%= class_name %>Metric < DatabaseMetric + operation :<%= operation%> + + relation do + # Insert ActiveRecord relation here + end + end + end + end + end +end diff --git a/lib/generators/gitlab/usage_metric/templates/generic_instrumentation_class.rb.template b/lib/generators/gitlab/usage_metric/templates/generic_instrumentation_class.rb.template new file mode 100644 index 00000000000..fa6c18a289c --- /dev/null +++ b/lib/generators/gitlab/usage_metric/templates/generic_instrumentation_class.rb.template @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class <%= class_name %>Metric < GenericMetric + value do + # Insert metric code logic here + end + end + end + end + end +end diff --git a/lib/generators/gitlab/usage_metric/templates/instrumentation_class.rb.template b/lib/generators/gitlab/usage_metric/templates/instrumentation_class.rb.template deleted file mode 100644 index 603b6f3bc8a..00000000000 --- a/lib/generators/gitlab/usage_metric/templates/instrumentation_class.rb.template +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class <%= class_name %>Metric < <%= metric_superclass %>Metric - def value - end - end - end - end - end -end diff --git a/lib/generators/gitlab/usage_metric/usage_metric_generator.rb b/lib/generators/gitlab/usage_metric/usage_metric_generator.rb deleted file mode 100644 index f7125fdc911..00000000000 --- a/lib/generators/gitlab/usage_metric/usage_metric_generator.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -require 'rails/generators' - -module Gitlab - class UsageMetricGenerator < Rails::Generators::Base - CE_DIR = 'lib/gitlab/usage/metrics/instrumentations' - EE_DIR = 'ee/lib/ee/gitlab/usage/metrics/instrumentations' - SPEC_CE_DIR = 'spec/lib/gitlab/usage/metrics/instrumentations' - SPEC_EE_DIR = 'ee/spec/lib/ee/gitlab/usage/metrics/instrumentations' - - ALLOWED_SUPERCLASSES = { - generic: 'Generic', - database: 'Database', - redis_hll: 'RedisHLL' - }.freeze - - source_root File.expand_path('templates', __dir__) - - class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if instrumentation is for EE' - class_option :type, type: :string, desc: "Metric type, must be one of: #{ALLOWED_SUPERCLASSES.keys.join(', ')}" - - argument :class_name, type: :string, desc: 'Instrumentation class name, e.g.: CountIssues' - - def create_class_files - validate! - - template "instrumentation_class.rb.template", file_path - template "instrumentation_class_spec.rb.template", spec_file_path - end - - private - - def validate! - raise ArgumentError, "Type is required, valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" unless type.present? - raise ArgumentError, "Unknown type '#{type}', valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" if metric_superclass.nil? - end - - def ee? - options[:ee] - end - - def type - options[:type] - end - - def file_path - dir = ee? ? EE_DIR : CE_DIR - - File.join(dir, file_name) - end - - def spec_file_path - dir = ee? ? SPEC_EE_DIR : SPEC_CE_DIR - - File.join(dir, spec_file_name) - end - - def file_name - "#{class_name.underscore}_metric.rb" - end - - def spec_file_name - "#{class_name.underscore}_metric_spec.rb" - end - - def metric_superclass - ALLOWED_SUPERCLASSES[type.to_sym] - end - end -end diff --git a/lib/generators/gitlab/usage_metric_generator.rb b/lib/generators/gitlab/usage_metric_generator.rb new file mode 100644 index 00000000000..c0fdcf21f20 --- /dev/null +++ b/lib/generators/gitlab/usage_metric_generator.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'rails/generators' + +module Gitlab + class UsageMetricGenerator < Rails::Generators::Base + CE_DIR = 'lib/gitlab/usage/metrics/instrumentations' + EE_DIR = 'ee/lib/ee/gitlab/usage/metrics/instrumentations' + SPEC_CE_DIR = 'spec/lib/gitlab/usage/metrics/instrumentations' + SPEC_EE_DIR = 'ee/spec/lib/ee/gitlab/usage/metrics/instrumentations' + + ALLOWED_SUPERCLASSES = { + generic: 'Generic', + database: 'Database', + redis: 'Redis' + }.freeze + + ALLOWED_OPERATIONS = %w(count distinct_count estimate_batch_distinct_count).freeze + + source_root File.expand_path('usage_metric/templates', __dir__) + + class_option :ee, type: :boolean, optional: true, default: false, desc: 'Indicates if instrumentation is for EE' + class_option :type, type: :string, desc: "Metric type, must be one of: #{ALLOWED_SUPERCLASSES.keys.join(', ')}" + class_option :operation, type: :string, desc: "Metric operation, must be one of: #{ALLOWED_OPERATIONS.join(', ')}" + + argument :class_name, type: :string, desc: 'Instrumentation class name, e.g.: CountIssues' + + def create_class_files + validate! + + template "database_instrumentation_class.rb.template", file_path if type == 'database' + template "generic_instrumentation_class.rb.template", file_path if type == 'generic' + + template "instrumentation_class_spec.rb.template", spec_file_path + end + + private + + def validate! + raise ArgumentError, "Type is required, valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" unless type.present? + raise ArgumentError, "Unknown type '#{type}', valid options are #{ALLOWED_SUPERCLASSES.keys.join(', ')}" if metric_superclass.nil? + raise ArgumentError, "Unknown operation '#{operation}' valid operations are #{ALLOWED_OPERATIONS.join(', ')}" if type == 'database' && !ALLOWED_OPERATIONS.include?(operation) + end + + def ee? + options[:ee] + end + + def type + options[:type] + end + + def operation + options[:operation] + end + + def file_path + dir = ee? ? EE_DIR : CE_DIR + + File.join(dir, file_name) + end + + def spec_file_path + dir = ee? ? SPEC_EE_DIR : SPEC_CE_DIR + + File.join(dir, spec_file_name) + end + + def file_name + "#{class_name.underscore}_metric.rb" + end + + def spec_file_name + "#{class_name.underscore}_metric_spec.rb" + end + + def metric_superclass + ALLOWED_SUPERCLASSES[type.to_sym] + end + end +end diff --git a/lib/generators/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/post_deployment_migration/post_deployment_migration_generator.rb new file mode 100644 index 00000000000..66ee0e2440f --- /dev/null +++ b/lib/generators/post_deployment_migration/post_deployment_migration_generator.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'rails/generators' + +module PostDeploymentMigration + class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase + def create_migration_file + timestamp = Time.now.utc.strftime('%Y%m%d%H%M%S') + + template "migration.rb", "db/post_migrate/#{timestamp}_#{file_name}.rb" + end + + def migration_class_name + file_name.camelize + end + end +end diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb deleted file mode 100644 index 568104cb30b..00000000000 --- a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -require 'rails/generators' - -module Rails - class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase - def create_migration_file - timestamp = Time.now.utc.strftime('%Y%m%d%H%M%S') - - template "migration.rb", "db/post_migrate/#{timestamp}_#{file_name}.rb" - end - - def migration_class_name - file_name.camelize - end - end -end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index f10168623e9..d93d7acbaad 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -136,7 +136,6 @@ module Gitlab def self.process_name return 'sidekiq' if Gitlab::Runtime.sidekiq? - return 'action_cable' if Gitlab::Runtime.action_cable? return 'console' if Gitlab::Runtime.console? return 'test' if Rails.env.test? diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 9a37a41ff81..f94696e3186 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -38,36 +38,19 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def serialized_records strong_memoize(:serialized_records) do - # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records - if default_test_stage? || default_staging_stage? - ci_build_join = mr_metrics_table - .join(build_table) - .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) - .join_sources - - records = ordered_and_limited_query - .joins(ci_build_join) - .select(build_table[:id], *time_columns) - - yield records if block_given? - ci_build_records = preload_ci_build_associations(records) - - AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] }) - else - records = ordered_and_limited_query.select(*columns, *time_columns) - - yield records if block_given? - records = preload_associations(records) - - records.map do |record| - project = record.project - attributes = record.attributes.merge({ - project_path: project.path, - namespace_path: project.namespace.route.path, - author: record.author - }) - serializer.represent(attributes) - end + records = ordered_and_limited_query.select(*columns, *time_columns) + + yield records if block_given? + records = preload_associations(records) + + records.map do |record| + project = record.project + attributes = record.attributes.merge({ + project_path: project.path, + namespace_path: project.namespace.route.path, + author: record.author + }) + serializer.represent(attributes) end end end @@ -83,26 +66,10 @@ module Gitlab end end - def default_test_stage? - stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage) - end - - def default_staging_stage? - stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_staging_stage) - end - def serializer MAPPINGS.fetch(subject_class).fetch(:serializer_class).new end - # rubocop: disable CodeReuse/ActiveRecord - def preload_ci_build_associations(records) - results = records.map(&:attributes) - - Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] })) - end - # rubocop: enable CodeReuse/ActiveRecord - def ordered_and_limited_query strong_memoize(:ordered_and_limited_query) do order_by(query, sort, direction, columns).page(page).per(per_page).without_count diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb new file mode 100644 index 00000000000..94e20762368 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -0,0 +1,186 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class RequestParams + include ActiveModel::Model + include ActiveModel::Validations + include ActiveModel::Attributes + include Gitlab::Utils::StrongMemoize + + MAX_RANGE_DAYS = 180.days.freeze + DEFAULT_DATE_RANGE = 29.days # 30 including Date.today + + STRONG_PARAMS_DEFINITION = [ + :created_before, + :created_after, + :author_username, + :milestone_title, + :sort, + :direction, + :page, + :stage_id, + :end_event_filter, + label_name: [].freeze, + assignee_username: [].freeze, + project_ids: [].freeze + ].freeze + + FINDER_PARAM_NAMES = [ + :assignee_username, + :author_username, + :milestone_title, + :label_name + ].freeze + + attr_writer :project_ids + + attribute :created_after, :datetime + attribute :created_before, :datetime + attribute :group + attribute :current_user + attribute :value_stream + attribute :sort + attribute :direction + attribute :page + attribute :project + attribute :stage_id + attribute :end_event_filter + + FINDER_PARAM_NAMES.each do |param_name| + attribute param_name + end + + validates :created_after, presence: true + validates :created_before, presence: true + + validate :validate_created_before + validate :validate_date_range + + def initialize(params = {}) + super(params) + + self.created_before = (self.created_before || Time.current).at_end_of_day + self.created_after = (created_after || default_created_after).at_beginning_of_day + self.end_event_filter ||= Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder::DEFAULT_END_EVENT_FILTER + end + + def project_ids + Array(@project_ids) + end + + def to_data_collector_params + { + current_user: current_user, + from: created_after, + to: created_before, + project_ids: project_ids, + sort: sort&.to_sym, + direction: direction&.to_sym, + page: page, + end_event_filter: end_event_filter.to_sym + }.merge(attributes.symbolize_keys.slice(*FINDER_PARAM_NAMES)) + end + + def to_data_attributes + {}.tap do |attrs| + attrs[:group] = group_data_attributes if group + attrs[:value_stream] = value_stream_data_attributes.to_json if value_stream + attrs[:created_after] = created_after.to_date.iso8601 + attrs[:created_before] = created_before.to_date.iso8601 + attrs[:projects] = group_projects(project_ids) if group && project_ids.present? + attrs[:labels] = label_name.to_json if label_name.present? + attrs[:assignees] = assignee_username.to_json if assignee_username.present? + attrs[:author] = author_username if author_username.present? + attrs[:milestone] = milestone_title if milestone_title.present? + attrs[:sort] = sort if sort.present? + attrs[:direction] = direction if direction.present? + attrs[:stage] = stage_data_attributes.to_json if stage_id.present? + end + end + + private + + def group_data_attributes + { + id: group.id, + name: group.name, + parent_id: group.parent_id, + full_path: group.full_path, + avatar_url: group.avatar_url + } + end + + def value_stream_data_attributes + { + id: value_stream.id, + name: value_stream.name, + is_custom: value_stream.custom? + } + end + + def group_projects(project_ids) + GroupProjectsFinder.new( + group: group, + current_user: current_user, + options: { include_subgroups: true }, + project_ids_relation: project_ids + ) + .execute + .with_route + .map { |project| project_data_attributes(project) } + .to_json + end + + def project_data_attributes(project) + { + id: project.to_gid.to_s, + name: project.name, + path_with_namespace: project.path_with_namespace, + avatar_url: project.avatar_url + } + end + + def stage_data_attributes + return unless stage + + { + id: stage.id || stage.name, + title: stage.name + } + end + + def validate_created_before + return if created_after.nil? || created_before.nil? + + errors.add(:created_before, :invalid) if created_after > created_before + end + + def validate_date_range + return if created_after.nil? || created_before.nil? + + if (created_before - created_after) > MAX_RANGE_DAYS + errors.add(:created_after, s_('CycleAnalytics|The given date range is larger than 180 days')) + end + end + + def default_created_after + if created_before + (created_before - DEFAULT_DATE_RANGE) + else + DEFAULT_DATE_RANGE.ago + end + end + + def stage + return unless value_stream + + strong_memoize(:stage) do + ::Analytics::CycleAnalytics::StageFinder.new(parent: project || group, stage_id: stage_id).execute if stage_id + end + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb index 11fe1dde12f..5648984ecbb 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -5,7 +5,7 @@ module Gitlab module CycleAnalytics module StageQueryHelpers def execute_query(query) - ActiveRecord::Base.connection.execute(query.to_sql) + ApplicationRecord.connection.execute(query.to_sql) end def zero_interval diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 13e78e72175..1afb2eda149 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -53,7 +53,7 @@ module Gitlab personal_access_token_check(password, project) || deploy_token_check(login, password, project) || user_with_password_for_git(login, password) || - Gitlab::Auth::Result.new + Gitlab::Auth::Result::EMPTY rate_limit!(rate_limiter, success: result.success?, login: login) look_to_limit_user(result.actor) @@ -202,13 +202,29 @@ module Gitlab return unless valid_scoped_token?(token, all_available_scopes) - return if project && token.user.project_bot? && !project.bots.include?(token.user) + if project && token.user.project_bot? + return unless token_bot_in_project?(token.user, project) || token_bot_in_group?(token.user, project) + end if can_user_login_with_non_expired_password?(token.user) || token.user.project_bot? Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end + def token_bot_in_project?(user, project) + project.bots.include?(user) + end + + # rubocop: disable CodeReuse/ActiveRecord + + # A workaround for adding group-level automation is to add the bot user of a project access token as a group member. + # In order to make project access tokens work this way during git authentication, we need to add an additional check for group membership. + # This is a temporary workaround until service accounts are implemented. + def token_bot_in_group?(user, project) + project.group && project.group.members_with_parents.where(user_id: user.id).exists? + end + # rubocop: enable CodeReuse/ActiveRecord + def valid_oauth_token?(token) token && token.accessible? && valid_scoped_token?(token, [:api]) end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index f54fa7504a3..a7312ac759a 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -298,7 +298,7 @@ module Gitlab when :api api_request? when :archive - archive_request? if Feature.enabled?(:allow_archive_as_web_access_format, default_enabled: :yaml) + archive_request? end end diff --git a/lib/gitlab/auth/otp/strategies/forti_token_cloud.rb b/lib/gitlab/auth/otp/strategies/forti_token_cloud.rb index 079d631e22a..7ef8a1076f4 100644 --- a/lib/gitlab/auth/otp/strategies/forti_token_cloud.rb +++ b/lib/gitlab/auth/otp/strategies/forti_token_cloud.rb @@ -35,7 +35,7 @@ module Gitlab end def access_token - Gitlab::Json.parse(access_token_create_response)['access_token'] + Gitlab::Json.parse(access_token_create_response.body)['access_token'] end def verify_otp(otp_code) diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index da874524826..69525a281e9 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -2,7 +2,18 @@ module Gitlab module Auth - Result = Struct.new(:actor, :project, :type, :authentication_abilities) do + class Result + attr_reader :actor, :project, :type, :authentication_abilities + + def initialize(actor, project, type, authentication_abilities) + @actor = actor + @project = project + @type = type + @authentication_abilities = authentication_abilities + end + + EMPTY = self.new(nil, nil, nil, nil).freeze + def ci?(for_project) type == :ci && project && @@ -21,6 +32,29 @@ module Gitlab def failed? !success? end + + def auth_user + actor.is_a?(User) ? actor : nil + end + alias_method :user, :auth_user + + def deploy_token + actor.is_a?(DeployToken) ? actor : nil + end + + def can?(action) + actor&.can?(action) + end + + def can_perform_action_on_project?(action, given_project) + Ability.allowed?(actor, action, given_project) + end + + def authentication_abilities_include?(ability) + return false if authentication_abilities.blank? + + authentication_abilities.include?(ability) + end end end end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index 9f4d6557023..0826887dd0a 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -31,7 +31,7 @@ module Gitlab queue.each do |job| migration_class, migration_args = job.args - next unless job.queue == self.queue + next unless job.klass == 'BackgroundMigrationWorker' next unless migration_class == steal_class next if block_given? && !(yield job) @@ -60,11 +60,14 @@ module Gitlab end def self.remaining - scheduled = Sidekiq::ScheduledSet.new.count do |job| - job.queue == self.queue - end + enqueued = Sidekiq::Queue.new(self.queue) + scheduled = Sidekiq::ScheduledSet.new - scheduled + Sidekiq::Queue.new(self.queue).size + [enqueued, scheduled].sum do |set| + set.count do |job| + job.klass == 'BackgroundMigrationWorker' + end + end end def self.exists?(migration_class, additional_queues = []) @@ -105,13 +108,11 @@ module Gitlab end def self.enqueued_job?(queues, migration_class) - queues.each do |queue| - queue.each do |job| - return true if job.queue == self.queue && job.args.first == migration_class + queues.any? do |queue| + queue.any? do |job| + job.klass == 'BackgroundMigrationWorker' && job.args.first == migration_class end end - - false end end end diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb index a0d0791b6af..b0a8c3a8cbb 100644 --- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests.rb @@ -27,6 +27,17 @@ module Gitlab eligible_mrs.each_slice(10) do |slice| MergeRequest.where(id: slice).update_all(draft: true) end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillDraftStatusOnMergeRequests', + arguments + ) end end end diff --git a/lib/gitlab/background_migration/backfill_integrations_type_new.rb b/lib/gitlab/background_migration/backfill_integrations_type_new.rb new file mode 100644 index 00000000000..d1a939af58e --- /dev/null +++ b/lib/gitlab/background_migration/backfill_integrations_type_new.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the new `integrations.type_new` column, which contains + # the real class name, rather than the legacy class name in `type` + # which is mapped via `Gitlab::Integrations::StiType`. + class BackfillIntegrationsTypeNew + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, stop_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = define_batchable_model(batch_table) + .where(batch_column => start_id..stop_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + process_sub_batch(sub_batch) + + sleep(pause_ms * 0.001) if pause_ms > 0 + end + end + + private + + def connection + ActiveRecord::Base.connection + end + + def process_sub_batch(sub_batch) + # Extract the start/stop IDs from the current sub-batch + sub_start_id, sub_stop_id = sub_batch.pluck(Arel.sql('MIN(id), MAX(id)')).first + + # This matches the mapping from the INSERT trigger added in + # db/migrate/20210721135638_add_triggers_to_integrations_type_new.rb + connection.execute(<<~SQL) + WITH mapping(old_type, new_type) AS (VALUES + ('AsanaService', 'Integrations::Asana'), + ('AssemblaService', 'Integrations::Assembla'), + ('BambooService', 'Integrations::Bamboo'), + ('BugzillaService', 'Integrations::Bugzilla'), + ('BuildkiteService', 'Integrations::Buildkite'), + ('CampfireService', 'Integrations::Campfire'), + ('ConfluenceService', 'Integrations::Confluence'), + ('CustomIssueTrackerService', 'Integrations::CustomIssueTracker'), + ('DatadogService', 'Integrations::Datadog'), + ('DiscordService', 'Integrations::Discord'), + ('DroneCiService', 'Integrations::DroneCi'), + ('EmailsOnPushService', 'Integrations::EmailsOnPush'), + ('EwmService', 'Integrations::Ewm'), + ('ExternalWikiService', 'Integrations::ExternalWiki'), + ('FlowdockService', 'Integrations::Flowdock'), + ('HangoutsChatService', 'Integrations::HangoutsChat'), + ('IrkerService', 'Integrations::Irker'), + ('JenkinsService', 'Integrations::Jenkins'), + ('JiraService', 'Integrations::Jira'), + ('MattermostService', 'Integrations::Mattermost'), + ('MattermostSlashCommandsService', 'Integrations::MattermostSlashCommands'), + ('MicrosoftTeamsService', 'Integrations::MicrosoftTeams'), + ('MockCiService', 'Integrations::MockCi'), + ('MockMonitoringService', 'Integrations::MockMonitoring'), + ('PackagistService', 'Integrations::Packagist'), + ('PipelinesEmailService', 'Integrations::PipelinesEmail'), + ('PivotaltrackerService', 'Integrations::Pivotaltracker'), + ('PrometheusService', 'Integrations::Prometheus'), + ('PushoverService', 'Integrations::Pushover'), + ('RedmineService', 'Integrations::Redmine'), + ('SlackService', 'Integrations::Slack'), + ('SlackSlashCommandsService', 'Integrations::SlackSlashCommands'), + ('TeamcityService', 'Integrations::Teamcity'), + ('UnifyCircuitService', 'Integrations::UnifyCircuit'), + ('WebexTeamsService', 'Integrations::WebexTeams'), + ('YoutrackService', 'Integrations::Youtrack'), + + -- EE-only integrations + ('GithubService', 'Integrations::Github'), + ('GitlabSlackApplicationService', 'Integrations::GitlabSlackApplication') + ) + + UPDATE integrations SET type_new = mapping.new_type + FROM mapping + WHERE integrations.id BETWEEN #{sub_start_id} AND #{sub_stop_id} + AND integrations.type = mapping.old_type + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index bc113a1e33d..f5c8796bd18 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -189,7 +189,7 @@ module Gitlab end def perform(start_id, stop_id) - Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert end private diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index 6f37f1846d2..b58f0a3a3e0 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -105,7 +105,7 @@ module Gitlab end def commit_attrs - @commit_attrs ||= { branch_name: 'master', message: 'Initial commit' } + @commit_attrs ||= { branch_name: 'main', message: 'Initial commit' } end def create_commit(snippet) diff --git a/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb new file mode 100644 index 00000000000..107ac9b0c3b --- /dev/null +++ b/lib/gitlab/background_migration/copy_ci_builds_columns_to_security_scans.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class CopyCiBuildsColumnsToSecurityScans + extend ::Gitlab::Utils::Override + + UPDATE_BATCH_SIZE = 500 + + def perform(start_id, stop_id) + (start_id..stop_id).step(UPDATE_BATCH_SIZE).each do |offset| + batch_start = offset + batch_stop = offset + UPDATE_BATCH_SIZE - 1 + + ActiveRecord::Base.connection.execute <<~SQL + UPDATE + security_scans + SET + project_id = ci_builds.project_id, + pipeline_id = ci_builds.commit_id + FROM ci_builds + WHERE ci_builds.type='Ci::Build' + AND ci_builds.id=security_scans.build_id + AND security_scans.id BETWEEN #{Integer(batch_start)} AND #{Integer(batch_stop)} + SQL + end + + mark_job_as_succeeded(start_id, stop_id) + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'CopyCiBuildsColumnsToSecurityScans', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/create_security_setting.rb b/lib/gitlab/background_migration/create_security_setting.rb new file mode 100644 index 00000000000..55b37bb03b5 --- /dev/null +++ b/lib/gitlab/background_migration/create_security_setting.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't create SecuritySetting + # as this feature exists only in EE + class CreateSecuritySetting + def perform(_from_id, _to_id) + end + end + end +end + +Gitlab::BackgroundMigration::CreateSecuritySetting.prepend_mod_with('Gitlab::BackgroundMigration::CreateSecuritySetting') diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb index d2a9939b9ee..1c60473750d 100644 --- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb +++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb @@ -34,7 +34,7 @@ module Gitlab end end - Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert execute("ANALYZE #{TEMP_TABLE}") diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb index 2bce5037d03..14c72bb4a72 100644 --- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb +++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb @@ -65,7 +65,7 @@ module Gitlab next if service_ids.empty? migrated_ids += service_ids - Gitlab::Database.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert end return if migrated_ids.empty? diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb index d6795296fb7..0a56ac1dae8 100644 --- a/lib/gitlab/background_migration/populate_issue_email_participants.rb +++ b/lib/gitlab/background_migration/populate_issue_email_participants.rb @@ -21,7 +21,7 @@ module Gitlab } end - Gitlab::Database.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert end end end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index a00d291245c..84ff7423254 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -9,6 +9,8 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid end class VulnerabilitiesFinding < ActiveRecord::Base + include ShaAttribute + self.table_name = "vulnerability_occurrences" belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' REPORT_TYPES = { @@ -21,6 +23,9 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid api_fuzzing: 6 }.with_indifferent_access.freeze enum report_type: REPORT_TYPES + + sha_attribute :fingerprint + sha_attribute :location_fingerprint end class CalculateFindingUUID diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 44106897df8..28f1a10f9a7 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -73,7 +73,7 @@ module Gitlab if project.persisted? && mv_repositories(project) log " * Created #{project.name} (#{project_full_path})".color(:green) - project.write_repository_config + project.set_full_path ProjectCacheWorker.perform_async(project.id) else diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index d29799f1029..2c60b2e36cb 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -7,7 +7,6 @@ module Gitlab attr_reader :project, :project_key, :repository_slug, :client, :errors, :users, :already_imported_cache_key attr_accessor :logger - REMOTE_NAME = 'bitbucket_server' BATCH_SIZE = 100 # The base cache key to use for tracking already imported objects. ALREADY_IMPORTED_CACHE_KEY = @@ -142,7 +141,7 @@ module Gitlab log_info(stage: 'import_repository', message: 'starting import') project.ensure_repository - project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap, remote_name: REMOTE_NAME) + project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap) log_info(stage: 'import_repository', message: 'finished import') rescue Gitlab::Shell::Error => e diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 4cbc0231bce..89c85cb50be 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -57,7 +57,7 @@ module Gitlab # Sets a cache key to the given value. # - # key - The cache key to write. + # raw_key - The cache key to write. # value - The value to set. # timeout - The time after which the cache key should expire. def self.write(raw_key, value, timeout: TIMEOUT) @@ -73,7 +73,7 @@ module Gitlab # Increment the integer value of a key by one. # Sets the value to zero if missing before incrementing # - # key - The cache key to increment. + # raw_key - The cache key to increment. # timeout - The time after which the cache key should expire. # @return - the incremented value def self.increment(raw_key, timeout: TIMEOUT) @@ -85,6 +85,22 @@ module Gitlab end end + # Increment the integer value of a key by the given value. + # Sets the value to zero if missing before incrementing + # + # raw_key - The cache key to increment. + # value - The value to increment the key + # timeout - The time after which the cache key should expire. + # @return - the incremented value + def self.increment_by(raw_key, value, timeout: TIMEOUT) + key = cache_key_for(raw_key) + + Redis::Cache.with do |redis| + redis.incrby(key, value) + redis.expire(key, timeout) + end + end + # Adds a value to a set. # # raw_key - The key of the set to add the value to. diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb index 495f12882e5..1b4a647d16f 100644 --- a/lib/gitlab/chaos.rb +++ b/lib/gitlab/chaos.rb @@ -31,7 +31,7 @@ module Gitlab expected_end_time = Time.now + duration_s while Time.now < expected_end_time - ActiveRecord::Base.connection.execute("SELECT 1") + ApplicationRecord.connection.execute("SELECT 1") end_interval_time = Time.now + [duration_s, interval_s].min rand while Time.now < end_interval_time diff --git a/lib/gitlab/chat/command.rb b/lib/gitlab/chat/command.rb index 49b7dcf4bbe..0add53f8174 100644 --- a/lib/gitlab/chat/command.rb +++ b/lib/gitlab/chat/command.rb @@ -54,10 +54,12 @@ module Gitlab } ) - service.execute(:chat) do |pipeline| + response = service.execute(:chat) do |pipeline| build_environment_variables(pipeline) build_chat_data(pipeline) end + + response.payload end # pipeline - The `Ci::Pipeline` to create the environment variables for. diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index a2d74d36b58..cfff6e919dc 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -122,7 +122,7 @@ module Gitlab def empty_project_push_message <<~MESSAGE - A default branch (e.g. master) does not yet exist for #{project.full_path} + A default branch (e.g. main) does not yet exist for #{project.full_path} Ask a project Owner or Maintainer to create a default branch: #{project_members_url} diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 4e8b293a3e6..9ecc93f871b 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -29,11 +29,60 @@ module Gitlab true end + # All commits which have been newly introduced via any of the given + # changes. This set may also contain commits which are not referenced by + # any of the new revisions. + def commits + newrevs = @changes.map do |change| + newrev = change[:newrev] + newrev unless newrev.blank? || Gitlab::Git.blank_ref?(newrev) + end.compact + + return [] if newrevs.empty? + + @commits ||= project.repository.new_commits(newrevs, allow_quarantine: true) + end + + # All commits which have been newly introduced via the given revision. + def commits_for(newrev) + commits_by_id = commits.index_by(&:id) + + result = [] + pending = Set[newrev] + + # We go up the parent chain of our newrev and collect all commits which + # are new. In case a commit's ID cannot be found in the set of new + # commits, then it must already be a preexisting commit. + while pending.any? + rev = pending.first + pending.delete(rev) + + # Remove the revision from commit candidates such that we don't walk + # it multiple times. If the hash doesn't contain the revision, then + # we have either already walked the commit or it's not new. + commit = commits_by_id.delete(rev) + next if commit.nil? + + # Only add the parent ID to the pending set if we actually know its + # commit to guards us against readding an ID which we have already + # queued up before. + commit.parent_ids.each do |parent_id| + pending.add(parent_id) if commits_by_id.has_key?(parent_id) + end + + result << commit + end + + result + end + protected def single_access_checks! # Iterate over all changes to find if user allowed all of them to be applied changes.each do |change| + commits = Gitlab::Lazy.new { commits_for(change[:newrev]) } if Feature.enabled?(:changes_batch_commits) + # If user does not have access to make at least one change, cancel all # push by allowing the exception to bubble up Checks::SingleChangeAccess.new( @@ -41,7 +90,8 @@ module Gitlab user_access: user_access, project: project, protocol: protocol, - logger: logger + logger: logger, + commits: commits ).validate! end end diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb index 280b2dd25e2..2fd48dfbfe2 100644 --- a/lib/gitlab/checks/single_change_access.rb +++ b/lib/gitlab/checks/single_change_access.rb @@ -11,7 +11,7 @@ module Gitlab def initialize( change, user_access:, project:, - protocol:, logger: + protocol:, logger:, commits: nil ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @@ -19,6 +19,7 @@ module Gitlab @user_access = user_access @project = project @protocol = protocol + @commits = commits @logger = logger @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}") diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 97988d8aa13..ef936581c10 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -33,6 +33,8 @@ module Gitlab Result = Struct.new(:html, :state, :append, :truncated, :offset, :size, :total, keyword_init: true) # rubocop:disable Lint/StructNewOverride class Converter + include EncodingHelper + def on_0(_) reset end @@ -256,6 +258,7 @@ module Gitlab start_offset = @offset stream.each_line do |line| + line = encode_utf8_no_detect(line) s = StringScanner.new(line) until s.eos? diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 8f2d47e7ccc..e48080993ab 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -9,6 +9,8 @@ module Gitlab # Line::Segment is a portion of a line that has its own style # and text. Multiple segments make the line content. class Segment + include EncodingHelper + attr_accessor :text, :style def initialize(style:) @@ -21,11 +23,12 @@ module Gitlab end def to_h - # Without force encoding to UTF-8 we could get an error - # when serializing the Hash to JSON. + # Without forcing the encoding to UTF-8 and then replacing + # invalid UTF-8 sequences we can get an error when serializing + # the Hash to JSON. # Encoding::UndefinedConversionError: # "\xE2" from ASCII-8BIT to UTF-8 - { text: text.force_encoding('UTF-8') }.tap do |result| + { text: encode_utf8_no_detect(text) }.tap do |result| result[:style] = style.to_s if style.set? end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 9c6428d701c..aceaf012f7e 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,13 +17,13 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :ref, :source + attr_reader :root, :context, :source_ref_path, :source - def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil, source: nil) - @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, source_ref_path: nil, source: nil) + @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline, ref: source_ref_path) @context.set_deadline(TIMEOUT_SECONDS) - @ref = ref + @source_ref_path = source_ref_path @source = source @config = expand_config(config) @@ -108,13 +108,37 @@ module Gitlab end end - def build_context(project:, sha:, user:, parent_pipeline:) + def build_context(project:, sha:, user:, parent_pipeline:, ref:) Config::External::Context.new( project: project, sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, - variables: project&.predefined_variables&.to_runner_variables) + variables: build_variables(project: project, ref: ref)) + end + + def build_variables(project:, ref:) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless project + + # The order of the following lines is important as priority of CI variables is + # defined globally within GitLab. + # + # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence + variables.concat(project.predefined_variables) + variables.concat(pipeline_predefined_variables(ref: ref)) + variables.concat(project.ci_instance_variables_for(ref: ref)) + variables.concat(project.group.ci_variables_for(ref, project)) if project.group + variables.concat(project.ci_variables_for(ref: ref)) + end + end + + # https://gitlab.com/gitlab-org/gitlab/-/issues/337633 aims to add all predefined variables + # to this list, but only CI_COMMIT_REF_NAME is available right now to support compliance pipelines. + def pipeline_predefined_variables(ref:) + Gitlab::Ci::Variables::Collection.new.tap do |v| + v.append(key: 'CI_COMMIT_REF_NAME', value: ref) + end end def track_and_raise_for_dev_exception(error) diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index ad0ed00aa6f..368d8f07f8d 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -9,8 +9,10 @@ module Gitlab # class Include < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[local file remote template artifact job project ref].freeze + ALLOWED_KEYS = %i[local file remote template artifact job project ref rules].freeze validations do validates :config, hash_or_string: true @@ -27,6 +29,20 @@ module Gitlab errors.add(:config, "must specify the file where to fetch the config from") end end + + with_options allow_nil: true do + validates :rules, array_of_hashes: true + end + end + + entry :rules, ::Gitlab::Ci::Config::Entry::Include::Rules, + description: 'List of evaluable Rules to determine file inclusion.', + inherit: false + + attributes :rules + + def skip_config_hash_validation? + true end end end diff --git a/lib/gitlab/ci/config/entry/include/rules.rb b/lib/gitlab/ci/config/entry/include/rules.rb new file mode 100644 index 00000000000..8eaf9e35aaf --- /dev/null +++ b/lib/gitlab/ci/config/entry/include/rules.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Include + class Rules < ::Gitlab::Config::Entry::ComposableArray + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + validates :config, type: Array + end + + def value + @config + end + + def composable_class + Entry::Include::Rules::Rule + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb new file mode 100644 index 00000000000..d3d0f098814 --- /dev/null +++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Include + class Rules::Rule < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[if].freeze + + attributes :if + + validations do + validates :config, presence: true + validates :config, type: { with: Hash } + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :if, expression: true + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/inherit/variables.rb b/lib/gitlab/ci/config/entry/inherit/variables.rb index aa68833bdb8..adef4d1636a 100644 --- a/lib/gitlab/ci/config/entry/inherit/variables.rb +++ b/lib/gitlab/ci/config/entry/inherit/variables.rb @@ -13,9 +13,6 @@ module Gitlab strategy :ArrayStrategy, if: -> (config) { config.is_a?(Array) } class BooleanStrategy < ::Gitlab::Config::Entry::Boolean - def inherit?(_key) - value - end end class ArrayStrategy < ::Gitlab::Config::Entry::Node @@ -25,20 +22,12 @@ module Gitlab validates :config, type: Array validates :config, array_of_strings: true end - - def inherit?(key) - value.include?(key.to_s) - end end class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["#{location} should be a bool or array of strings"] end - - def inherit?(key) - false - end end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index e6d63969161..bd4d5f33689 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -16,11 +16,8 @@ module Gitlab environment coverage retry parallel interruptible timeout release dast_configuration secrets].freeze - REQUIRED_BY_NEEDS = %i[stage].freeze - validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS - validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? validates :script, presence: true with_options allow_nil: true do diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 79dfb0eec1d..3543b5493bd 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -31,7 +31,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true - validates :rules, array_of_hashes: true + validates :rules, nested_array_of_hashes: true validates :resource_group, type: String end end @@ -88,9 +88,6 @@ module Gitlab validate_against_warnings end - # inherit root variables - @root_variables_value = deps&.variables_value # rubocop:disable Gitlab/ModuleWithInstanceVariables - yield if block_given? end end @@ -123,27 +120,13 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, - variables: root_and_job_variables_value, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 - job_variables: job_variables, + job_variables: variables_value.to_h, root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, resource_group: resource_group }.compact end - def root_and_job_variables_value - root_variables = @root_variables_value.to_h # rubocop:disable Gitlab/ModuleWithInstanceVariables - root_variables = root_variables.select do |key, _| - inherit_entry&.variables_entry&.inherit?(key) - end - - root_variables.merge(variables_value.to_h) - end - - def job_variables - variables_value.to_h - end - def root_variables_inheritance inherit_entry&.variables_entry&.value end diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb index bf74f995e80..53e52981471 100644 --- a/lib/gitlab/ci/config/entry/rules.rb +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -13,7 +13,7 @@ module Gitlab end def value - @config + [@config].flatten end def composable_class diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index 567a86c47e5..4bd8e250d7a 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -45,7 +45,7 @@ module Gitlab errors.push("Remote file `#{location}` could not be fetched because of HTTP code `#{response.code}` error!") end - response.to_s if errors.none? + response.body if errors.none? end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 3216d4eaac4..97e4922b2a1 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -33,6 +33,7 @@ module Gitlab locations .compact .map(&method(:normalize_location)) + .filter_map(&method(:verify_rules)) .flat_map(&method(:expand_project_files)) .flat_map(&method(:expand_wildcard_paths)) .map(&method(:expand_variables)) @@ -56,6 +57,15 @@ module Gitlab end end + def verify_rules(location) + # Behaves like there is no `rules` + return location unless ::Feature.enabled?(:ci_include_rules, context.project, default_enabled: :yaml) + + return unless Rules.new(location[:rules]).evaluate(context).pass? + + location + end + def expand_project_files(location) return location unless location[:project] @@ -65,8 +75,6 @@ module Gitlab end def expand_wildcard_paths(location) - return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml) - # We only support local files for wildcard paths return location unless location[:local] && location[:local].include?('*') diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb new file mode 100644 index 00000000000..5a788427172 --- /dev/null +++ b/lib/gitlab/ci/config/external/rules.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Rules + def initialize(rule_hashes) + @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) + end + + def evaluate(context) + Result.new(@rule_list.nil? || match_rule(context)) + end + + private + + def match_rule(context) + @rule_list.find { |rule| rule.matches?(nil, context) } + end + + Result = Struct.new(:result) do + def pass? + !!result + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb index 5cabbc86d3e..312f98f850a 100644 --- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -43,7 +43,6 @@ module Gitlab { name: name, instance: instance, - variables: variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 job_variables: variables, parallel: { total: total } }.compact diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index d26a903c1f8..51051b0490f 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -3,7 +3,7 @@ module Gitlab module Ci ## - # Ci::Features is a class that aggregates all CI/CD feature flags in one place. + # Deprecated: Ci::Features is a class that aggregates all CI/CD feature flags in one place. # module Features # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` diff --git a/lib/gitlab/ci/limit.rb b/lib/gitlab/ci/limit.rb index c22a3c503d5..4f914388969 100644 --- a/lib/gitlab/ci/limit.rb +++ b/lib/gitlab/ci/limit.rb @@ -24,10 +24,13 @@ module Gitlab end def log_error!(extra_context = {}) - error = LimitExceededError.new(message) - # TODO: change this to Gitlab::ErrorTracking.log_exception(error, extra_context) - # https://gitlab.com/gitlab-org/gitlab/issues/32906 - ::Gitlab::ErrorTracking.track_exception(error, extra_context) + ::Gitlab::ErrorTracking.log_exception(limit_exceeded_error, extra_context) + end + + protected + + def limit_exceeded_error + LimitExceededError.new(message) end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 4a7c11ee26e..cd2c135dd7e 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -38,6 +38,7 @@ module Gitlab pipeline = ::Ci::CreatePipelineService .new(@project, @current_user, ref: @project.default_branch) .execute(:push, dry_run: true, content: content) + .payload Result.new( jobs: dry_run_convert_to_jobs(pipeline.stages), diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb deleted file mode 100644 index 1625cb841b6..00000000000 --- a/lib/gitlab/ci/model.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Model - def table_name_prefix - "ci_" - end - - def model_name - @model_name ||= ActiveModel::Name.new(self, nil, self.name.demodulize) - end - end - end -end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 3469537a2e2..1223d664214 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -11,7 +11,9 @@ module Gitlab cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura, terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan, accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y, - codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate + codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate, + sast: ::Gitlab::Ci::Parsers::Security::Sast, + secret_detection: ::Gitlab::Ci::Parsers::Security::SecretDetection } end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb new file mode 100644 index 00000000000..41acb4d5040 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -0,0 +1,266 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + class Common + SecurityReportParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + + def self.parse!(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) + new(json_data, report, vulnerability_finding_signatures_enabled, validate: validate).parse! + end + + def initialize(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) + @json_data = json_data + @report = report + @validate = validate + @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled + end + + def parse! + return report_data unless valid? + + raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash) + + create_scanner + create_scan + create_analyzer + set_report_version + + create_findings + + report_data + rescue JSON::ParserError + raise SecurityReportParserError, 'JSON parsing failed' + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + raise SecurityReportParserError, "#{report.type} security report parsing failed" + end + + private + + attr_reader :json_data, :report, :validate + + def valid? + return true if !validate || schema_validator.valid? + + schema_validator.errors.each { |error| report.add_error('Schema', error) } + + false + end + + def schema_validator + @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data) + end + + def report_data + @report_data ||= Gitlab::Json.parse!(json_data) + end + + def report_version + @report_version ||= report_data['version'] + end + + def top_level_scanner + @top_level_scanner ||= report_data.dig('scan', 'scanner') + end + + def scan_data + @scan_data ||= report_data.dig('scan') + end + + def analyzer_data + @analyzer_data ||= report_data.dig('scan', 'analyzer') + end + + def tracking_data(data) + data['tracking'] + end + + def create_findings + if report_data["vulnerabilities"] + report_data["vulnerabilities"].each { |finding| create_finding(finding) } + end + end + + def create_finding(data, remediations = []) + identifiers = create_identifiers(data['identifiers']) + links = create_links(data['links']) + location = create_location(data['location'] || {}) + signatures = create_signatures(tracking_data(data)) + + if @vulnerability_finding_signatures_enabled && !signatures.empty? + # NOT the signature_sha - the compare key is hashed + # to create the project_fingerprint + highest_priority_signature = signatures.max_by(&:priority) + uuid = calculate_uuid_v5(identifiers.first, highest_priority_signature.signature_hex) + else + uuid = calculate_uuid_v5(identifiers.first, location&.fingerprint) + end + + report.add_finding( + ::Gitlab::Ci::Reports::Security::Finding.new( + uuid: uuid, + report_type: report.type, + name: finding_name(data, identifiers, location), + compare_key: data['cve'] || '', + location: location, + severity: parse_severity_level(data['severity']), + confidence: parse_confidence_level(data['confidence']), + scanner: create_scanner(data['scanner']), + scan: report&.scan, + identifiers: identifiers, + links: links, + remediations: remediations, + raw_metadata: data.to_json, + metadata_version: report_version, + details: data['details'] || {}, + signatures: signatures, + project_id: report.project_id, + vulnerability_finding_signatures_enabled: @vulnerability_finding_signatures_enabled)) + end + + def create_signatures(tracking) + tracking ||= { 'items' => [] } + + signature_algorithms = Hash.new { |hash, key| hash[key] = [] } + + tracking['items'].each do |item| + next unless item.key?('signatures') + + item['signatures'].each do |signature| + alg = signature['algorithm'] + signature_algorithms[alg] << signature['value'] + end + end + + signature_algorithms.map do |algorithm, values| + value = values.join('|') + signature = ::Gitlab::Ci::Reports::Security::FindingSignature.new( + algorithm_type: algorithm, + signature_value: value + ) + + if signature.valid? + signature + else + e = SecurityReportParserError.new("Vulnerability tracking signature is not valid: #{signature}") + Gitlab::ErrorTracking.track_exception(e) + nil + end + end.compact + end + + def create_scan + return unless scan_data.is_a?(Hash) + + report.scan = ::Gitlab::Ci::Reports::Security::Scan.new(scan_data) + end + + def set_report_version + report.version = report_version + end + + def create_analyzer + return unless analyzer_data.is_a?(Hash) + + params = { + id: analyzer_data.dig('id'), + name: analyzer_data.dig('name'), + version: analyzer_data.dig('version'), + vendor: analyzer_data.dig('vendor', 'name') + } + + return unless params.values.all? + + report.analyzer = ::Gitlab::Ci::Reports::Security::Analyzer.new(**params) + end + + def create_scanner(scanner_data = top_level_scanner) + return unless scanner_data.is_a?(Hash) + + report.add_scanner( + ::Gitlab::Ci::Reports::Security::Scanner.new( + external_id: scanner_data['id'], + name: scanner_data['name'], + vendor: scanner_data.dig('vendor', 'name'), + version: scanner_data.dig('version'))) + end + + def create_identifiers(identifiers) + return [] unless identifiers.is_a?(Array) + + identifiers.map { |identifier| create_identifier(identifier) }.compact + end + + def create_identifier(identifier) + return unless identifier.is_a?(Hash) + + report.add_identifier( + ::Gitlab::Ci::Reports::Security::Identifier.new( + external_type: identifier['type'], + external_id: identifier['value'], + name: identifier['name'], + url: identifier['url'])) + end + + def create_links(links) + return [] unless links.is_a?(Array) + + links.map { |link| create_link(link) }.compact + end + + def create_link(link) + return unless link.is_a?(Hash) + + ::Gitlab::Ci::Reports::Security::Link.new(name: link['name'], url: link['url']) + end + + def parse_severity_level(input) + input&.downcase.then { |value| ::Enums::Vulnerability.severity_levels.key?(value) ? value : 'unknown' } + end + + def parse_confidence_level(input) + input&.downcase.then { |value| ::Enums::Vulnerability.confidence_levels.key?(value) ? value : 'unknown' } + end + + def create_location(location_data) + raise NotImplementedError + end + + def finding_name(data, identifiers, location) + return data['message'] if data['message'].present? + return data['name'] if data['name'].present? + + identifier = identifiers.find(&:cve?) || identifiers.find(&:cwe?) || identifiers.first + "#{identifier.name} in #{location&.fingerprint_path}" + end + + def calculate_uuid_v5(primary_identifier, location_fingerprint) + uuid_v5_name_components = { + report_type: report.type, + primary_identifier_fingerprint: primary_identifier&.fingerprint, + location_fingerprint: location_fingerprint, + project_id: report.project_id + } + + if uuid_v5_name_components.values.any?(&:nil?) + Gitlab::AppLogger.warn(message: "One or more UUID name components are nil", components: uuid_v5_name_components) + return + end + + ::Security::VulnerabilityUUID.generate( + report_type: uuid_v5_name_components[:report_type], + primary_identifier_fingerprint: uuid_v5_name_components[:primary_identifier_fingerprint], + location_fingerprint: uuid_v5_name_components[:location_fingerprint], + project_id: uuid_v5_name_components[:project_id] + ) + end + end + end + end + end +end + +Gitlab::Ci::Parsers::Security::Common.prepend_mod_with("Gitlab::Ci::Parsers::Security::Common") diff --git a/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb new file mode 100644 index 00000000000..24613a441be --- /dev/null +++ b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + module Concerns + module DeprecatedSyntax + extend ActiveSupport::Concern + + included do + extend ::Gitlab::Utils::Override + + override :parse_report + end + + def report_data + @report_data ||= begin + data = super + + if data.is_a?(Array) + data = { + "version" => self.class::DEPRECATED_REPORT_VERSION, + "vulnerabilities" => data + } + end + + data + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/sast.rb b/lib/gitlab/ci/parsers/security/sast.rb new file mode 100644 index 00000000000..e3c62614cd8 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/sast.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + class Sast < Common + include Security::Concerns::DeprecatedSyntax + + DEPRECATED_REPORT_VERSION = "1.2" + + private + + def create_location(location_data) + ::Gitlab::Ci::Reports::Security::Locations::Sast.new( + file_path: location_data['file'], + start_line: location_data['start_line'], + end_line: location_data['end_line'], + class_name: location_data['class'], + method_name: location_data['method']) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/secret_detection.rb b/lib/gitlab/ci/parsers/security/secret_detection.rb new file mode 100644 index 00000000000..c6d95c1d391 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/secret_detection.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + class SecretDetection < Common + include Security::Concerns::DeprecatedSyntax + + DEPRECATED_REPORT_VERSION = "1.2" + + private + + def create_location(location_data) + ::Gitlab::Ci::Reports::Security::Locations::SecretDetection.new( + file_path: location_data['file'], + start_line: location_data['start_line'], + end_line: location_data['end_line'], + class_name: location_data['class'], + method_name: location_data['method'] + ) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb new file mode 100644 index 00000000000..3d92886cba8 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + module Validators + class SchemaValidator + class Schema + def root_path + File.join(__dir__, 'schemas') + end + + def initialize(report_type) + @report_type = report_type + end + + delegate :validate, to: :schemer + + private + + attr_reader :report_type + + def schemer + JSONSchemer.schema(pathname) + end + + def pathname + Pathname.new(schema_path) + end + + def schema_path + File.join(root_path, file_name) + end + + def file_name + "#{report_type}.json" + end + end + + def initialize(report_type, report_data) + @report_type = report_type + @report_data = report_data + end + + def valid? + errors.empty? + end + + def errors + @errors ||= schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + end + + private + + attr_reader :report_type, :report_data + + def schema + Schema.new(report_type) + end + end + end + end + end + end +end + +Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema.prepend_mod_with("Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema") diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/sast.json b/lib/gitlab/ci/parsers/security/validators/schemas/sast.json new file mode 100644 index 00000000000..a7159be0190 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/sast.json @@ -0,0 +1,706 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.0.0" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "type": "object", + "description": "The vendor/maintainer of the scanner.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability.", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json b/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json new file mode 100644 index 00000000000..462e23a151c --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json @@ -0,0 +1,729 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.0.0" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "type": "object", + "description": "The vendor/maintainer of the scanner.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability.", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 7564d0c3ed5..626eba97817 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -97,15 +97,16 @@ module Gitlab .observe({ source: pipeline.source.to_s }, pipeline.total_size) end + def observe_jobs_count_in_alive_pipelines + metrics.active_jobs_histogram + .observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines) + end + def increment_pipeline_failure_reason_counter(reason) metrics.pipeline_failure_reason_counter .increment(reason: (reason || :unknown_failure).to_s) end - def dangling_build? - %i[ondemand_dast_scan webide].include?(source) - end - private # Verifies that origin_ref is a fully qualified tag reference (refs/tags/) diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 49ec1250a5f..5251dd3d40a 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -14,7 +14,7 @@ module Gitlab result = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, - ref: @pipeline.ref, + source_ref_path: @pipeline.source_ref_path, sha: @pipeline.sha, source: @pipeline.source, user: current_user, diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index dc648568129..bbfc6759b35 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -22,6 +22,7 @@ module Gitlab @command.observe_creation_duration(Time.now - @start) @command.observe_pipeline_size(@pipeline) + @command.observe_jobs_count_in_alive_pipelines @pipeline end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index e4e4f4f484a..76dfb4cbd87 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -22,16 +22,16 @@ module Gitlab end end - def skipped? - !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?) - end - def break? skipped? end private + def skipped? + !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?) + end + def commit_message_skips_ci? return false unless @pipeline.git_commit_message diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 514241e8ae2..c7106f3ec39 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -11,7 +11,7 @@ module Gitlab PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze def initialize(regexp) - super(regexp.gsub(/\\\//, '/')) + super(regexp.gsub(%r{\\/}, '/')) unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value) raise Lexer::SyntaxError, 'Invalid regular expression!' diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 84b88374a7f..10de77afe74 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -24,7 +24,16 @@ module Gitlab name = :gitlab_ci_pipeline_size_builds comment = 'Pipeline size' labels = { source: nil } - buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 3000] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + + def self.active_jobs_histogram + name = :gitlab_ci_active_jobs + comment = 'Total amount of active jobs' + labels = { plan: nil } + buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000] ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 54d92745992..c393fed26de 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -39,7 +39,7 @@ module Gitlab @cache = Gitlab::Ci::Build::Cache .new(attributes.delete(:cache), @pipeline) - recalculate_yaml_variables! + calculate_yaml_variables! end def name @@ -232,7 +232,7 @@ module Gitlab { options: { allow_failure_criteria: nil } } end - def recalculate_yaml_variables! + def calculate_yaml_variables! @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance ) diff --git a/lib/gitlab/ci/reports/security/aggregated_report.rb b/lib/gitlab/ci/reports/security/aggregated_report.rb new file mode 100644 index 00000000000..a8bb2196043 --- /dev/null +++ b/lib/gitlab/ci/reports/security/aggregated_report.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Used to represent combined Security Reports. This is typically done for vulnerability deduplication purposes. + +module Gitlab + module Ci + module Reports + module Security + class AggregatedReport + attr_reader :findings + + def initialize(reports, findings) + @reports = reports + @findings = findings + end + + def created_at + @reports.map(&:created_at).compact.min + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb new file mode 100644 index 00000000000..dc1c51b3ed0 --- /dev/null +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Finding + include ::VulnerabilityFindingHelpers + + attr_reader :compare_key + attr_reader :confidence + attr_reader :identifiers + attr_reader :links + attr_reader :location + attr_reader :metadata_version + attr_reader :name + attr_reader :old_location + attr_reader :project_fingerprint + attr_reader :raw_metadata + attr_reader :report_type + attr_reader :scanner + attr_reader :scan + attr_reader :severity + attr_accessor :uuid + attr_accessor :overridden_uuid + attr_reader :remediations + attr_reader :details + attr_reader :signatures + attr_reader :project_id + + delegate :file_path, :start_line, :end_line, to: :location + + def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + @compare_key = compare_key + @confidence = confidence + @identifiers = identifiers + @links = links + @location = location + @metadata_version = metadata_version + @name = name + @raw_metadata = raw_metadata + @report_type = report_type + @scanner = scanner + @scan = scan + @severity = severity + @uuid = uuid + @remediations = remediations + @details = details + @signatures = signatures + @project_id = project_id + @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled + + @project_fingerprint = generate_project_fingerprint + end + + def to_hash + %i[ + compare_key + confidence + identifiers + links + location + metadata_version + name + project_fingerprint + raw_metadata + report_type + scanner + scan + severity + uuid + details + signatures + ].each_with_object({}) do |key, hash| + hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def primary_identifier + identifiers.first + end + + def update_location(new_location) + @old_location = location + @location = new_location + end + + def unsafe?(severity_levels) + severity.in?(severity_levels) + end + + def eql?(other) + return false unless report_type == other.report_type && primary_identifier_fingerprint == other.primary_identifier_fingerprint + + if @vulnerability_finding_signatures_enabled + matches_signatures(other.signatures, other.uuid) + else + location.fingerprint == other.location.fingerprint + end + end + + def hash + if @vulnerability_finding_signatures_enabled && !signatures.empty? + highest_signature = signatures.max_by(&:priority) + report_type.hash ^ highest_signature.signature_hex.hash ^ primary_identifier_fingerprint.hash + else + report_type.hash ^ location.fingerprint.hash ^ primary_identifier_fingerprint.hash + end + end + + def valid? + scanner.present? && primary_identifier.present? && location.present? && uuid.present? + end + + def keys + @keys ||= identifiers.reject(&:type_identifier?).map do |identifier| + FindingKey.new(location_fingerprint: location&.fingerprint, identifier_fingerprint: identifier.fingerprint) + end + end + + def primary_identifier_fingerprint + primary_identifier&.fingerprint + end + + def <=>(other) + if severity == other.severity + compare_key <=> other.compare_key + else + ::Enums::Vulnerability.severity_levels[other.severity] <=> + ::Enums::Vulnerability.severity_levels[severity] + end + end + + def scanner_order_to(other) + return 1 unless scanner + return -1 unless other&.scanner + + scanner <=> other.scanner + end + + private + + def generate_project_fingerprint + Digest::SHA1.hexdigest(compare_key) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding_key.rb b/lib/gitlab/ci/reports/security/finding_key.rb new file mode 100644 index 00000000000..0acd923a60f --- /dev/null +++ b/lib/gitlab/ci/reports/security/finding_key.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class FindingKey + def initialize(location_fingerprint:, identifier_fingerprint:) + @location_fingerprint = location_fingerprint + @identifier_fingerprint = identifier_fingerprint + end + + def ==(other) + has_fingerprints? && other.has_fingerprints? && + location_fingerprint == other.location_fingerprint && + identifier_fingerprint == other.identifier_fingerprint + end + + def hash + location_fingerprint.hash ^ identifier_fingerprint.hash + end + + alias_method :eql?, :== + + protected + + attr_reader :location_fingerprint, :identifier_fingerprint + + def has_fingerprints? + location_fingerprint.present? && identifier_fingerprint.present? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding_signature.rb b/lib/gitlab/ci/reports/security/finding_signature.rb new file mode 100644 index 00000000000..d1d7ef5c377 --- /dev/null +++ b/lib/gitlab/ci/reports/security/finding_signature.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class FindingSignature + include VulnerabilityFindingSignatureHelpers + + attr_accessor :algorithm_type, :signature_value + + def initialize(params = {}) + @algorithm_type = params.dig(:algorithm_type) + @signature_value = params.dig(:signature_value) + end + + def signature_sha + Digest::SHA1.digest(signature_value) + end + + def signature_hex + signature_sha.unpack1("H*") + end + + def to_hash + { + algorithm_type: algorithm_type, + signature_sha: signature_sha + } + end + + def valid? + algorithm_types.key?(algorithm_type) + end + + def eql?(other) + other.algorithm_type == algorithm_type && + other.signature_sha == signature_sha + end + + alias_method :==, :eql? + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/locations/base.rb b/lib/gitlab/ci/reports/security/locations/base.rb new file mode 100644 index 00000000000..9ad1d81287f --- /dev/null +++ b/lib/gitlab/ci/reports/security/locations/base.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Locations + class Base + include ::Gitlab::Utils::StrongMemoize + + def ==(other) + other.fingerprint == fingerprint + end + + def fingerprint + strong_memoize(:fingerprint) do + Digest::SHA1.hexdigest(fingerprint_data) + end + end + + def as_json(options = nil) + fingerprint # side-effect call to initialize the ivar for serialization + + super + end + + def fingerprint_path + fingerprint_data + end + + private + + def fingerprint_data + raise NotImplementedError + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/locations/sast.rb b/lib/gitlab/ci/reports/security/locations/sast.rb new file mode 100644 index 00000000000..23ffa91e720 --- /dev/null +++ b/lib/gitlab/ci/reports/security/locations/sast.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Locations + class Sast < Base + include Security::Concerns::FingerprintPathFromFile + + attr_reader :class_name + attr_reader :end_line + attr_reader :file_path + attr_reader :method_name + attr_reader :start_line + + def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil) + @class_name = class_name + @end_line = end_line + @file_path = file_path + @method_name = method_name + @start_line = start_line + end + + def fingerprint_data + "#{file_path}:#{start_line}:#{end_line}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/locations/secret_detection.rb b/lib/gitlab/ci/reports/security/locations/secret_detection.rb new file mode 100644 index 00000000000..0fd5cc5af11 --- /dev/null +++ b/lib/gitlab/ci/reports/security/locations/secret_detection.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Locations + class SecretDetection < Base + include Security::Concerns::FingerprintPathFromFile + + attr_reader :class_name + attr_reader :end_line + attr_reader :file_path + attr_reader :method_name + attr_reader :start_line + + def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil) + @class_name = class_name + @end_line = end_line + @file_path = file_path + @method_name = method_name + @start_line = start_line + end + + def fingerprint_data + "#{file_path}:#{start_line}:#{end_line}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb new file mode 100644 index 00000000000..1ba2d909d99 --- /dev/null +++ b/lib/gitlab/ci/reports/security/report.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Report + attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers + attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version + + delegate :project_id, to: :pipeline + + def initialize(type, pipeline, created_at) + @type = type + @pipeline = pipeline + @created_at = created_at + @findings = [] + @scanners = {} + @identifiers = {} + @scanned_resources = [] + @errors = [] + end + + def commit_sha + pipeline.sha + end + + def add_error(type, message = 'An unexpected error happened!') + errors << { type: type, message: message } + end + + def errored? + errors.present? + end + + def add_scanner(scanner) + scanners[scanner.key] ||= scanner + end + + def add_identifier(identifier) + identifiers[identifier.key] ||= identifier + end + + def add_finding(finding) + findings << finding + end + + def clone_as_blank + Report.new(type, pipeline, created_at) + end + + def replace_with!(other) + instance_variables.each do |ivar| + instance_variable_set(ivar, other.public_send(ivar.to_s[1..-1])) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def merge!(other) + replace_with!(::Security::MergeReportsService.new(self, other).execute) + end + + def primary_scanner + scanners.first&.second + end + + def primary_scanner_order_to(other) + return 1 unless primary_scanner + return -1 unless other.primary_scanner + + primary_scanner <=> other.primary_scanner + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb new file mode 100644 index 00000000000..b7a5e36b108 --- /dev/null +++ b/lib/gitlab/ci/reports/security/reports.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Reports + attr_reader :reports, :pipeline + + delegate :each, :empty?, to: :reports + + def initialize(pipeline) + @reports = {} + @pipeline = pipeline + end + + def get_report(report_type, report_artifact) + reports[report_type] ||= Report.new(report_type, pipeline, report_artifact.created_at) + end + + def findings + reports.values.flat_map(&:findings) + end + + def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels) + unsafe_findings_count(target_reports, severity_levels) > vulnerabilities_allowed + end + + private + + def findings_diff(target_reports) + findings - target_reports&.findings.to_a + end + + def unsafe_findings_count(target_reports, severity_levels) + findings_diff(target_reports).count {|finding| finding.unsafe?(severity_levels)} + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb new file mode 100644 index 00000000000..6cb2e0ddb33 --- /dev/null +++ b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class VulnerabilityReportsComparer + include Gitlab::Utils::StrongMemoize + + attr_reader :base_report, :head_report + + ACCEPTABLE_REPORT_AGE = 1.week + + def initialize(project, base_report, head_report) + @base_report = base_report + @head_report = head_report + + @signatures_enabled = project.licensed_feature_available?(:vulnerability_finding_signatures) + + if @signatures_enabled + @added_findings = [] + @fixed_findings = [] + calculate_changes + end + end + + def base_report_created_at + @base_report.created_at + end + + def head_report_created_at + @head_report.created_at + end + + def base_report_out_of_date + return false unless @base_report.created_at + + ACCEPTABLE_REPORT_AGE.ago > @base_report.created_at + end + + def added + strong_memoize(:added) do + if @signatures_enabled + @added_findings + else + head_report.findings - base_report.findings + end + end + end + + def fixed + strong_memoize(:fixed) do + if @signatures_enabled + @fixed_findings + else + base_report.findings - head_report.findings + end + end + end + + private + + def calculate_changes + # This is a deconstructed version of the eql? method on + # Ci::Reports::Security::Finding. It: + # + # * precomputes for the head_findings (using FindingMatcher): + # * sets of signature shas grouped by priority + # * mappings of signature shas to the head finding object + # + # These are then used when iterating the base findings to perform + # fast(er) prioritized, signature-based comparisons between each base finding + # and the head findings. + # + # Both the head_findings and base_findings arrays are iterated once + + base_findings = base_report.findings + head_findings = head_report.findings + + matcher = FindingMatcher.new(head_findings) + + base_findings.each do |base_finding| + matched_head_finding = matcher.find_and_remove_match!(base_finding) + + @fixed_findings << base_finding if matched_head_finding.nil? + end + + @added_findings = matcher.unmatched_head_findings.values + end + end + + class FindingMatcher + attr_reader :unmatched_head_findings, :head_findings + + include Gitlab::Utils::StrongMemoize + + def initialize(head_findings) + @head_findings = head_findings + @unmatched_head_findings = @head_findings.index_by(&:object_id) + end + + def find_and_remove_match!(base_finding) + matched_head_finding = find_matched_head_finding_for(base_finding) + + # no signatures matched, so check the normal uuids of the base and head findings + # for a match + matched_head_finding = head_signatures_shas[base_finding.uuid] if matched_head_finding.nil? + + @unmatched_head_findings.delete(matched_head_finding.object_id) unless matched_head_finding.nil? + + matched_head_finding + end + + private + + def find_matched_head_finding_for(base_finding) + base_signature = sorted_signatures_for(base_finding).find do |signature| + # at this point a head_finding exists that has a signature with a + # matching priority, and a matching sha --> lookup the actual finding + # object from head_signatures_shas + head_signatures_shas[signature.signature_sha].eql?(base_finding) + end + + base_signature.present? ? head_signatures_shas[base_signature.signature_sha] : nil + end + + def sorted_signatures_for(base_finding) + base_finding.signatures.select { |signature| head_finding_signature?(signature) } + .sort_by { |sig| -sig.priority } + end + + def head_finding_signature?(signature) + head_signatures_priorities[signature.priority].include?(signature.signature_sha) + end + + def head_signatures_priorities + strong_memoize(:head_signatures_priorities) do + signatures_priorities = Hash.new { |hash, key| hash[key] = Set.new } + + head_findings.each_with_object(signatures_priorities) do |head_finding, memo| + head_finding.signatures.each do |signature| + memo[signature.priority].add(signature.signature_sha) + end + end + end + end + + def head_signatures_shas + strong_memoize(:head_signatures_shas) do + head_findings.each_with_object({}) do |head_finding, memo| + head_finding.signatures.each do |signature| + memo[signature.signature_sha] = head_finding + end + # for the final uuid check when no signatures have matched + memo[head_finding.uuid] = head_finding + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml index ebb0b5948f1..71f38ededd9 100644 --- a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml @@ -5,7 +5,7 @@ # This template is on early stage of development. # Use it with caution. For usage instruction please read -# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md +# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v3.0.0/README.md include: # workflow rules to prevent duplicate detached pipelines diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 1910913f2bd..f39a84bceec 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml -# See https://docs.gitlab.com/ee/ci/yaml/README.html for all available options +# See https://docs.gitlab.com/ee/ci/yaml/index.html for all available options # you can delete this line if you're not using Docker image: busybox:latest diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index d2d3b3ed61e..f147ad9332d 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -18,7 +18,7 @@ variables: POSTGRES_DB: database_name # This folder is cached between builds -# http://docs.gitlab.com/ee/ci/yaml/README.html#cache +# https://docs.gitlab.com/ee/ci/yaml/index.html#cache cache: paths: - ~/.cache/pip/ diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 38036c1f964..21a599fc78d 100644 --- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -10,7 +10,7 @@ # A pipeline is composed of independent jobs that run scripts, grouped into stages. # Stages run in sequential order, but jobs within stages run in parallel. # -# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages +# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages stages: # List of stages for jobs, and their order of execution - build diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 48e877684f6..43ecc4b96d5 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -27,7 +27,7 @@ code_quality: } - docker pull --quiet "$CODE_QUALITY_IMAGE" - | - docker run \ + docker run --rm \ $(propagate_env_vars \ SOURCE_CODE \ TIMEOUT_SECONDS \ diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 00fcfa64a18..208951fa1a1 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 530ab1d0f99..5c466f0984c 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 80125a9bc01..917a28bb1ee 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -252,6 +252,7 @@ semgrep-sast: - '**/*.jsx' - '**/*.ts' - '**/*.tsx' + - '**/*.c' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index d0595491400..18f0f20203d 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -27,8 +27,8 @@ secret_detection: when: never - if: $CI_COMMIT_BRANCH script: - - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - - if [[ $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ]]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi + - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi + - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 43e4ac02d41..ff7bac15017 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -18,7 +18,7 @@ variables: MYSQL_ROOT_PASSWORD: secret # This folder is cached between builds -# http://docs.gitlab.com/ee/ci/yaml/README.html#cache +# https://docs.gitlab.com/ee/ci/yaml/index.html#cache cache: paths: - vendor/ diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index e48801b7970..16bc0026aa8 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -16,7 +16,7 @@ services: - postgres:latest # This folder is cached between builds -# http://docs.gitlab.com/ee/ci/yaml/README.html#cache +# https://docs.gitlab.com/ee/ci/yaml/index.html#cache cache: paths: - node_modules/ diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index d3726fe34c5..9da50439be8 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -6,7 +6,7 @@ image: node:latest # This folder is cached between builds -# http://docs.gitlab.com/ee/ci/yaml/README.html#cache +# https://docs.gitlab.com/ee/ci/yaml/index.html#cache cache: paths: - node_modules/ diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 490fc779e17..0c8b98dc1cf 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -29,7 +29,8 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby + - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby + - bundle install -j $(nproc) # Optional - Delete if not using `rubocop` rubocop: diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml index f4f066cc7c2..ed4876c2bcc 100644 --- a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml @@ -8,7 +8,7 @@ # - A `test` stage to be present in the pipeline. # - You must define the `CIS_KUBECONFIG` variable to allow analyzer to connect to your Kubernetes cluster and fetch found vulnerabilities. # -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml new file mode 100644 index 00000000000..d27a08db181 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml @@ -0,0 +1,23 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml + +stages: + - build + - test + - deploy + - dast + +variables: + DAST_RUNNER_VALIDATION_VERSION: 1 + +validation: + stage: dast + image: + name: "registry.gitlab.com/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION" + variables: + GIT_STRATEGY: none + allow_failure: false + script: + - ~/validate.sh diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index e30777d8401..86b7d57d3cb 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -18,7 +18,7 @@ variables: bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, license-finder, - dast, api-fuzzing + dast, dast-runner-validation, api-fuzzing SECURE_BINARIES_DOWNLOAD_IMAGES: "true" SECURE_BINARIES_PUSH_IMAGES: "true" @@ -230,6 +230,16 @@ dast: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/ +dast-runner-validation: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "1" + SECURE_BINARIES_IMAGE: "registry.gitlab.com/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bdast-runner-validation\b/ + api-fuzzing: extends: .download_images variables: diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 272b980b4b2..1a857ef3eb3 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -4,7 +4,7 @@ # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml include: - - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml + - template: Terraform/Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml stages: - init diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index d34a847f2d5..a9f6fd88d0b 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -14,15 +14,22 @@ stages: - cleanup init: - extends: .init + extends: .terraform:init validate: - extends: .validate + extends: .terraform:validate build: - extends: .build + extends: .terraform:build deploy: - extends: .deploy + extends: .terraform:deploy dependencies: - build + environment: + name: $TF_STATE_NAME + +cleanup: + extends: .terraform:destroy + dependencies: + - deploy diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml new file mode 100644 index 00000000000..39c3374e534 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -0,0 +1,64 @@ +# Terraform/Base.latest +# +# The purpose of this template is to provide flexibility to the user so +# they are able to only include the jobs that they find interesting. +# +# Therefore, this template is not supposed to run any jobs. The idea is to only +# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs +# +# There is a more opinionated template which we suggest the users to abide, +# which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml + +image: + name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.0.3 + +variables: + TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project + TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend + +cache: + key: "${TF_ROOT}" + paths: + - ${TF_ROOT}/.terraform/ + - ${TF_ROOT}/.terraform.lock.hcl + +.init: &init + stage: init + script: + - cd ${TF_ROOT} + - gitlab-terraform init + +.validate: &validate + stage: validate + script: + - cd ${TF_ROOT} + - gitlab-terraform validate + +.build: &build + stage: build + script: + - cd ${TF_ROOT} + - gitlab-terraform plan + - gitlab-terraform plan-json + artifacts: + paths: + - ${TF_ROOT}/plan.cache + reports: + terraform: ${TF_ROOT}/plan.json + +.deploy: &deploy + stage: deploy + script: + - cd ${TF_ROOT} + - gitlab-terraform apply + when: manual + only: + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +.destroy: &destroy + stage: cleanup + script: + - cd ${TF_ROOT} + - gitlab-terraform destroy + when: manual diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 200388a274c..c30860ad174 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -13,7 +13,8 @@ image: name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest variables: - TF_ROOT: ${CI_PROJECT_DIR} + TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project + TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend cache: key: "${TF_ROOT}" @@ -21,43 +22,46 @@ cache: - ${TF_ROOT}/.terraform/ - ${TF_ROOT}/.terraform.lock.hcl -.init: &init +.terraform:init: &terraform_init stage: init script: - cd ${TF_ROOT} - gitlab-terraform init -.validate: &validate +.terraform:validate: &terraform_validate stage: validate script: - cd ${TF_ROOT} - gitlab-terraform validate -.build: &build +.terraform:build: &terraform_build stage: build script: - cd ${TF_ROOT} - gitlab-terraform plan - gitlab-terraform plan-json + resource_group: ${TF_STATE_NAME} artifacts: paths: - ${TF_ROOT}/plan.cache reports: terraform: ${TF_ROOT}/plan.json -.deploy: &deploy +.terraform:deploy: &terraform_deploy stage: deploy script: - cd ${TF_ROOT} - gitlab-terraform apply + resource_group: ${TF_STATE_NAME} when: manual only: variables: - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH -.destroy: &destroy +.terraform:destroy: &terraform_destroy stage: cleanup script: - cd ${TF_ROOT} - gitlab-terraform destroy + resource_group: ${TF_STATE_NAME} when: manual diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb index 0140218d9bc..8ab9573dd20 100644 --- a/lib/gitlab/ci/yaml_processor/dag.rb +++ b/lib/gitlab/ci/yaml_processor/dag.rb @@ -23,7 +23,7 @@ module Gitlab new(nodes).tsort rescue TSort::Cyclic - raise ValidationError, 'The pipeline has circular dependencies.' + raise ValidationError, 'The pipeline has circular dependencies' rescue MissingNodeError end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index dd5107bad9a..a97c7050fbb 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -69,7 +69,7 @@ module Gitlab when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], - yaml_variables: transform_to_yaml_variables(job[:variables]), # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + # yaml_variables is calculated with using job_variables in Seed::Build job_variables: transform_to_yaml_variables(job[:job_variables]), root_variables_inheritance: job[:root_variables_inheritance], needs_attributes: job.dig(:needs, :job), diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 8120f2c1243..13c6eaf4993 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -78,10 +78,26 @@ module Gitlab include LegacyValidationHelpers def validate_each(record, attribute, value) - unless value.is_a?(Array) && value.map { |hsh| hsh.is_a?(Hash) }.all? + unless validate_array_of_hashes(value) record.errors.add(attribute, 'should be an array of hashes') end end + + private + + def validate_array_of_hashes(value) + value.is_a?(Array) && value.all? { |obj| obj.is_a?(Hash) } + end + end + + class NestedArrayOfHashesValidator < ArrayOfHashesValidator + include NestedArrayHelpers + + def validate_each(record, attribute, value) + unless validate_nested_array(value, 1, &method(:validate_array_of_hashes)) + record.errors.add(attribute, 'should be an array containing hashes and arrays of hashes') + end + end end class ArrayOrStringValidator < ActiveModel::EachValidator diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb index 606d45e0f0f..a56f2413615 100644 --- a/lib/gitlab/config_checker/external_database_checker.rb +++ b/lib/gitlab/config_checker/external_database_checker.rb @@ -6,7 +6,7 @@ module Gitlab extend self def check - return [] if Gitlab::Database.postgresql_minimum_supported_version? + return [] if Gitlab::Database.main.postgresql_minimum_supported_version? [ { @@ -15,7 +15,7 @@ module Gitlab '%{pg_version_minimum} is required for this version of GitLab. ' \ 'Please upgrade your environment to a supported PostgreSQL version, ' \ 'see %{pg_requirements_url} for details.') % { - pg_version_current: Gitlab::Database.version, + pg_version_current: Gitlab::Database.main.version, pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, pg_requirements_url: 'database requirements' } diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index fbf021345ca..d40a6323d4f 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -23,7 +23,7 @@ module Gitlab # 'raw' holds the Gitlab::Git::Conflict::File that this instance wraps attr_reader :raw - delegate :type, :content, :their_path, :our_path, :our_mode, :our_blob, :repository, to: :raw + delegate :type, :content, :path, :ancestor_path, :their_path, :our_path, :our_mode, :our_blob, :repository, to: :raw def initialize(raw, merge_request:) @raw = raw @@ -227,6 +227,26 @@ module Gitlab new_path: our_path) end + def conflict_type(diff_file) + if ancestor_path.present? + if our_path.present? && their_path.present? + :both_modified + elsif their_path.blank? + :modified_source_removed_target + else + :modified_target_removed_source + end + else + if our_path.present? && their_path.present? + :both_added + elsif their_path.blank? + diff_file.renamed_file? ? :renamed_same_file : :removed_target_renamed_source + else + :removed_source_renamed_target + end + end + end + private def map_raw_lines(raw_lines) diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 047600af267..c4ee1dafe20 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -7,14 +7,14 @@ module Gitlab attr_reader :merge_request, :resolver - def initialize(merge_request) + def initialize(merge_request, allow_tree_conflicts: false) our_commit = merge_request.source_branch_head.raw their_commit = merge_request.target_branch_head.raw @target_repo = merge_request.target_project.repository @source_repo = merge_request.source_project.repository.raw @our_commit_id = our_commit.id @their_commit_id = their_commit.id - @resolver = Gitlab::Git::Conflict::Resolver.new(@target_repo.raw, @our_commit_id, @their_commit_id) + @resolver = Gitlab::Git::Conflict::Resolver.new(@target_repo.raw, @our_commit_id, @their_commit_id, allow_tree_conflicts: allow_tree_conflicts) @merge_request = merge_request end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 842920ba02e..bdcedd1896d 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -7,39 +7,40 @@ module Gitlab form_action frame_ancestors frame_src img_src manifest_src media_src object_src report_uri script_src style_src worker_src).freeze - def self.default_settings_hash - settings_hash = { - 'enabled' => Rails.env.development? || Rails.env.test?, - 'report_only' => false, - 'directives' => { - 'default_src' => "'self'", - 'base_uri' => "'self'", - 'connect_src' => "'self'", - 'font_src' => "'self'", - 'form_action' => "'self' https: http:", - 'frame_ancestors' => "'self'", - 'frame_src' => "'self' https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com", - 'img_src' => "'self' data: blob: http: https:", - 'manifest_src' => "'self'", - 'media_src' => "'self'", - 'script_src' => "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com", - 'style_src' => "'self' 'unsafe-inline'", - 'worker_src' => "'self' blob: data:", - 'object_src' => "'none'", - 'report_uri' => nil - } + def self.default_enabled + Rails.env.development? || Rails.env.test? + end + + def self.default_directives + directives = { + 'default_src' => "'self'", + 'base_uri' => "'self'", + 'connect_src' => "'self'", + 'font_src' => "'self'", + 'form_action' => "'self' https: http:", + 'frame_ancestors' => "'self'", + 'frame_src' => "'self' https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com", + 'img_src' => "'self' data: blob: http: https:", + 'manifest_src' => "'self'", + 'media_src' => "'self'", + 'script_src' => "'strict-dynamic' 'self' 'unsafe-inline' 'unsafe-eval' https://www.google.com/recaptcha/ https://www.recaptcha.net https://apis.google.com", + 'style_src' => "'self' 'unsafe-inline'", + 'worker_src' => "'self' blob: data:", + 'object_src' => "'none'", + 'report_uri' => nil } # frame-src was deprecated in CSP level 2 in favor of child-src # CSP level 3 "undeprecated" frame-src and browsers fall back on child-src if it's missing # However Safari seems to read child-src first so we'll just keep both equal - settings_hash['directives']['child_src'] = settings_hash['directives']['frame_src'] + directives['child_src'] = directives['frame_src'] - allow_webpack_dev_server(settings_hash) if Rails.env.development? - allow_cdn(settings_hash) if ENV['GITLAB_CDN_HOST'].present? - allow_customersdot(settings_hash) if Rails.env.development? && ENV['CUSTOMER_PORTAL_URL'].present? + allow_webpack_dev_server(directives) if Rails.env.development? + allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present? + allow_customersdot(directives) if Rails.env.development? && ENV['CUSTOMER_PORTAL_URL'].present? + allow_sentry(directives) if Gitlab.config.sentry&.enabled && Gitlab.config.sentry&.clientside_dsn - settings_hash + directives end def initialize(csp_directives) @@ -66,31 +67,37 @@ module Gitlab arguments.strip.split(' ').map(&:strip) end - def self.allow_webpack_dev_server(settings_hash) + def self.allow_webpack_dev_server(directives) secure = Settings.webpack.dev_server['https'] host_and_port = "#{Settings.webpack.dev_server['host']}:#{Settings.webpack.dev_server['port']}" http_url = "#{secure ? 'https' : 'http'}://#{host_and_port}" ws_url = "#{secure ? 'wss' : 'ws'}://#{host_and_port}" - append_to_directive(settings_hash, 'connect_src', "#{http_url} #{ws_url}") + append_to_directive(directives, 'connect_src', "#{http_url} #{ws_url}") end - def self.allow_cdn(settings_hash) - cdn_host = ENV['GITLAB_CDN_HOST'] - - append_to_directive(settings_hash, 'script_src', cdn_host) - append_to_directive(settings_hash, 'style_src', cdn_host) - append_to_directive(settings_hash, 'font_src', cdn_host) + def self.allow_cdn(directives, cdn_host) + append_to_directive(directives, 'script_src', cdn_host) + append_to_directive(directives, 'style_src', cdn_host) + append_to_directive(directives, 'font_src', cdn_host) end - def self.append_to_directive(settings_hash, directive, text) - settings_hash['directives'][directive] = "#{settings_hash['directives'][directive]} #{text}".strip + def self.append_to_directive(directives, directive, text) + directives[directive] = "#{directives[directive]} #{text}".strip end - def self.allow_customersdot(settings_hash) + def self.allow_customersdot(directives) customersdot_host = ENV['CUSTOMER_PORTAL_URL'] - append_to_directive(settings_hash, 'frame_src', customersdot_host) + append_to_directive(directives, 'frame_src', customersdot_host) + end + + def self.allow_sentry(directives) + sentry_dsn = Gitlab.config.sentry.clientside_dsn + sentry_uri = URI(sentry_dsn) + sentry_uri.user = nil + + append_to_directive(directives, 'connect_src', sentry_uri.to_s) end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index e7ffeeb9849..bfe3f06a56b 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -85,7 +85,7 @@ module Gitlab active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - Gitlab::Database.cached_table_exists?('application_settings') + Gitlab::Database.main.cached_table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index f50ca5119b7..267c2d32ca9 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -16,6 +16,7 @@ module Gitlab object_kind: 'deployment', status: deployment.status, status_changed_at: status_changed_at, + deployment_id: deployment.id, deployable_id: deployment.deployable_id, deployable_url: deployable_url, environment: deployment.environment.name, diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 4d70e3949dd..385f1e57705 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -2,20 +2,56 @@ module Gitlab module DataBuilder - module Pipeline - extend self + # Some callers want to include retried builds, so we wrap the payload hash + # in a SimpleDelegator with additional methods. + class Pipeline < SimpleDelegator + def self.build(pipeline) + new(pipeline) + end - def build(pipeline) - { + def initialize(pipeline) + @pipeline = pipeline + + super( object_kind: 'pipeline', object_attributes: hook_attrs(pipeline), merge_request: pipeline.merge_request && merge_request_attrs(pipeline.merge_request), user: pipeline.user.try(:hook_attrs), project: pipeline.project.hook_attrs(backward: false), commit: pipeline.commit.try(:hook_attrs), - builds: pipeline.builds.latest.map(&method(:build_hook_attrs)) - } + builds: Gitlab::Lazy.new do + preload_builds(pipeline, :latest_builds) + pipeline.latest_builds.map(&method(:build_hook_attrs)) + end + ) + end + + def with_retried_builds + merge( + builds: Gitlab::Lazy.new do + preload_builds(@pipeline, :builds) + @pipeline.builds.map(&method(:build_hook_attrs)) + end + ) + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def preload_builds(pipeline, association) + ActiveRecord::Associations::Preloader.new.preload(pipeline, + { + association => { + **::Ci::Pipeline::PROJECT_ROUTE_AND_NAMESPACE_ROUTE, + runner: :tags, + job_artifacts_archive: [], + user: [], + metadata: [] + } + } + ) end + # rubocop: enable CodeReuse/ActiveRecord def hook_attrs(pipeline) { @@ -91,7 +127,8 @@ module Gitlab { name: build.expanded_environment_name, - action: build.environment_action + action: build.environment_action, + deployment_tier: build.persisted_environment.try(:tier) } end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index a269b8d0366..acad19e096c 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -45,27 +45,18 @@ module Gitlab # It does not include the default public schema EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze - DEFAULT_POOL_HEADROOM = 10 - - # We configure the database connection pool size automatically based on the - # configured concurrency. We also add some headroom, to make sure we don't run - # out of connections when more threads besides the 'user-facing' ones are - # running. - # - # Read more about this in doc/development/database/client_side_connection_pool.md - def self.default_pool_size - headroom = (ENV["DB_POOL_HEADROOM"].presence || DEFAULT_POOL_HEADROOM).to_i - - Gitlab::Runtime.max_threads + headroom - end + DATABASES = ActiveRecord::Base + .connection_handler + .connection_pools + .each_with_object({}) do |pool, hash| + hash[pool.db_config.name.to_sym] = Connection.new(pool.connection_klass) + end + .freeze - def self.config - default_config_hash = ActiveRecord::Base.configurations.find_db_config(Rails.env)&.configuration_hash || {} + PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym - default_config_hash.with_indifferent_access.tap do |hash| - # Match config/initializers/database_config.rb - hash[:pool] ||= default_pool_size - end + def self.main + DATABASES[PRIMARY_DATABASE_NAME] end def self.has_config?(database_name) @@ -87,93 +78,34 @@ module Gitlab name.to_s == CI_DATABASE_NAME end - def self.username - config['username'] || ENV['USER'] - end - - def self.database_name - config['database'] - end - - def self.adapter_name - config['adapter'] - end - - def self.human_adapter_name - if postgresql? - 'PostgreSQL' - else - 'Unknown' - end - end - - # Disables prepared statements for the current database connection. - def self.disable_prepared_statements - ActiveRecord::Base.establish_connection(config.merge(prepared_statements: false)) - end - - # @deprecated - def self.postgresql? - adapter_name.casecmp('postgresql') == 0 - end - - def self.read_only? - false - end - - def self.read_write? - !self.read_only? - end - - # Check whether the underlying database is in read-only mode - def self.db_read_only? - pg_is_in_recovery = - ActiveRecord::Base - .connection - .execute('SELECT pg_is_in_recovery()') - .first - .fetch('pg_is_in_recovery') - - Gitlab::Utils.to_boolean(pg_is_in_recovery) - end - - def self.db_read_write? - !self.db_read_only? - end - - def self.version - @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] - end - - def self.postgresql_minimum_supported_version? - version.to_f >= MINIMUM_POSTGRES_VERSION - end - def self.check_postgres_version_and_print_warning - return if Gitlab::Database.postgresql_minimum_supported_version? return if Gitlab::Runtime.rails_runner? - Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result - - ██  ██  █████  ██████  ███  ██ ██ ███  ██  ██████  - ██  ██ ██   ██ ██   ██ ████  ██ ██ ████  ██ ██       - ██  █  ██ ███████ ██████  ██ ██  ██ ██ ██ ██  ██ ██  ███  - ██ ███ ██ ██   ██ ██   ██ ██  ██ ██ ██ ██  ██ ██ ██  ██  -  ███ ███  ██  ██ ██  ██ ██   ████ ██ ██   ████  ██████   - - ****************************************************************************** - You are using PostgreSQL <%= Gitlab::Database.version %>, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %> - is required for this version of GitLab. - <% if Rails.env.development? || Rails.env.test? %> - If using gitlab-development-kit, please find the relevant steps here: - https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql - <% end %> - Please upgrade your environment to a supported PostgreSQL version, see - https://docs.gitlab.com/ee/install/requirements.html#database for details. - ****************************************************************************** - EOS - rescue ActiveRecord::ActiveRecordError, PG::Error - # ignore - happens when Rake tasks yet have to create a database, e.g. for testing + DATABASES.each do |name, connection| + next if connection.postgresql_minimum_supported_version? + + Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result + + ██  ██  █████  ██████  ███  ██ ██ ███  ██  ██████  + ██  ██ ██   ██ ██   ██ ████  ██ ██ ████  ██ ██       + ██  █  ██ ███████ ██████  ██ ██  ██ ██ ██ ██  ██ ██  ███  + ██ ███ ██ ██   ██ ██   ██ ██  ██ ██ ██ ██  ██ ██ ██  ██  +  ███ ███  ██  ██ ██  ██ ██   ████ ██ ██   ████  ██████   + + ****************************************************************************** + You are using PostgreSQL <%= Gitlab::Database.main.version %> for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %> + is required for this version of GitLab. + <% if Rails.env.development? || Rails.env.test? %> + If using gitlab-development-kit, please find the relevant steps here: + https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql + <% end %> + Please upgrade your environment to a supported PostgreSQL version, see + https://docs.gitlab.com/ee/install/requirements.html#database for details. + ****************************************************************************** + EOS + rescue ActiveRecord::ActiveRecordError, PG::Error + # ignore - happens when Rake tasks yet have to create a database, e.g. for testing + end end def self.nulls_order(field, direction = :asc, nulls_order = :nulls_last) @@ -206,136 +138,20 @@ module Gitlab "'f'" end - def self.with_connection_pool(pool_size) - pool = create_connection_pool(pool_size) - - begin - yield(pool) - ensure - pool.disconnect! - end - end - - # Bulk inserts a number of rows into a table, optionally returning their - # IDs. - # - # table - The name of the table to insert the rows into. - # rows - An Array of Hash instances, each mapping the columns to their - # values. - # return_ids - When set to true the return value will be an Array of IDs of - # the inserted rows - # disable_quote - A key or an Array of keys to exclude from quoting (You - # become responsible for protection from SQL injection for - # these keys!) - # on_conflict - Defines an upsert. Values can be: :disabled (default) or - # :do_nothing - def self.bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) - return if rows.empty? - - keys = rows.first.keys - columns = keys.map { |key| connection.quote_column_name(key) } - - disable_quote = Array(disable_quote).to_set - tuples = rows.map do |row| - keys.map do |k| - disable_quote.include?(k) ? row[k] : connection.quote(row[k]) - end - end - - sql = <<-EOF - INSERT INTO #{table} (#{columns.join(', ')}) - VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} - EOF - - sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing - - sql = "#{sql} RETURNING id" if return_ids - - result = connection.execute(sql) - - if return_ids - result.values.map { |tuple| tuple[0].to_i } - else - [] - end - end - def self.sanitize_timestamp(timestamp) MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup end - # pool_size - The size of the DB pool. - # host - An optional host name to use instead of the default one. - def self.create_connection_pool(pool_size, host = nil, port = nil) - original_config = Gitlab::Database.config - - env_config = original_config.merge(pool: pool_size) - env_config[:host] = host if host - env_config[:port] = port if port - - ActiveRecord::ConnectionAdapters::ConnectionHandler.new.establish_connection(env_config) - end - - def self.connection - ActiveRecord::Base.connection + def self.allow_cross_joins_across_databases(url:) + # this method is implemented in: + # spec/support/database/prevent_cross_joins.rb end - private_class_method :connection - def self.cached_column_exists?(table_name, column_name) - connection.schema_cache.columns_hash(table_name).has_key?(column_name.to_s) + def self.allow_cross_database_modification_within_transaction(url:) + # this method is implemented in: + # spec/support/database/cross_database_modification_check.rb end - def self.cached_table_exists?(table_name) - exists? && connection.schema_cache.data_source_exists?(table_name) - end - - def self.database_version - row = connection.execute("SELECT VERSION()").first - - row['version'] - end - - def self.exists? - connection - - true - rescue StandardError - false - end - - def self.system_id - row = connection.execute('SELECT system_identifier FROM pg_control_system()').first - - row['system_identifier'] - end - - # @param [ActiveRecord::Connection] ar_connection - # @return [String] - def self.get_write_location(ar_connection) - use_new_load_balancer_query = Gitlab::Utils.to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true) - - sql = if use_new_load_balancer_query - <<~NEWSQL - SELECT CASE - WHEN pg_is_in_recovery() = true AND EXISTS (SELECT 1 FROM pg_stat_get_wal_senders()) - THEN pg_last_wal_replay_lsn()::text - WHEN pg_is_in_recovery() = false - THEN pg_current_wal_insert_lsn()::text - ELSE NULL - END AS location; - NEWSQL - else - <<~SQL - SELECT pg_current_wal_insert_lsn()::text AS location - SQL - end - - row = ar_connection.select_all(sql).first - row['location'] if row - end - - private_class_method :database_version - def self.add_post_migrate_path_to_rails(force: false) return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force @@ -352,47 +168,40 @@ module Gitlab end end - def self.dbname(ar_connection) + def self.db_config_names + ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) + end + + def self.db_config_name(ar_connection) if ar_connection.respond_to?(:pool) && ar_connection.pool.respond_to?(:db_config) && - ar_connection.pool.db_config.respond_to?(:database) - return ar_connection.pool.db_config.database + ar_connection.pool.db_config.respond_to?(:name) + return ar_connection.pool.db_config.name end 'unknown' end - # inside_transaction? will return true if the caller is running within a transaction. Handles special cases - # when running inside a test environment, where tests may be wrapped in transactions - def self.inside_transaction? - if Rails.env.test? - ActiveRecord::Base.connection.open_transactions > open_transactions_baseline - else - ActiveRecord::Base.connection.open_transactions > 0 - end - end - - # These methods that access @open_transactions_baseline are not thread-safe. - # These are fine though because we only call these in RSpec's main thread. If we decide to run - # specs multi-threaded, we would need to use something like ThreadGroup to keep track of this value - def self.set_open_transactions_baseline - @open_transactions_baseline = ActiveRecord::Base.connection.open_transactions - end - - def self.reset_open_transactions_baseline - @open_transactions_baseline = 0 + def self.read_only? + false end - def self.open_transactions_baseline - @open_transactions_baseline ||= 0 + def self.read_write? + !read_only? end - private_class_method :open_transactions_baseline # Monkeypatch rails with upgraded database observability - def self.install_monkey_patches + def self.install_transaction_metrics_patches! ActiveRecord::Base.prepend(ActiveRecordBaseTransactionMetrics) end + def self.install_transaction_context_patches! + ActiveRecord::ConnectionAdapters::TransactionManager + .prepend(TransactionManagerContext) + ActiveRecord::ConnectionAdapters::RealTransaction + .prepend(RealTransactionContext) + end + # MonkeyPatch for ActiveRecord::Base for adding observability module ActiveRecordBaseTransactionMetrics extend ActiveSupport::Concern @@ -407,6 +216,32 @@ module Gitlab end end end + + # rubocop:disable Gitlab/ModuleWithInstanceVariables + module TransactionManagerContext + def transaction_context + @stack.first.try(:gitlab_transaction_context) + end + end + + module RealTransactionContext + def gitlab_transaction_context + @gitlab_transaction_context ||= ::Gitlab::Database::Transaction::Context.new + end + + def commit + gitlab_transaction_context.commit + + super + end + + def rollback + gitlab_transaction_context.rollback + + super + end + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables end end diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index eda991efbd5..07809c5b592 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -19,7 +19,7 @@ module Gitlab # Note: to be deleted after the minimum PG version is set to 12.0 def self.materialized_supported? strong_memoize(:materialized_supported) do - Gitlab::Database.version.match?(/^1[2-9]\./) # version 12.x and above + Gitlab::Database.main.version.match?(/^1[2-9]\./) # version 12.x and above end end diff --git a/lib/gitlab/database/async_indexes.rb b/lib/gitlab/database/async_indexes.rb new file mode 100644 index 00000000000..d89d5238356 --- /dev/null +++ b/lib/gitlab/database/async_indexes.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncIndexes + DEFAULT_INDEXES_PER_INVOCATION = 2 + + def self.create_pending_indexes!(how_many: DEFAULT_INDEXES_PER_INVOCATION) + PostgresAsyncIndex.order(:id).limit(how_many).each do |async_index| + IndexCreator.new(async_index).perform + end + end + end + end +end diff --git a/lib/gitlab/database/async_indexes/index_creator.rb b/lib/gitlab/database/async_indexes/index_creator.rb new file mode 100644 index 00000000000..00de79ec970 --- /dev/null +++ b/lib/gitlab/database/async_indexes/index_creator.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncIndexes + class IndexCreator + include ExclusiveLeaseGuard + + TIMEOUT_PER_ACTION = 1.day + STATEMENT_TIMEOUT = 9.hours + + def initialize(async_index) + @async_index = async_index + end + + def perform + try_obtain_lease do + if index_exists? + log_index_info('Skipping index creation as the index exists') + else + log_index_info('Creating async index') + + set_statement_timeout do + connection.execute(async_index.definition) + end + + log_index_info('Finished creating async index') + end + + async_index.destroy + end + end + + private + + attr_reader :async_index + + def index_exists? + connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name } + end + + def connection + @connection ||= ApplicationRecord.connection + end + + def lease_timeout + TIMEOUT_PER_ACTION + end + + def set_statement_timeout + connection.execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT) + yield + ensure + connection.execute('RESET statement_timeout') + end + + def log_index_info(message) + Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name) + end + end + end + end +end diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb new file mode 100644 index 00000000000..dff6376270a --- /dev/null +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncIndexes + module MigrationHelpers + def unprepare_async_index(table_name, column_name, **options) + return unless async_index_creation_available? + + index_name = options[:name] || index_name(table_name, column_name) + + raise 'Specifying index name is mandatory - specify name: argument' unless index_name + + unprepare_async_index_by_name(table_name, index_name) + end + + def unprepare_async_index_by_name(table_name, index_name, **options) + return unless async_index_creation_available? + + PostgresAsyncIndex.find_by(name: index_name).try do |async_index| + async_index.destroy + end + end + + # Prepares an index for asynchronous creation. + # + # Stores the index information in the postgres_async_indexes table to be created later. The + # index will be always be created CONCURRENTLY, so that option does not need to be given. + # If an existing asynchronous definition exists with the same name, the existing entry will be + # updated with the new definition. + # + # If the requested index has already been created, it is not stored in the table for + # asynchronous creation. + def prepare_async_index(table_name, column_name, **options) + return unless async_index_creation_available? + + index_name = options[:name] || index_name(table_name, column_name) + + raise 'Specifying index name is mandatory - specify name: argument' unless index_name + + options = options.merge({ algorithm: :concurrently }) + + if index_exists?(table_name, column_name, **options) + Gitlab::AppLogger.warn( + message: 'Index not prepared because it already exists', + table_name: table_name, + index_name: index_name) + + return + end + + index, algorithm, if_not_exists = add_index_options(table_name, column_name, **options) + + create_index = ActiveRecord::ConnectionAdapters::CreateIndexDefinition.new(index, algorithm, if_not_exists) + schema_creation = ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaCreation.new(ApplicationRecord.connection) + definition = schema_creation.accept(create_index) + + async_index = PostgresAsyncIndex.safe_find_or_create_by!(name: index_name) do |rec| + rec.table_name = table_name + rec.definition = definition + end + + Gitlab::AppLogger.info( + message: 'Prepared index for async creation', + table_name: async_index.table_name, + index_name: async_index.name) + + async_index + end + + private + + def async_index_creation_available? + ApplicationRecord.connection.table_exists?(:postgres_async_indexes) && + Feature.enabled?(:database_async_index_creation, type: :ops) + end + end + end + end +end diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb new file mode 100644 index 00000000000..236459e6216 --- /dev/null +++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncIndexes + class PostgresAsyncIndex < ApplicationRecord + self.table_name = 'postgres_async_indexes' + + MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH + MAX_DEFINITION_LENGTH = 2048 + + validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH } + + def to_s + definition + end + end + end + end +end diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index 5f2e404c9da..7efa5b46ecb 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -31,7 +31,7 @@ module Gitlab end def count(batch_size: nil, mode: :itself, start: nil, finish: nil) - raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? + raise 'BatchCount can not be run inside a transaction' if @relation.connection.transaction_open? check_mode!(mode) diff --git a/lib/gitlab/database/connection.rb b/lib/gitlab/database/connection.rb new file mode 100644 index 00000000000..21861e4fba8 --- /dev/null +++ b/lib/gitlab/database/connection.rb @@ -0,0 +1,249 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # Configuration settings and methods for interacting with a PostgreSQL + # database, with support for multiple databases. + class Connection + DEFAULT_POOL_HEADROOM = 10 + + attr_reader :scope + + # Initializes a new `Database`. + # + # The `scope` argument must be an object (such as `ActiveRecord::Base`) + # that supports retrieving connections and connection pools. + def initialize(scope = ActiveRecord::Base) + @config = nil + @scope = scope + @version = nil + @open_transactions_baseline = 0 + end + + # We configure the database connection pool size automatically based on + # the configured concurrency. We also add some headroom, to make sure we + # don't run out of connections when more threads besides the 'user-facing' + # ones are running. + # + # Read more about this in + # doc/development/database/client_side_connection_pool.md + def default_pool_size + headroom = + (ENV["DB_POOL_HEADROOM"].presence || DEFAULT_POOL_HEADROOM).to_i + + Gitlab::Runtime.max_threads + headroom + end + + def config + # The result of this method must not be cached, as other methods may use + # it after making configuration changes and expect those changes to be + # present. For example, `disable_prepared_statements` expects the + # configuration settings to always be up to date. + # + # See the following for more information: + # + # - https://gitlab.com/gitlab-org/release/retrospectives/-/issues/39 + # - https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5238 + scope.connection_db_config.configuration_hash.with_indifferent_access + end + + def pool_size + config[:pool] || default_pool_size + end + + def username + config[:username] || ENV['USER'] + end + + def database_name + config[:database] + end + + def adapter_name + config[:adapter] + end + + def human_adapter_name + if postgresql? + 'PostgreSQL' + else + 'Unknown' + end + end + + def postgresql? + adapter_name.casecmp('postgresql') == 0 + end + + def db_config_with_default_pool_size + db_config_object = scope.connection_db_config + config = db_config_object.configuration_hash.merge(pool: default_pool_size) + + ActiveRecord::DatabaseConfigurations::HashConfig.new( + db_config_object.env_name, + db_config_object.name, + config + ) + end + + # Disables prepared statements for the current database connection. + def disable_prepared_statements + scope.establish_connection(config.merge(prepared_statements: false)) + end + + # Check whether the underlying database is in read-only mode + def db_read_only? + pg_is_in_recovery = + scope + .connection + .execute('SELECT pg_is_in_recovery()') + .first + .fetch('pg_is_in_recovery') + + Gitlab::Utils.to_boolean(pg_is_in_recovery) + end + + def db_read_write? + !db_read_only? + end + + def version + @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] + end + + def database_version + connection.execute("SELECT VERSION()").first['version'] + end + + def postgresql_minimum_supported_version? + version.to_f >= MINIMUM_POSTGRES_VERSION + end + + # Bulk inserts a number of rows into a table, optionally returning their + # IDs. + # + # table - The name of the table to insert the rows into. + # rows - An Array of Hash instances, each mapping the columns to their + # values. + # return_ids - When set to true the return value will be an Array of IDs of + # the inserted rows + # disable_quote - A key or an Array of keys to exclude from quoting (You + # become responsible for protection from SQL injection for + # these keys!) + # on_conflict - Defines an upsert. Values can be: :disabled (default) or + # :do_nothing + def bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) + return if rows.empty? + + keys = rows.first.keys + columns = keys.map { |key| connection.quote_column_name(key) } + + disable_quote = Array(disable_quote).to_set + tuples = rows.map do |row| + keys.map do |k| + disable_quote.include?(k) ? row[k] : connection.quote(row[k]) + end + end + + sql = <<-EOF + INSERT INTO #{table} (#{columns.join(', ')}) + VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + EOF + + sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing + + sql = "#{sql} RETURNING id" if return_ids + + result = connection.execute(sql) + + if return_ids + result.values.map { |tuple| tuple[0].to_i } + else + [] + end + end + + def cached_column_exists?(table_name, column_name) + connection + .schema_cache.columns_hash(table_name) + .has_key?(column_name.to_s) + end + + def cached_table_exists?(table_name) + exists? && connection.schema_cache.data_source_exists?(table_name) + end + + def exists? + connection + + true + rescue StandardError + false + end + + def system_id + row = connection + .execute('SELECT system_identifier FROM pg_control_system()') + .first + + row['system_identifier'] + end + + # @param [ActiveRecord::Connection] ar_connection + # @return [String] + def get_write_location(ar_connection) + use_new_load_balancer_query = Gitlab::Utils + .to_boolean(ENV['USE_NEW_LOAD_BALANCER_QUERY'], default: true) + + sql = + if use_new_load_balancer_query + <<~NEWSQL + SELECT CASE + WHEN pg_is_in_recovery() = true AND EXISTS (SELECT 1 FROM pg_stat_get_wal_senders()) + THEN pg_last_wal_replay_lsn()::text + WHEN pg_is_in_recovery() = false + THEN pg_current_wal_insert_lsn()::text + ELSE NULL + END AS location; + NEWSQL + else + <<~SQL + SELECT pg_current_wal_insert_lsn()::text AS location + SQL + end + + row = ar_connection.select_all(sql).first + row['location'] if row + end + + # inside_transaction? will return true if the caller is running within a + # transaction. Handles special cases when running inside a test + # environment, where tests may be wrapped in transactions + def inside_transaction? + base = Rails.env.test? ? @open_transactions_baseline : 0 + + scope.connection.open_transactions > base + end + + # These methods that access @open_transactions_baseline are not + # thread-safe. These are fine though because we only call these in + # RSpec's main thread. If we decide to run specs multi-threaded, we would + # need to use something like ThreadGroup to keep track of this value + def set_open_transactions_baseline + @open_transactions_baseline = scope.connection.open_transactions + end + + def reset_open_transactions_baseline + @open_transactions_baseline = 0 + end + + private + + def connection + scope.connection + end + end + end +end + +Gitlab::Database::Connection.prepend_mod_with('Gitlab::Database::Connection') diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index a7bfafe2815..870cf25984b 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -54,7 +54,7 @@ module Gitlab # Querying tuple stats only works on the primary. Due to load balancing, the # easiest way to do this is to start a transaction. - ActiveRecord::Base.transaction do + ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases get_statistics(non_sti_table_names, check_statistics: check_statistics).each_with_object({}) do |row, data| model = table_to_model[row.table_name] data[model] = row.estimate diff --git a/lib/gitlab/database/count/tablesample_count_strategy.rb b/lib/gitlab/database/count/tablesample_count_strategy.rb index e9387a91a14..489bc0aacea 100644 --- a/lib/gitlab/database/count/tablesample_count_strategy.rb +++ b/lib/gitlab/database/count/tablesample_count_strategy.rb @@ -61,7 +61,7 @@ module Gitlab #{where_clause(model)} SQL - rows = ActiveRecord::Base.connection.select_all(query) + rows = ActiveRecord::Base.connection.select_all(query) # rubocop: disable Database/MultipleDatabases Integer(rows.first['count']) end diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index 7774dd9fffe..c8a30c68bc6 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -10,7 +10,7 @@ module Gitlab # We _must not_ use quote_table_name as this will produce double # quotes on PostgreSQL and for "has_table_privilege" we need single # quotes. - connection = ActiveRecord::Base.connection + connection = ActiveRecord::Base.connection # rubocop: disable Database/MultipleDatabases quoted_table = connection.quote(table) begin diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 31d41a6d6c0..08f108eb8e4 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -23,7 +23,7 @@ module Gitlab # The connection proxy to use for load balancing (if enabled). def self.proxy - unless @proxy + unless load_balancing_proxy = ActiveRecord::Base.load_balancing_proxy Gitlab::ErrorTracking.track_exception( ProxyNotConfiguredError.new( "Attempting to access the database load balancing proxy, but it wasn't configured.\n" \ @@ -31,12 +31,12 @@ module Gitlab )) end - @proxy + load_balancing_proxy end # Returns a Hash containing the load balancing configuration. def self.configuration - Gitlab::Database.config[:load_balancing] || {} + Gitlab::Database.main.config[:load_balancing] || {} end # Returns the maximum replica lag size in bytes. @@ -79,7 +79,7 @@ module Gitlab end def self.pool_size - Gitlab::Database.config[:pool] + Gitlab::Database.main.pool_size end # Returns true if load balancing is to be enabled. @@ -107,12 +107,12 @@ module Gitlab # Configures proxying of requests. def self.configure_proxy(proxy = ConnectionProxy.new(hosts)) - @proxy = proxy + ActiveRecord::Base.load_balancing_proxy = proxy - # This hijacks the "connection" method to ensure both - # `ActiveRecord::Base.connection` and all models use the same load - # balancing proxy. - ActiveRecord::Base.singleton_class.prepend(ActiveRecordProxy) + # Populate service discovery immediately if it is configured + if service_discovery_enabled? + ServiceDiscovery.new(service_discovery_configuration).perform_service_discovery + end end def self.active_record_models @@ -132,9 +132,22 @@ module Gitlab # recognize the connection, this method returns the primary role # directly. In future, we may need to check for other sources. def self.db_role_for_connection(connection) - return ROLE_PRIMARY if !enable? || @proxy.blank? + return ROLE_UNKNOWN unless connection + + # The connection proxy does not have a role assigned + # as this is dependent on a execution context + return ROLE_UNKNOWN if connection.is_a?(ConnectionProxy) - proxy.load_balancer.db_role_for_connection(connection) + # During application init we might receive `NullPool` + return ROLE_UNKNOWN unless connection.respond_to?(:pool) && + connection.pool.respond_to?(:db_config) && + connection.pool.db_config.respond_to?(:name) + + if connection.pool.db_config.name.ends_with?(LoadBalancer::REPLICA_SUFFIX) + ROLE_REPLICA + else + ROLE_PRIMARY + end end end end diff --git a/lib/gitlab/database/load_balancing/active_record_proxy.rb b/lib/gitlab/database/load_balancing/active_record_proxy.rb index 7763497e770..deaea62d774 100644 --- a/lib/gitlab/database/load_balancing/active_record_proxy.rb +++ b/lib/gitlab/database/load_balancing/active_record_proxy.rb @@ -7,7 +7,7 @@ module Gitlab # "connection" method. module ActiveRecordProxy def connection - LoadBalancing.proxy + ::Gitlab::Database::LoadBalancing.proxy end end end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 3a09689a724..938f4951532 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -41,31 +41,31 @@ module Gitlab def select_all(arel, name = nil, binds = [], preparable: nil) if arel.respond_to?(:locked) && arel.locked # SELECT ... FOR UPDATE queries should be sent to the primary. - write_using_load_balancer(:select_all, [arel, name, binds], + write_using_load_balancer(:select_all, arel, name, binds, sticky: true) else - read_using_load_balancer(:select_all, [arel, name, binds]) + read_using_load_balancer(:select_all, arel, name, binds) end end NON_STICKY_READS.each do |name| - define_method(name) do |*args, &block| - read_using_load_balancer(name, args, &block) + define_method(name) do |*args, **kwargs, &block| + read_using_load_balancer(name, *args, **kwargs, &block) end end STICKY_WRITES.each do |name| - define_method(name) do |*args, &block| - write_using_load_balancer(name, args, sticky: true, &block) + define_method(name) do |*args, **kwargs, &block| + write_using_load_balancer(name, *args, sticky: true, **kwargs, &block) end end - def transaction(*args, &block) + def transaction(*args, **kwargs, &block) if current_session.fallback_to_replicas_for_ambiguous_queries? track_read_only_transaction! - read_using_load_balancer(:transaction, args, &block) + read_using_load_balancer(:transaction, *args, **kwargs, &block) else - write_using_load_balancer(:transaction, args, sticky: true, &block) + write_using_load_balancer(:transaction, *args, sticky: true, **kwargs, &block) end ensure @@ -73,26 +73,26 @@ module Gitlab end # Delegates all unknown messages to a read-write connection. - def method_missing(name, *args, &block) + def method_missing(...) if current_session.fallback_to_replicas_for_ambiguous_queries? - read_using_load_balancer(name, args, &block) + read_using_load_balancer(...) else - write_using_load_balancer(name, args, &block) + write_using_load_balancer(...) end end # Performs a read using the load balancer. # # name - The name of the method to call on a connection object. - def read_using_load_balancer(name, args, &block) + def read_using_load_balancer(...) if current_session.use_primary? && !current_session.use_replicas_for_read_queries? @load_balancer.read_write do |connection| - connection.send(name, *args, &block) + connection.send(...) end else @load_balancer.read do |connection| - connection.send(name, *args, &block) + connection.send(...) end end end @@ -102,7 +102,7 @@ module Gitlab # name - The name of the method to call on a connection object. # sticky - If set to true the session will stick to the master after # the write. - def write_using_load_balancer(name, args, sticky: false, &block) + def write_using_load_balancer(name, *args, sticky: false, **kwargs, &block) if read_only_transaction? raise WriteInsideReadOnlyTransactionError, 'A write query is performed inside a read-only transaction' end @@ -113,7 +113,7 @@ module Gitlab # secondary instead of on a primary (when necessary). current_session.write! if sticky - connection.send(name, *args, &block) + connection.send(name, *args, **kwargs, &block) end end diff --git a/lib/gitlab/database/load_balancing/host.rb b/lib/gitlab/database/load_balancing/host.rb index 3e74b5ea727..4c5357ae8e3 100644 --- a/lib/gitlab/database/load_balancing/host.rb +++ b/lib/gitlab/database/load_balancing/host.rb @@ -29,11 +29,11 @@ module Gitlab @host = host @port = port @load_balancer = load_balancer - @pool = Database.create_connection_pool(LoadBalancing.pool_size, host, port) + @pool = load_balancer.create_replica_connection_pool(::Gitlab::Database::LoadBalancing.pool_size, host, port) @online = true @last_checked_at = Time.zone.now - interval = LoadBalancing.replica_check_interval + interval = ::Gitlab::Database::LoadBalancing.replica_check_interval @intervals = (interval..(interval * 2)).step(0.5).to_a end @@ -41,10 +41,10 @@ module Gitlab # # timeout - The time after which the pool should be forcefully # disconnected. - def disconnect!(timeout = 120) - start_time = Metrics::System.monotonic_time + def disconnect!(timeout: 120) + start_time = ::Gitlab::Metrics::System.monotonic_time - while (Metrics::System.monotonic_time - start_time) <= timeout + while (::Gitlab::Metrics::System.monotonic_time - start_time) <= timeout break if pool.connections.none?(&:in_use?) sleep(2) @@ -54,7 +54,7 @@ module Gitlab end def offline! - LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Logger.warn( event: :host_offline, message: 'Marking host as offline', db_host: @host, @@ -72,14 +72,14 @@ module Gitlab refresh_status if @online - LoadBalancing::Logger.info( + ::Gitlab::Database::LoadBalancing::Logger.info( event: :host_online, message: 'Host is online after replica status check', db_host: @host, db_port: @port ) else - LoadBalancing::Logger.warn( + ::Gitlab::Database::LoadBalancing::Logger.warn( event: :host_offline, message: 'Host is offline after replica status check', db_host: @host, @@ -108,7 +108,7 @@ module Gitlab def replication_lag_below_threshold? if (lag_time = replication_lag_time) - lag_time <= LoadBalancing.max_replication_lag_time + lag_time <= ::Gitlab::Database::LoadBalancing.max_replication_lag_time else false end @@ -125,7 +125,7 @@ module Gitlab # only do this if we haven't replicated in a while so we only need # to connect to the primary when truly necessary. if (lag_size = replication_lag_size) - lag_size <= LoadBalancing.max_replication_difference + lag_size <= ::Gitlab::Database::LoadBalancing.max_replication_difference else false end diff --git a/lib/gitlab/database/load_balancing/host_list.rb b/lib/gitlab/database/load_balancing/host_list.rb index 24800012947..aa731521732 100644 --- a/lib/gitlab/database/load_balancing/host_list.rb +++ b/lib/gitlab/database/load_balancing/host_list.rb @@ -8,13 +8,11 @@ module Gitlab # hosts - The list of secondary hosts to add. def initialize(hosts = []) @hosts = hosts.shuffle - @pools = Set.new @index = 0 @mutex = Mutex.new @hosts_gauge = Gitlab::Metrics.gauge(:db_load_balancing_hosts, 'Current number of load balancing hosts') set_metrics! - update_pools end def hosts @@ -35,15 +33,16 @@ module Gitlab @mutex.synchronize { @hosts.map { |host| [host.host, host.port] } } end - def manage_pool?(pool) - @pools.include?(pool) - end - def hosts=(hosts) @mutex.synchronize do + ::Gitlab::Database::LoadBalancing::Logger.info( + event: :host_list_update, + message: "Updating the host list for service discovery", + host_list_length: hosts.length, + old_host_list_length: @hosts.length + ) @hosts = hosts unsafe_shuffle - update_pools end set_metrics! @@ -89,10 +88,6 @@ module Gitlab def set_metrics! @hosts_gauge.set({}, @hosts.length) end - - def update_pools - @pools = Set.new(@hosts.map(&:pool)) - end end end end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index a5d67ebc050..e3f5d0ac470 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -7,20 +7,21 @@ module Gitlab # # Each host in the load balancer uses the same credentials as the primary # database. - # - # This class *requires* that `ActiveRecord::Base.retrieve_connection` - # always returns a connection to the primary. class LoadBalancer CACHE_KEY = :gitlab_load_balancer_host - VALID_HOSTS_CACHE_KEY = :gitlab_load_balancer_valid_hosts + + REPLICA_SUFFIX = '_replica' attr_reader :host_list # hosts - The hostnames/addresses of the additional databases. - def initialize(hosts = []) + def initialize(hosts = [], model = ActiveRecord::Base) + @model = model @host_list = HostList.new(hosts.map { |addr| Host.new(addr, self) }) - @connection_db_roles = {}.compare_by_identity - @connection_db_roles_count = {}.compare_by_identity + end + + def disconnect!(timeout: 120) + host_list.hosts.each { |host| host.disconnect!(timeout: timeout) } end # Yields a connection that can be used for reads. @@ -28,7 +29,6 @@ module Gitlab # If no secondaries were available this method will use the primary # instead. def read(&block) - connection = nil conflict_retried = 0 while host @@ -36,12 +36,8 @@ module Gitlab begin connection = host.connection - track_connection_role(connection, ROLE_REPLICA) - return yield connection rescue StandardError => error - untrack_connection_role(connection) - if serialization_failure?(error) # This error can occur when a query conflicts. See # https://www.postgresql.org/docs/current/static/hot-standby.html#HOT-STANDBY-CONFLICT @@ -84,8 +80,6 @@ module Gitlab ) read_write(&block) - ensure - untrack_connection_role(connection) end # Yields a connection that can be used for both reads and writes. @@ -95,22 +89,9 @@ module Gitlab # Instead of immediately grinding to a halt we'll retry the operation # a few times. retry_with_backoff do - connection = ActiveRecord::Base.retrieve_connection - track_connection_role(connection, ROLE_PRIMARY) - + connection = pool.connection yield connection end - ensure - untrack_connection_role(connection) - end - - # Recognize the role (primary/replica) of the database this connection - # is connecting to. If the connection is not issued by this load - # balancer, return nil - def db_role_for_connection(connection) - return @connection_db_roles[connection] if @connection_db_roles[connection] - return ROLE_REPLICA if @host_list.manage_pool?(connection.pool) - return ROLE_PRIMARY if connection.pool == ActiveRecord::Base.connection_pool end # Returns a host to use for queries. @@ -118,28 +99,27 @@ module Gitlab # Hosts are scoped per thread so that multiple threads don't # accidentally re-use the same host + connection. def host - RequestStore[CACHE_KEY] ||= current_host_list.next + request_cache[CACHE_KEY] ||= @host_list.next end # Releases the host and connection for the current thread. def release_host - if host = RequestStore[CACHE_KEY] + if host = request_cache[CACHE_KEY] host.disable_query_cache! host.release_connection end - RequestStore.delete(CACHE_KEY) - RequestStore.delete(VALID_HOSTS_CACHE_KEY) + request_cache.delete(CACHE_KEY) end def release_primary_connection - ActiveRecord::Base.connection_pool.release_connection + pool.release_connection end # Returns the transaction write location of the primary. def primary_write_location location = read_write do |connection| - ::Gitlab::Database.get_write_location(connection) + ::Gitlab::Database.main.get_write_location(connection) end return location if location @@ -148,55 +128,17 @@ module Gitlab end # Returns true if there was at least one host that has caught up with the given transaction. - # - # In case of a retry, this method also stores the set of hosts that have caught up. - # - # UPD: `select_caught_up_hosts` seems to have redundant logic managing host list (`:gitlab_load_balancer_valid_hosts`), - # while we only need a single host: https://gitlab.com/gitlab-org/gitlab/-/issues/326125#note_615271604 - # Also, shuffling the list afterwards doesn't seem to be necessary. - # This may be improved by merging this method with `select_up_to_date_host`. - # Could be removed when `:load_balancing_refine_load_balancer_methods` FF is rolled out - def select_caught_up_hosts(location) - all_hosts = @host_list.hosts - valid_hosts = all_hosts.select { |host| host.caught_up?(location) } - - return false if valid_hosts.empty? - - # Hosts can come online after the time when this scan was done, - # so we need to remember the ones that can be used. If the host went - # offline, we'll just rely on the retry mechanism to use the primary. - set_consistent_hosts_for_request(HostList.new(valid_hosts)) - - # Since we will be using a subset from the original list, let's just - # pick a random host and mix up the original list to ensure we don't - # only end up using one replica. - RequestStore[CACHE_KEY] = valid_hosts.sample - @host_list.shuffle - - true - end - - # Returns true if there was at least one host that has caught up with the given transaction. - # Similar to `#select_caught_up_hosts`, picks a random host, to rotate replicas we use. - # Unlike `#select_caught_up_hosts`, does not iterate over all hosts if finds any. - # - # It is going to be merged with `select_caught_up_hosts`, because they intend to do the same. def select_up_to_date_host(location) all_hosts = @host_list.hosts.shuffle host = all_hosts.find { |host| host.caught_up?(location) } return false unless host - RequestStore[CACHE_KEY] = host + request_cache[CACHE_KEY] = host true end - # Could be removed when `:load_balancing_refine_load_balancer_methods` FF is rolled out - def set_consistent_hosts_for_request(hosts) - RequestStore[VALID_HOSTS_CACHE_KEY] = hosts - end - # Yields a block, retrying it upon error using an exponential backoff. def retry_with_backoff(retries = 3, time = 2) retried = 0 @@ -247,30 +189,50 @@ module Gitlab end end - private + # pool_size - The size of the DB pool. + # host - An optional host name to use instead of the default one. + # port - An optional port to connect to. + def create_replica_connection_pool(pool_size, host = nil, port = nil) + db_config = pool.db_config - def ensure_caching! - host.enable_query_cache! unless host.query_cache_enabled - end + env_config = db_config.configuration_hash.dup + env_config[:pool] = pool_size + env_config[:host] = host if host + env_config[:port] = port if port + + replica_db_config = ActiveRecord::DatabaseConfigurations::HashConfig.new( + db_config.env_name, + db_config.name + REPLICA_SUFFIX, + env_config + ) - def track_connection_role(connection, role) - @connection_db_roles[connection] = role - @connection_db_roles_count[connection] ||= 0 - @connection_db_roles_count[connection] += 1 + # We cannot use ActiveRecord::Base.connection_handler.establish_connection + # as it will rewrite ActiveRecord::Base.connection + ActiveRecord::ConnectionAdapters::ConnectionHandler + .new + .establish_connection(replica_db_config) end - def untrack_connection_role(connection) - return if connection.blank? || @connection_db_roles_count[connection].blank? + private - @connection_db_roles_count[connection] -= 1 - if @connection_db_roles_count[connection] <= 0 - @connection_db_roles.delete(connection) - @connection_db_roles_count.delete(connection) - end + # ActiveRecord::ConnectionAdapters::ConnectionHandler handles fetching, + # and caching for connections pools for each "connection", so we + # leverage that. + def pool + ActiveRecord::Base.connection_handler.retrieve_connection_pool( + @model.connection_specification_name, + role: ActiveRecord::Base.writing_role, + shard: ActiveRecord::Base.default_shard + ) + end + + def ensure_caching! + host.enable_query_cache! unless host.query_cache_enabled end - def current_host_list - RequestStore[VALID_HOSTS_CACHE_KEY] || @host_list + def request_cache + base = RequestStore[:gitlab_load_balancer] ||= {} + base[pool] ||= {} end end end diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb index 8e7e6865402..f8a31622b7d 100644 --- a/lib/gitlab/database/load_balancing/rack_middleware.rb +++ b/lib/gitlab/database/load_balancing/rack_middleware.rb @@ -18,9 +18,9 @@ module Gitlab # namespace - The namespace to use for sticking. # id - The identifier to use for sticking. def self.stick_or_unstick(env, namespace, id) - return unless LoadBalancing.enable? + return unless ::Gitlab::Database::LoadBalancing.enable? - Sticking.unstick_or_continue_sticking(namespace, id) + ::Gitlab::Database::LoadBalancing::Sticking.unstick_or_continue_sticking(namespace, id) env[STICK_OBJECT] ||= Set.new env[STICK_OBJECT] << [namespace, id] @@ -56,7 +56,7 @@ module Gitlab namespaces_and_ids = sticking_namespaces_and_ids(env) namespaces_and_ids.each do |namespace, id| - Sticking.unstick_or_continue_sticking(namespace, id) + ::Gitlab::Database::LoadBalancing::Sticking.unstick_or_continue_sticking(namespace, id) end end @@ -65,17 +65,17 @@ module Gitlab namespaces_and_ids = sticking_namespaces_and_ids(env) namespaces_and_ids.each do |namespace, id| - Sticking.stick_if_necessary(namespace, id) + ::Gitlab::Database::LoadBalancing::Sticking.stick_if_necessary(namespace, id) end end def clear load_balancer.release_host - Session.clear_session + ::Gitlab::Database::LoadBalancing::Session.clear_session end def load_balancer - LoadBalancing.proxy.load_balancer + ::Gitlab::Database::LoadBalancing.proxy.load_balancer end # Determines the sticking namespace and identifier based on the Rack diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 9b42b25be1c..251961c8246 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -13,7 +13,8 @@ module Gitlab # balancer with said hosts. Requests may continue to use the old hosts # until they complete. class ServiceDiscovery - attr_reader :interval, :record, :record_type, :disconnect_timeout + attr_reader :interval, :record, :record_type, :disconnect_timeout, + :load_balancer MAX_SLEEP_ADJUSTMENT = 10 @@ -40,7 +41,17 @@ module Gitlab # disconnect_timeout - The time after which an old host should be # forcefully disconnected. # use_tcp - Use TCP instaed of UDP to look up resources - def initialize(nameserver:, port:, record:, record_type: 'A', interval: 60, disconnect_timeout: 120, use_tcp: false) + # load_balancer - The load balancer instance to use + def initialize( + nameserver:, + port:, + record:, + record_type: 'A', + interval: 60, + disconnect_timeout: 120, + use_tcp: false, + load_balancer: LoadBalancing.proxy.load_balancer + ) @nameserver = nameserver @port = port @record = record @@ -48,34 +59,36 @@ module Gitlab @interval = interval @disconnect_timeout = disconnect_timeout @use_tcp = use_tcp + @load_balancer = load_balancer end def start Thread.new do loop do - interval = - begin - refresh_if_necessary - rescue StandardError => error - # Any exceptions that might occur should be reported to - # Sentry, instead of silently terminating this thread. - Gitlab::ErrorTracking.track_exception(error) - - Gitlab::AppLogger.error( - "Service discovery encountered an error: #{error.message}" - ) - - self.interval - end + next_sleep_duration = perform_service_discovery # We slightly randomize the sleep() interval. This should reduce # the likelihood of _all_ processes refreshing at the same time, # possibly putting unnecessary pressure on the DNS server. - sleep(interval + rand(MAX_SLEEP_ADJUSTMENT)) + sleep(next_sleep_duration + rand(MAX_SLEEP_ADJUSTMENT)) end end end + def perform_service_discovery + refresh_if_necessary + rescue StandardError => error + # Any exceptions that might occur should be reported to + # Sentry, instead of silently terminating this thread. + Gitlab::ErrorTracking.track_exception(error) + + Gitlab::AppLogger.error( + "Service discovery encountered an error: #{error.message}" + ) + + interval + end + # Refreshes the hosts, but only if the DNS record returned a new list of # addresses. # @@ -108,7 +121,7 @@ module Gitlab # host/connection. While this connection will be checked in and out, # it won't be explicitly disconnected. old_hosts.each do |host| - host.disconnect!(disconnect_timeout) + host.disconnect!(timeout: disconnect_timeout) end end @@ -147,10 +160,6 @@ module Gitlab end.sort end - def load_balancer - LoadBalancing.proxy.load_balancer - end - def resolver @resolver ||= Net::DNS::Resolver.new( nameservers: Resolver.new(@nameserver).resolve, diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index 8e1aa079216..20d42b9a694 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -53,14 +53,8 @@ module Gitlab # write location. If no such location exists, err on the side of caution. return false unless location - if ::Feature.enabled?(:load_balancing_refine_load_balancer_methods) - load_balancer.select_up_to_date_host(location).tap do |selected| - unstick(namespace, id) if selected - end - else - load_balancer.select_caught_up_hosts(location).tap do |selected| - unstick(namespace, id) if selected - end + load_balancer.select_up_to_date_host(location).tap do |selected| + unstick(namespace, id) if selected end end @@ -109,7 +103,7 @@ module Gitlab if LoadBalancing.enable? load_balancer.primary_write_location else - Gitlab::Database.get_write_location(ActiveRecord::Base.connection) + Gitlab::Database.main.get_write_location(ActiveRecord::Base.connection) end return if location.blank? diff --git a/lib/gitlab/database/metrics.rb b/lib/gitlab/database/metrics.rb new file mode 100644 index 00000000000..5dabbc81b9c --- /dev/null +++ b/lib/gitlab/database/metrics.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class Metrics + extend ::Gitlab::Utils::StrongMemoize + + class << self + def subtransactions_increment(model_name) + subtransactions_counter.increment(model: model_name) + end + + private + + def subtransactions_counter + strong_memoize(:subtransactions_counter) do + name = :gitlab_active_record_subtransactions_total + comment = 'Total amount of subtransactions created by ActiveRecord' + + ::Gitlab::Metrics.counter(name, comment) + end + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 842ab4f7b80..23d9b16dc09 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -6,6 +6,7 @@ module Gitlab include Migrations::BackgroundMigrationHelpers include DynamicModelHelpers include RenameTableHelpers + include AsyncIndexes::MigrationHelpers # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 @@ -152,6 +153,9 @@ module Gitlab disable_statement_timeout do add_index(table_name, column_name, **options) end + + # We created this index. Now let's remove the queuing entry for async creation in case it's still there. + unprepare_async_index(table_name, column_name, **options) end # Removes an existed index, concurrently @@ -178,6 +182,9 @@ module Gitlab disable_statement_timeout do remove_index(table_name, **options.merge({ column: column_name })) end + + # We removed this index. Now let's make sure it's not queued for async creation. + unprepare_async_index(table_name, column_name, **options) end # Removes an existing index, concurrently @@ -208,6 +215,9 @@ module Gitlab disable_statement_timeout do remove_index(table_name, **options.merge({ name: index_name })) end + + # We removed this index. Now let's make sure it's not queued for async creation. + unprepare_async_index_by_name(table_name, index_name, **options) end # Adds a foreign key with only minimal locking on the tables involved. @@ -221,8 +231,13 @@ module Gitlab # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". # name - The name of the foreign key. + # validate - Flag that controls whether the new foreign key will be validated after creation. + # If the flag is not set, the constraint will only be enforced for new data. + # reverse_lock_order - Flag that controls whether we should attempt to acquire locks in the reverse + # order of the ALTER TABLE. This can be useful in situations where the foreign + # key creation could deadlock with another process. # - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, target_column: :id, name: nil, validate: true) + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, target_column: :id, name: nil, validate: true, reverse_lock_order: false) # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? @@ -250,6 +265,8 @@ module Gitlab # data. with_lock_retries do + execute("LOCK TABLE #{target}, #{source} IN SHARE ROW EXCLUSIVE MODE") if reverse_lock_order + execute <<-EOF.strip_heredoc ALTER TABLE #{source} ADD CONSTRAINT #{options[:name]} @@ -324,9 +341,9 @@ module Gitlab # - Per connection (requires a cleanup after the execution) # # When using a per connection disable statement, code must be inside - # a block so we can automatically execute `RESET ALL` after block finishes + # a block so we can automatically execute `RESET statement_timeout` after block finishes # otherwise the statement will still be disabled until connection is dropped - # or `RESET ALL` is executed + # or `RESET statement_timeout` is executed def disable_statement_timeout if block_given? if statement_timeout_disabled? @@ -340,7 +357,7 @@ module Gitlab yield ensure - execute('RESET ALL') + execute('RESET statement_timeout') end end else @@ -1248,8 +1265,8 @@ module Gitlab def check_trigger_permissions!(table) unless Grant.create_and_execute_trigger?(table) - dbname = Database.database_name - user = Database.username + dbname = Database.main.database_name + user = Database.main.username raise <<-EOF Your database user is not allowed to create, drop, or execute triggers on the @@ -1569,8 +1586,8 @@ into similar problems in the future (e.g. when new tables are created). def create_extension(extension) execute('CREATE EXTENSION IF NOT EXISTS %s' % extension) rescue ActiveRecord::StatementInvalid => e - dbname = Database.database_name - user = Database.username + dbname = Database.main.database_name + user = Database.main.username warn(<<~MSG) if e.to_s =~ /permission denied/ GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but @@ -1597,8 +1614,8 @@ into similar problems in the future (e.g. when new tables are created). def drop_extension(extension) execute('DROP EXTENSION IF EXISTS %s' % extension) rescue ActiveRecord::StatementInvalid => e - dbname = Database.database_name - user = Database.username + dbname = Database.main.database_name + user = Database.main.username warn(<<~MSG) if e.to_s =~ /permission denied/ This migration attempts to drop the PostgreSQL extension '#{extension}' diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 28491a934a0..19d80ba1d64 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -264,6 +264,34 @@ module Gitlab migration end + # Force a background migration to complete. + # + # WARNING: This method will block the caller and move the background migration from an + # asynchronous migration to a synchronous migration. + # + # 1. Steal work from sidekiq and perform immediately (avoid duplicates generated by step 2). + # 2. Process any pending tracked jobs. + # 3. Steal work from sidekiq and perform immediately (clear anything left from step 2). + # 4. Optionally remove job tracking information. + # + # This method does not garauntee that all jobs completed successfully. + def finalize_background_migration(class_name, delete_tracking_jobs: ['succeeded']) + # Empty the sidekiq queue. + Gitlab::BackgroundMigration.steal(class_name) + + # Process pending tracked jobs. + jobs = Gitlab::Database::BackgroundMigrationJob.pending.for_migration_class(class_name) + jobs.find_each do |job| + BackgroundMigrationWorker.new.perform(job.class_name, job.arguments) + end + + # Empty the sidekiq queue. + Gitlab::BackgroundMigration.steal(class_name) + + # Delete job tracking rows. + delete_job_tracking(class_name, status: delete_tracking_jobs) if delete_tracking_jobs + end + def perform_background_migration_inline? Rails.env.test? || Rails.env.development? end @@ -304,6 +332,12 @@ module Gitlab end end + def delete_job_tracking(class_name, status: 'succeeded') + status = Array(status).map { |s| Gitlab::Database::BackgroundMigrationJob.statuses[s] } + jobs = Gitlab::Database::BackgroundMigrationJob.where(status: status).for_migration_class(class_name) + jobs.each_batch { |batch| batch.delete_all } + end + private def track_in_database(class_name, arguments) diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index e9ef80d5198..d1e55eb825c 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -9,18 +9,20 @@ module Gitlab attr_reader :observations - def initialize(observers = ::Gitlab::Database::Migrations::Observers.all_observers) - @observers = observers + def initialize(observer_classes = ::Gitlab::Database::Migrations::Observers.all_observers) + @observer_classes = observer_classes @observations = [] end - def observe(migration, &block) - observation = Observation.new(migration) + def observe(version:, name:, &block) + observation = Observation.new(version, name) observation.success = true + observers = observer_classes.map { |c| c.new(observation) } + exception = nil - on_each_observer { |observer| observer.before } + on_each_observer(observers) { |observer| observer.before } observation.walltime = Benchmark.realtime do yield @@ -29,8 +31,8 @@ module Gitlab observation.success = false end - on_each_observer { |observer| observer.after } - on_each_observer { |observer| observer.record(observation) } + on_each_observer(observers) { |observer| observer.after } + on_each_observer(observers) { |observer| observer.record } record_observation(observation) @@ -41,13 +43,13 @@ module Gitlab private - attr_reader :observers + attr_reader :observer_classes def record_observation(observation) @observations << observation end - def on_each_observer(&block) + def on_each_observer(observers, &block) observers.each do |observer| yield observer rescue StandardError => e diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index 046843824a4..54eedec3c7b 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -4,7 +4,8 @@ module Gitlab module Database module Migrations Observation = Struct.new( - :migration, + :version, + :name, :walltime, :success, :total_database_size_change, diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb index 979a098d699..140b3feed64 100644 --- a/lib/gitlab/database/migrations/observers.rb +++ b/lib/gitlab/database/migrations/observers.rb @@ -6,10 +6,10 @@ module Gitlab module Observers def self.all_observers [ - TotalDatabaseSizeChange.new, - QueryStatistics.new, - QueryLog.new, - QueryDetails.new + TotalDatabaseSizeChange, + QueryStatistics, + QueryLog, + QueryDetails ] end end diff --git a/lib/gitlab/database/migrations/observers/migration_observer.rb b/lib/gitlab/database/migrations/observers/migration_observer.rb index 9bfbf35887d..85d18abb9ef 100644 --- a/lib/gitlab/database/migrations/observers/migration_observer.rb +++ b/lib/gitlab/database/migrations/observers/migration_observer.rb @@ -5,10 +5,11 @@ module Gitlab module Migrations module Observers class MigrationObserver - attr_reader :connection + attr_reader :connection, :observation - def initialize + def initialize(observation) @connection = ActiveRecord::Base.connection + @observation = observation end def before @@ -19,7 +20,7 @@ module Gitlab # implement in subclass end - def record(observation) + def record raise NotImplementedError, 'implement in subclass' end end diff --git a/lib/gitlab/database/migrations/observers/query_details.rb b/lib/gitlab/database/migrations/observers/query_details.rb index 52b6464d449..dadacd2d2fc 100644 --- a/lib/gitlab/database/migrations/observers/query_details.rb +++ b/lib/gitlab/database/migrations/observers/query_details.rb @@ -6,8 +6,8 @@ module Gitlab module Observers class QueryDetails < MigrationObserver def before - @file_path = File.join(Instrumentation::RESULT_DIR, 'current-details.json') - @file = File.open(@file_path, 'wb') + file_path = File.join(Instrumentation::RESULT_DIR, "#{observation.version}_#{observation.name}-query-details.json") + @file = File.open(file_path, 'wb') @writer = Oj::StreamWriter.new(@file, {}) @writer.push_array @subscriber = ActiveSupport::Notifications.subscribe('sql.active_record') do |*args| @@ -22,8 +22,8 @@ module Gitlab @file.close end - def record(observation) - File.rename(@file_path, File.join(Instrumentation::RESULT_DIR, "#{observation.migration}-query-details.json")) + def record + # no-op end def record_sql_event(_name, started, finished, _unique_id, payload) diff --git a/lib/gitlab/database/migrations/observers/query_log.rb b/lib/gitlab/database/migrations/observers/query_log.rb index 45df07fe391..e15d733d2a2 100644 --- a/lib/gitlab/database/migrations/observers/query_log.rb +++ b/lib/gitlab/database/migrations/observers/query_log.rb @@ -7,8 +7,8 @@ module Gitlab class QueryLog < MigrationObserver def before @logger_was = ActiveRecord::Base.logger - @log_file_path = File.join(Instrumentation::RESULT_DIR, 'current.log') - @logger = Logger.new(@log_file_path) + file_path = File.join(Instrumentation::RESULT_DIR, "#{observation.version}_#{observation.name}.log") + @logger = Logger.new(file_path) ActiveRecord::Base.logger = @logger end @@ -17,8 +17,8 @@ module Gitlab @logger.close end - def record(observation) - File.rename(@log_file_path, File.join(Instrumentation::RESULT_DIR, "#{observation.migration}.log")) + def record + # no-op end end end diff --git a/lib/gitlab/database/migrations/observers/query_statistics.rb b/lib/gitlab/database/migrations/observers/query_statistics.rb index 466f4724256..54504646a79 100644 --- a/lib/gitlab/database/migrations/observers/query_statistics.rb +++ b/lib/gitlab/database/migrations/observers/query_statistics.rb @@ -16,7 +16,7 @@ module Gitlab connection.execute('select pg_stat_statements_reset()') end - def record(observation) + def record return unless enabled? observation.query_statistics = connection.execute(<<~SQL) diff --git a/lib/gitlab/database/migrations/observers/total_database_size_change.rb b/lib/gitlab/database/migrations/observers/total_database_size_change.rb index 0b76b0bef5e..2e89498b79f 100644 --- a/lib/gitlab/database/migrations/observers/total_database_size_change.rb +++ b/lib/gitlab/database/migrations/observers/total_database_size_change.rb @@ -13,7 +13,7 @@ module Gitlab @size_after = get_total_database_size end - def record(observation) + def record return unless @size_after && @size_before observation.total_database_size_change = @size_after - @size_before diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb deleted file mode 100644 index 65a6cb8e369..00000000000 --- a/lib/gitlab/database/multi_threaded_migration.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module MultiThreadedMigration - MULTI_THREAD_AR_CONNECTION = :thread_local_ar_connection - - # This overwrites the default connection method so that every thread can - # use a thread-local connection, while still supporting all of Rails' - # migration methods. - def connection - Thread.current[MULTI_THREAD_AR_CONNECTION] || - ActiveRecord::Base.connection - end - - # Starts a thread-pool for N threads, along with N threads each using a - # single connection. The provided block is yielded from inside each - # thread. - # - # Example: - # - # with_multiple_threads(4) do - # execute('SELECT ...') - # end - # - # thread_count - The number of threads to start. - # - # join - When set to true this method will join the threads, blocking the - # caller until all threads have finished running. - # - # Returns an Array containing the started threads. - def with_multiple_threads(thread_count, join: true) - pool = Gitlab::Database.create_connection_pool(thread_count) - - threads = Array.new(thread_count) do - Thread.new do - pool.with_connection do |connection| - Thread.current[MULTI_THREAD_AR_CONNECTION] = connection - yield - ensure - Thread.current[MULTI_THREAD_AR_CONNECTION] = nil - end - end - end - - threads.each(&:join) if join - - threads - end - end - end -end diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb new file mode 100644 index 00000000000..dc63d93fd07 --- /dev/null +++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true +module Gitlab + module Database + module Partitioning + class DetachedPartitionDropper + def perform + return unless Feature.enabled?(:drop_detached_partitions, default_enabled: :yaml) + + Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop") + Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition| + conn.transaction do + # Another process may have already dropped the table and deleted this entry + next unless (detached_partition = Postgresql::DetachedPartition.lock.find_by(id: detached_partition.id)) + + unless check_partition_detached?(detached_partition) + Gitlab::AppLogger.error(message: "Attempt to drop attached database partition", partition_name: detached_partition.table_name) + detached_partition.destroy! + next + end + + drop_one(detached_partition) + end + rescue StandardError => e + Gitlab::AppLogger.error(message: "Failed to drop previously detached partition", + partition_name: detached_partition.table_name, + exception_class: e.class, + exception_message: e.message) + end + end + + private + + def drop_one(detached_partition) + conn.transaction do + conn.execute(<<~SQL) + DROP TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{conn.quote_table_name(detached_partition.table_name)} + SQL + + detached_partition.destroy! + end + Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: detached_partition.table_name) + end + + def check_partition_detached?(detached_partition) + # PostgresPartition checks the pg_inherits view, so our partition will only show here if it's still attached + # and thus should not be dropped + !PostgresPartition.for_identifier("#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{detached_partition.table_name}").exists? + end + + def conn + @conn ||= ApplicationRecord.connection + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index 4c68399cb68..7992c2fdaa7 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -86,7 +86,7 @@ module Gitlab end def pruning_old_partitions? - Feature.enabled?(:partition_pruning_dry_run) && retain_for.present? + retain_for.present? end def oldest_active_date diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index c2a9422a42a..7e433ecdd39 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -4,6 +4,8 @@ module Gitlab module Database module Partitioning class PartitionManager + UnsafeToDetachPartitionError = Class.new(StandardError) + def self.register(model) raise ArgumentError, "Only models with a #partitioning_strategy can be registered." unless model.respond_to?(:partitioning_strategy) @@ -16,6 +18,7 @@ module Gitlab LEASE_TIMEOUT = 1.minute MANAGEMENT_LEASE_KEY = 'database_partition_management_%s' + RETAIN_DETACHED_PARTITIONS_FOR = 1.week attr_reader :models @@ -35,13 +38,16 @@ module Gitlab partitions_to_create = missing_partitions(model) create(partitions_to_create) unless partitions_to_create.empty? - if Feature.enabled?(:partition_pruning_dry_run) + if Feature.enabled?(:partition_pruning, default_enabled: :yaml) partitions_to_detach = extra_partitions(model) detach(partitions_to_detach) unless partitions_to_detach.empty? end end rescue StandardError => e - Gitlab::AppLogger.error("Failed to create / detach partition(s) for #{model.table_name}: #{e.class}: #{e.message}") + Gitlab::AppLogger.error(message: "Failed to create / detach partition(s)", + table_name: model.table_name, + exception_class: e.class, + exception_message: e.message) end end @@ -54,7 +60,6 @@ module Gitlab end def extra_partitions(model) - return [] unless Feature.enabled?(:partition_pruning_dry_run) return [] unless connection.table_exists?(model.table_name) model.partitioning_strategy.extra_partitions @@ -74,7 +79,9 @@ module Gitlab partitions.each do |partition| connection.execute partition.to_sql - Gitlab::AppLogger.info("Created partition #{partition.partition_name} for table #{partition.table}") + Gitlab::AppLogger.info(message: "Created partition", + partition_name: partition.partition_name, + table_name: partition.table) end end end @@ -89,7 +96,24 @@ module Gitlab end def detach_one_partition(partition) - Gitlab::AppLogger.info("Planning to detach #{partition.partition_name} for table #{partition.table}") + assert_partition_detachable!(partition) + + connection.execute partition.to_detach_sql + + Postgresql::DetachedPartition.create!(table_name: partition.partition_name, + drop_after: RETAIN_DETACHED_PARTITIONS_FOR.from_now) + + Gitlab::AppLogger.info(message: "Detached Partition", + partition_name: partition.partition_name, + table_name: partition.table) + end + + def assert_partition_detachable!(partition) + parent_table_identifier = "#{connection.current_schema}.#{partition.table}" + + if (example_fk = PostgresForeignKey.by_referenced_table_identifier(parent_table_identifier).first) + raise UnsafeToDetachPartitionError, "Cannot detach #{partition.partition_name}, it would block while checking foreign key #{example_fk.name} on #{example_fk.constrained_table_identifier}" + end end def with_lock_retries(&block) diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb index ad122fd47fe..6963ecd2cc1 100644 --- a/lib/gitlab/database/partitioning/partition_monitoring.rb +++ b/lib/gitlab/database/partitioning/partition_monitoring.rb @@ -16,6 +16,7 @@ module Gitlab gauge_present.set({ table: model.table_name }, strategy.current_partitions.size) gauge_missing.set({ table: model.table_name }, strategy.missing_partitions.size) + gauge_extra.set({ table: model.table_name }, strategy.extra_partitions.size) end end @@ -28,6 +29,10 @@ module Gitlab def gauge_missing @gauge_missing ||= Gitlab::Metrics.gauge(:db_partitions_missing, 'Number of database partitions currently expected, but not present') end + + def gauge_extra + @gauge_extra ||= Gitlab::Metrics.gauge(:db_partitions_extra, 'Number of database partitions currently attached to tables, but outside of their retention window and scheduled to be dropped') + end end end end diff --git a/lib/gitlab/database/partitioning/time_partition.rb b/lib/gitlab/database/partitioning/time_partition.rb index 7dca60c0854..1221f042530 100644 --- a/lib/gitlab/database/partitioning/time_partition.rb +++ b/lib/gitlab/database/partitioning/time_partition.rb @@ -47,6 +47,13 @@ module Gitlab SQL end + def to_detach_sql + <<~SQL + ALTER TABLE #{conn.quote_table_name(table)} + DETACH PARTITION #{fully_qualified_partition} + SQL + end + def ==(other) table == other.table && partition_name == other.partition_name && from == other.from && to == other.to end diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb new file mode 100644 index 00000000000..94f74724295 --- /dev/null +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresForeignKey < ApplicationRecord + self.primary_key = :oid + + scope :by_referenced_table_identifier, ->(identifier) do + raise ArgumentError, "Referenced table name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ + + where(referenced_table_identifier: identifier) + end + end + end +end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index 580cab5622d..2e3f674cf82 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -57,7 +57,7 @@ module Gitlab # @param finish final pkey range # @return [Gitlab::Database::PostgresHll::Buckets] HyperLogLog data structure instance that can estimate number of unique elements def execute(batch_size: nil, start: nil, finish: nil) - raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? + raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? # rubocop: disable Database/MultipleDatabases batch_size ||= DEFAULT_BATCH_SIZE start = actual_start(start) diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 58e4e7e7924..1079bfdeda3 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -19,7 +19,12 @@ module Gitlab end # Indexes with reindexing support - scope :reindexing_support, -> { where(partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) } + scope :reindexing_support, -> do + where(partitioned: false, exclusion: false, expression: false, type: Gitlab::Database::Reindexing::SUPPORTED_TYPES) + .not_match("#{Gitlab::Database::Reindexing::ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$") + end + + scope :reindexing_leftovers, -> { match("#{Gitlab::Database::Reindexing::ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$") } scope :not_match, ->(regex) { where("name !~ ?", regex) } diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index 0986372586b..7da60d8375d 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -7,10 +7,14 @@ module Gitlab belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier' - scope :by_identifier, ->(identifier) do + scope :for_identifier, ->(identifier) do raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ - find(identifier) + where(primary_key => identifier) + end + + scope :by_identifier, ->(identifier) do + for_identifier(identifier).first! end scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) } diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 841e04ccbd1..04b409a9306 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -8,6 +8,13 @@ module Gitlab SUPPORTED_TYPES = %w(btree gist).freeze + # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, + # which only conflicts with DDL and vacuum. We therefore execute this with a rather + # high lock timeout and a long pause in between retries. This is an alternative to + # setting a high statement timeout, which would lead to a long running query with effects + # on e.g. vacuum. + REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 + # candidate_indexes: Array of Gitlab::Database::PostgresIndex def self.perform(candidate_indexes, how_many: DEFAULT_INDEXES_PER_INVOCATION) IndexSelection.new(candidate_indexes).take(how_many).each do |index| @@ -15,10 +22,22 @@ module Gitlab end end - def self.candidate_indexes - Gitlab::Database::PostgresIndex - .not_match("#{ReindexConcurrently::TEMPORARY_INDEX_PATTERN}$") - .reindexing_support + def self.cleanup_leftovers! + PostgresIndex.reindexing_leftovers.each do |index| + Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity") + + retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new( + timing_configuration: REMOVE_INDEX_RETRY_CONFIG, + klass: self.class, + logger: Gitlab::AppLogger + ) + + retries.run(raise_on_exhaustion: false) do + ApplicationRecord.connection.tap do |conn| + conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}") + end + end + end end end end diff --git a/lib/gitlab/database/reindexing/reindex_concurrently.rb b/lib/gitlab/database/reindexing/reindex_concurrently.rb index 8d9f9f5abdd..7a720f7c539 100644 --- a/lib/gitlab/database/reindexing/reindex_concurrently.rb +++ b/lib/gitlab/database/reindexing/reindex_concurrently.rb @@ -11,13 +11,6 @@ module Gitlab STATEMENT_TIMEOUT = 9.hours PG_MAX_INDEX_NAME_LENGTH = 63 - # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, - # which only conflicts with DDL and vacuum. We therefore execute this with a rather - # high lock timeout and a long pause in between retries. This is an alternative to - # setting a high statement timeout, which would lead to a long running query with effects - # on e.g. vacuum. - REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 - attr_reader :index, :logger def initialize(index, logger: Gitlab::AppLogger) diff --git a/lib/gitlab/database/schema_migrations/context.rb b/lib/gitlab/database/schema_migrations/context.rb index bd8b9bed2c1..35105121bbd 100644 --- a/lib/gitlab/database/schema_migrations/context.rb +++ b/lib/gitlab/database/schema_migrations/context.rb @@ -6,17 +6,14 @@ module Gitlab class Context attr_reader :connection + DEFAULT_SCHEMA_MIGRATIONS_PATH = "db/schema_migrations" + def initialize(connection) @connection = connection end def schema_directory - @schema_directory ||= - if ActiveRecord::Base.configurations.primary?(database_name) - File.join(db_dir, 'schema_migrations') - else - File.join(db_dir, "#{database_name}_schema_migrations") - end + @schema_directory ||= Rails.root.join(database_schema_migrations_path).to_s end def versions_to_create @@ -32,8 +29,8 @@ module Gitlab @database_name ||= @connection.pool.db_config.name end - def db_dir - @db_dir ||= Rails.application.config.paths["db"].first + def database_schema_migrations_path + @connection.pool.db_config.configuration_hash[:schema_migrations_path] || DEFAULT_SCHEMA_MIGRATIONS_PATH end end end diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb index 20bf6fa4d30..bb8b9f333fe 100644 --- a/lib/gitlab/database/similarity_score.rb +++ b/lib/gitlab/database/similarity_score.rb @@ -67,7 +67,7 @@ module Gitlab def self.build_expression(search:, rules:) return EXPRESSION_ON_INVALID_INPUT if search.blank? || rules.empty? - quoted_search = ActiveRecord::Base.connection.quote(search.to_s) + quoted_search = ApplicationRecord.connection.quote(search.to_s) first_expression, *expressions = rules.map do |rule| rule_to_arel(quoted_search, rule) @@ -110,7 +110,7 @@ module Gitlab # CAST(multiplier AS numeric) def self.multiplier_expression(rule) - quoted_multiplier = ActiveRecord::Base.connection.quote(rule.fetch(:multiplier, DEFAULT_MULTIPLIER).to_s) + quoted_multiplier = ApplicationRecord.connection.quote(rule.fetch(:multiplier, DEFAULT_MULTIPLIER).to_s) Arel::Nodes::NamedFunction.new('CAST', [Arel.sql(quoted_multiplier).as('numeric')]) end diff --git a/lib/gitlab/database/transaction/context.rb b/lib/gitlab/database/transaction/context.rb new file mode 100644 index 00000000000..a50dd30b75b --- /dev/null +++ b/lib/gitlab/database/transaction/context.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Transaction + class Context + attr_reader :context + + LOG_DEPTH_THRESHOLD = 8 + LOG_SAVEPOINTS_THRESHOLD = 32 + LOG_DURATION_S_THRESHOLD = 300 + LOG_THROTTLE_DURATION = 1 + + def initialize + @context = {} + end + + def set_start_time + @context[:start_time] = current_timestamp + end + + def increment_savepoints + @context[:savepoints] = @context[:savepoints].to_i + 1 + end + + def increment_rollbacks + @context[:rollbacks] = @context[:rollbacks].to_i + 1 + end + + def increment_releases + @context[:releases] = @context[:releases].to_i + 1 + end + + def set_depth(depth) + @context[:depth] = [@context[:depth].to_i, depth].max + end + + def track_sql(sql) + (@context[:queries] ||= []).push(sql) + end + + def duration + return unless @context[:start_time].present? + + current_timestamp - @context[:start_time] + end + + def depth_threshold_exceeded? + @context[:depth].to_i > LOG_DEPTH_THRESHOLD + end + + def savepoints_threshold_exceeded? + @context[:savepoints].to_i > LOG_SAVEPOINTS_THRESHOLD + end + + def duration_threshold_exceeded? + duration.to_i > LOG_DURATION_S_THRESHOLD + end + + def log_savepoints? + depth_threshold_exceeded? || savepoints_threshold_exceeded? + end + + def log_duration? + duration_threshold_exceeded? + end + + def should_log? + !logged_already? && (log_savepoints? || log_duration?) + end + + def commit + log(:commit) + end + + def rollback + log(:rollback) + end + + private + + def queries + @context[:queries].to_a.join("\n") + end + + def current_timestamp + ::Gitlab::Metrics::System.monotonic_time + end + + def logged_already? + return false if @context[:last_log_timestamp].nil? + + (current_timestamp - @context[:last_log_timestamp].to_i) < LOG_THROTTLE_DURATION + end + + def set_last_log_timestamp + @context[:last_log_timestamp] = current_timestamp + end + + def log(operation) + return unless should_log? + + set_last_log_timestamp + + attributes = { + class: self.class.name, + result: operation, + duration_s: duration, + depth: @context[:depth].to_i, + savepoints_count: @context[:savepoints].to_i, + rollbacks_count: @context[:rollbacks].to_i, + releases_count: @context[:releases].to_i, + sql: queries + } + + application_info(attributes) + end + + def application_info(attributes) + Gitlab::AppJsonLogger.info(attributes) + end + end + end + end +end diff --git a/lib/gitlab/database/transaction/observer.rb b/lib/gitlab/database/transaction/observer.rb new file mode 100644 index 00000000000..7888f0916e3 --- /dev/null +++ b/lib/gitlab/database/transaction/observer.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Transaction + class Observer + INSTRUMENTED_STATEMENTS = %w[BEGIN SAVEPOINT ROLLBACK RELEASE].freeze + LONGEST_COMMAND_LENGTH = 'ROLLBACK TO SAVEPOINT'.length + START_COMMENT = '/*' + END_COMMENT = '*/' + + def self.instrument_transactions(cmd, event) + connection = event.payload[:connection] + manager = connection&.transaction_manager + return unless manager.respond_to?(:transaction_context) + + context = manager.transaction_context + return if context.nil? + + if cmd.start_with?('BEGIN') + context.set_start_time + context.set_depth(0) + context.track_sql(event.payload[:sql]) + elsif cmd.start_with?('SAVEPOINT ') + context.set_depth(manager.open_transactions) + context.increment_savepoints + elsif cmd.start_with?('ROLLBACK TO SAVEPOINT') + context.increment_rollbacks + elsif cmd.start_with?('RELEASE SAVEPOINT ') + context.increment_releases + end + end + + def self.register! + ActiveSupport::Notifications.subscribe('sql.active_record') do |event| + sql = event.payload.dig(:sql).to_s + cmd = extract_sql_command(sql) + + if cmd.start_with?(*INSTRUMENTED_STATEMENTS) + self.instrument_transactions(cmd, event) + end + end + end + + def self.extract_sql_command(sql) + return sql unless sql.start_with?(START_COMMENT) + + index = sql.index(END_COMMENT) + + return sql unless index + + # /* comment */ SELECT + # + # We offset using a position of the end comment + 1 character to + # accomodate a space between Marginalia comment and a SQL statement. + offset = index + END_COMMENT.length + 1 + + # Avoid duplicating the entire string. This isn't optimized to + # strip extra spaces, but we assume that this doesn't happen + # for performance reasons. + sql[offset..offset + LONGEST_COMMAND_LENGTH] + end + end + end + end +end diff --git a/lib/gitlab/deprecation_json_logger.rb b/lib/gitlab/deprecation_json_logger.rb new file mode 100644 index 00000000000..9796b24868b --- /dev/null +++ b/lib/gitlab/deprecation_json_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class DeprecationJsonLogger < Gitlab::JsonLogger + def self.file_name_noext + 'deprecation_json' + end + end +end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 6d04c4874c7..f73e060be7f 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -43,10 +43,16 @@ module Gitlab end end + # This is either the new path, otherwise the old path for the diff_file def diff_file_paths diff_files.map(&:file_path) end + # This is both the new and old paths for the diff_file + def diff_paths + diff_files.map(&:paths).flatten.uniq + end + def pagination_data @pagination_data || empty_pagination_data end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 28200643296..4fa2fe1724e 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -5,6 +5,7 @@ require 'gitlab/email/handler/reply_processing' # handles note/reply creation emails with these formats: # incoming+1234567890abcdef1234567890abcdef@incoming.gitlab.com +# Quoted material is _not_ stripped but appended as a `details` section module Gitlab module Email module Handler @@ -24,7 +25,7 @@ module Gitlab validate_permission!(:create_note) raise NoteableNotFoundError unless noteable - raise EmptyEmailError if message.blank? + raise EmptyEmailError if note_message.blank? verify_record!( record: create_note, @@ -47,7 +48,13 @@ module Gitlab end def create_note - sent_notification.create_reply(message) + sent_notification.create_reply(note_message) + end + + def note_message + return message unless sent_notification.noteable_type == "Issue" + + message_with_appended_reply end end end diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb index d508cf9360e..a717509e24d 100644 --- a/lib/gitlab/email/handler/reply_processing.rb +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -35,13 +35,20 @@ module Gitlab @message_with_reply ||= process_message(trim_reply: false) end + def message_with_appended_reply + @message_with_appended_reply ||= process_message(append_reply: true) + end + def process_message(**kwargs) - message = ReplyParser.new(mail, **kwargs).execute.strip + message, stripped_text = ReplyParser.new(mail, **kwargs).execute + message = message.strip + message_with_attachments = add_attachments(message) + # Support bot is specifically forbidden from using slash commands. + message = strip_quick_actions(message_with_attachments) + return message unless kwargs[:append_reply] - # Support bot is specifically forbidden - # from using slash commands. - strip_quick_actions(message_with_attachments) + append_reply(message, stripped_text) end def add_attachments(reply) @@ -92,10 +99,22 @@ module Gitlab def strip_quick_actions(content) return content unless author.support_bot? + quick_actions_extractor.redact_commands(content) + end + + def quick_actions_extractor command_definitions = ::QuickActions::InterpretService.command_definitions - extractor = ::Gitlab::QuickActions::Extractor.new(command_definitions) + ::Gitlab::QuickActions::Extractor.new(command_definitions) + end + + def append_reply(message, reply) + return message if message.blank? || reply.blank? + + # Do not append if message only contains slash commands + body, _commands = quick_actions_extractor.extract_commands(message) + return message if body.empty? - extractor.redact_commands(content) + message + "\n\n
...\n\n#{reply}\n\n
" end end end diff --git a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb new file mode 100644 index 00000000000..234b93594b5 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class AdminVerify < Base + def subject_line + s_('InProductMarketing|Create a custom CI runner with just a few clicks') + end + + def tagline + nil + end + + def title + s_('InProductMarketing|Spin up an autoscaling runner in GitLab') + end + + def subtitle + s_('InProductMarketing|Use our AWS cloudformation template to spin up your runners in just a few clicks!') + end + + def body_line1 + '' + end + + def body_line2 + '' + end + + def cta_text + s_('InProductMarketing|Create a custom runner') + end + + def progress + super(track_name: 'Admin') + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb index 89acc058a46..96551c89837 100644 --- a/lib/gitlab/email/message/in_product_marketing/base.rb +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -67,11 +67,11 @@ module Gitlab end end - def progress + def progress(current: series + 1, total: total_series, track_name: track.to_s.humanize) if Gitlab.com? - s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series.') % { current_series: series + 1, total_series: total_series, track: track.to_s.humanize } + s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series.') % { current_series: current, total_series: total, track: track_name } else - s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series. To disable notification emails sent by your local GitLab instance, either contact your administrator or %{unsubscribe_link}.') % { current_series: series + 1, total_series: total_series, track: track.to_s.humanize, unsubscribe_link: unsubscribe_link } + s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series. To disable notification emails sent by your local GitLab instance, either contact your administrator or %{unsubscribe_link}.') % { current_series: current, total_series: total, track: track_name, unsubscribe_link: unsubscribe_link } end end @@ -109,7 +109,7 @@ module Gitlab private def track - self.class.name.demodulize.downcase.to_sym + self.class.name.demodulize.underscore.to_sym end def total_series diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb index 5d3cac0a121..4b0c4af4911 100644 --- a/lib/gitlab/email/message/in_product_marketing/create.rb +++ b/lib/gitlab/email/message/in_product_marketing/create.rb @@ -84,7 +84,7 @@ module Gitlab end def basics_link - link(s_('InProductMarketing|Git basics'), help_page_url('gitlab-basics/README')) + link(s_('InProductMarketing|Git basics'), help_page_url('gitlab-basics/index')) end def import_link diff --git a/lib/gitlab/email/message/in_product_marketing/team.rb b/lib/gitlab/email/message/in_product_marketing/team.rb index 46c2797e534..cf723ad5efd 100644 --- a/lib/gitlab/email/message/in_product_marketing/team.rb +++ b/lib/gitlab/email/message/in_product_marketing/team.rb @@ -73,6 +73,10 @@ module Gitlab s_('InProductMarketing|Invite your team now') ][series] end + + def progress + super(current: series + 2, total: 4) + end end end end diff --git a/lib/gitlab/email/message/in_product_marketing/team_short.rb b/lib/gitlab/email/message/in_product_marketing/team_short.rb new file mode 100644 index 00000000000..1d60a5fe4e5 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/team_short.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class TeamShort < Base + def subject_line + s_('InProductMarketing|Team up in GitLab for greater efficiency') + end + + def tagline + nil + end + + def title + s_('InProductMarketing|Turn coworkers into collaborators') + end + + def subtitle + s_('InProductMarketing|Invite your team today to build better code (and processes) together') + end + + def body_line1 + '' + end + + def body_line2 + '' + end + + def cta_text + s_('InProductMarketing|Invite your colleagues today') + end + + def progress + super(total: 4, track_name: 'Team') + end + + def logo_path + 'mailers/in_product_marketing/team-0.png' + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/trial.rb b/lib/gitlab/email/message/in_product_marketing/trial.rb index d87dc5c1b81..222046a3966 100644 --- a/lib/gitlab/email/message/in_product_marketing/trial.rb +++ b/lib/gitlab/email/message/in_product_marketing/trial.rb @@ -68,6 +68,10 @@ module Gitlab s_('InProductMarketing|Start your trial now!') ][series] end + + def progress + super(current: series + 2, total: 4) + end end end end diff --git a/lib/gitlab/email/message/in_product_marketing/trial_short.rb b/lib/gitlab/email/message/in_product_marketing/trial_short.rb new file mode 100644 index 00000000000..0fcd3fde4a6 --- /dev/null +++ b/lib/gitlab/email/message/in_product_marketing/trial_short.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Message + module InProductMarketing + class TrialShort < Base + def subject_line + s_('InProductMarketing|Be a DevOps hero') + end + + def tagline + nil + end + + def title + s_('InProductMarketing|Expand your DevOps journey with a free GitLab trial') + end + + def subtitle + s_('InProductMarketing|Start your trial today to experience single application success and discover all the features of GitLab Ultimate for free!') + end + + def body_line1 + '' + end + + def body_line2 + '' + end + + def cta_text + s_('InProductMarketing|Start a trial') + end + + def progress + super(total: 4, track_name: 'Trial') + end + + def logo_path + 'mailers/in_product_marketing/trial-0.png' + end + end + end + end + end +end diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb index 88140c67804..e731c65121e 100644 --- a/lib/gitlab/email/message/in_product_marketing/verify.rb +++ b/lib/gitlab/email/message/in_product_marketing/verify.rb @@ -72,7 +72,7 @@ module Gitlab end def quick_start_link - link(s_('InProductMarketing|quick start guide'), help_page_url('ci/quick_start/README')) + link(s_('InProductMarketing|quick start guide'), help_page_url('ci/quick_start/index')) end def performance_link diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 7579f3d8680..0f0f4800062 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -6,20 +6,17 @@ module Gitlab class ReplyParser attr_accessor :message - def initialize(message, trim_reply: true) + def initialize(message, trim_reply: true, append_reply: false) @message = message @trim_reply = trim_reply + @append_reply = append_reply end def execute body = select_body(message) encoding = body.encoding - - if @trim_reply - body = EmailReplyTrimmer.trim(body) - end - + body, stripped_text = EmailReplyTrimmer.trim(body, @append_reply) if @trim_reply return '' unless body # not using /\s+$/ here because that deletes empty lines @@ -30,7 +27,10 @@ module Gitlab # so we detect it manually here. return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') } - body.force_encoding(encoding).encode("UTF-8") + encoded_body = body.force_encoding(encoding).encode("UTF-8") + return encoded_body unless @append_reply + + [encoded_body, stripped_text.force_encoding(encoding).encode("UTF-8")] end private diff --git a/lib/gitlab/email/smtp_config.rb b/lib/gitlab/email/smtp_config.rb new file mode 100644 index 00000000000..c9deb3fe324 --- /dev/null +++ b/lib/gitlab/email/smtp_config.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Email + class SmtpConfig + def self.encrypted_secrets + Settings.encrypted(Gitlab.config.gitlab.email_smtp_secret_file) + end + + def self.secrets + self.new + end + + def initialize + @secrets ||= self.class.encrypted_secrets.config + rescue StandardError => e + Gitlab::AppLogger.error "SMTP encrypted secrets are invalid: #{e.inspect}" + end + + def username + @secrets&.fetch(:user_name, nil)&.chomp + end + + def password + @secrets&.fetch(:password, nil)&.chomp + end + end + end +end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 8ee53d0de28..50e7631d983 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -64,6 +64,15 @@ module Gitlab detect && detect[:type] == :binary end + # This is like encode_utf8 except we skip autodetection of the encoding. We + # assume the data must be interpreted as UTF-8. + def encode_utf8_no_detect(message) + message = force_encode_utf8(message) + return message if message.valid_encoding? + + message.encode(Encoding::UTF_8, invalid: :replace, undef: :replace) + end + def encode_utf8(message, replace: "") message = force_encode_utf8(message) return message if message.valid_encoding? diff --git a/lib/gitlab/encrypted_command_base.rb b/lib/gitlab/encrypted_command_base.rb new file mode 100644 index 00000000000..b35c28b85cd --- /dev/null +++ b/lib/gitlab/encrypted_command_base.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +# rubocop:disable Rails/Output +module Gitlab + class EncryptedCommandBase + DISPLAY_NAME = "Base" + EDIT_COMMAND_NAME = "base" + + class << self + def encrypted_secrets + raise NotImplementedError + end + + def write(contents) + encrypted = encrypted_secrets + return unless validate_config(encrypted) + + validate_contents(contents) + encrypted.write(contents) + + puts "File encrypted and saved." + rescue Interrupt + warn "Aborted changing file: nothing saved." + rescue ActiveSupport::MessageEncryptor::InvalidMessage + warn "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?" + end + + def edit + encrypted = encrypted_secrets + return unless validate_config(encrypted) + + if ENV["EDITOR"].blank? + warn 'No $EDITOR specified to open file. Please provide one when running the command:' + warn "gitlab-rake #{self::EDIT_COMMAND_NAME} EDITOR=vim" + return + end + + temp_file = Tempfile.new(File.basename(encrypted.content_path), File.dirname(encrypted.content_path)) + contents_changed = false + + encrypted.change do |contents| + contents = encrypted_file_template unless File.exist?(encrypted.content_path) + File.write(temp_file.path, contents) + system(ENV['EDITOR'], temp_file.path) + changes = File.read(temp_file.path) + contents_changed = contents != changes + validate_contents(changes) + changes + end + + puts "Contents were unchanged." unless contents_changed + puts "File encrypted and saved." + rescue Interrupt + warn "Aborted changing file: nothing saved." + rescue ActiveSupport::MessageEncryptor::InvalidMessage + warn "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?" + ensure + temp_file&.unlink + end + + def show + encrypted = encrypted_secrets + return unless validate_config(encrypted) + + puts encrypted.read.presence || "File '#{encrypted.content_path}' does not exist. Use `gitlab-rake #{self::EDIT_COMMAND_NAME}` to change that." + rescue ActiveSupport::MessageEncryptor::InvalidMessage + warn "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?" + end + + def validate_config(encrypted) + dir_path = File.dirname(encrypted.content_path) + + unless File.exist?(dir_path) + warn "Directory #{dir_path} does not exist. Create the directory and try again." + return false + end + + if encrypted.key.nil? + warn "Missing encryption key encrypted_settings_key_base." + return false + end + + true + end + + def validate_contents(contents) + begin + config = YAML.safe_load(contents, permitted_classes: [Symbol]) + error_contents = "Did not include any key-value pairs" unless config.is_a?(Hash) + rescue Psych::Exception => e + error_contents = e.message + end + + puts "WARNING: Content was not a valid #{self::DISPLAY_NAME} secret yml file. #{error_contents}" if error_contents + + contents + end + + def encrypted_file_template + raise NotImplementedError + end + end + end +end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/encrypted_ldap_command.rb b/lib/gitlab/encrypted_ldap_command.rb index cdb3e268b51..3675646185e 100644 --- a/lib/gitlab/encrypted_ldap_command.rb +++ b/lib/gitlab/encrypted_ldap_command.rb @@ -2,93 +2,13 @@ # rubocop:disable Rails/Output module Gitlab - class EncryptedLdapCommand - class << self - def write(contents) - encrypted = Gitlab::Auth::Ldap::Config.encrypted_secrets - return unless validate_config(encrypted) - - validate_contents(contents) - encrypted.write(contents) - - puts "File encrypted and saved." - rescue Interrupt - puts "Aborted changing file: nothing saved." - rescue ActiveSupport::MessageEncryptor::InvalidMessage - puts "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?" - end - - def edit - encrypted = Gitlab::Auth::Ldap::Config.encrypted_secrets - return unless validate_config(encrypted) - - if ENV["EDITOR"].blank? - puts 'No $EDITOR specified to open file. Please provide one when running the command:' - puts 'gitlab-rake gitlab:ldap:secret:edit EDITOR=vim' - return - end - - temp_file = Tempfile.new(File.basename(encrypted.content_path), File.dirname(encrypted.content_path)) - contents_changed = false - - encrypted.change do |contents| - contents = encrypted_file_template unless File.exist?(encrypted.content_path) - File.write(temp_file.path, contents) - system(ENV['EDITOR'], temp_file.path) - changes = File.read(temp_file.path) - contents_changed = contents != changes - validate_contents(changes) - changes - end - - puts "Contents were unchanged." unless contents_changed - puts "File encrypted and saved." - rescue Interrupt - puts "Aborted changing file: nothing saved." - rescue ActiveSupport::MessageEncryptor::InvalidMessage - puts "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?" - ensure - temp_file&.unlink - end - - def show - encrypted = Gitlab::Auth::Ldap::Config.encrypted_secrets - return unless validate_config(encrypted) + class EncryptedLdapCommand < EncryptedCommandBase + DISPLAY_NAME = "LDAP" + EDIT_COMMAND_NAME = "gitlab:ldap:secret:edit" - puts encrypted.read.presence || "File '#{encrypted.content_path}' does not exist. Use `gitlab-rake gitlab:ldap:secret:edit` to change that." - rescue ActiveSupport::MessageEncryptor::InvalidMessage - puts "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?" - end - - private - - def validate_config(encrypted) - dir_path = File.dirname(encrypted.content_path) - - unless File.exist?(dir_path) - puts "Directory #{dir_path} does not exist. Create the directory and try again." - return false - end - - if encrypted.key.nil? - puts "Missing encryption key encrypted_settings_key_base." - return false - end - - true - end - - def validate_contents(contents) - begin - config = YAML.safe_load(contents, permitted_classes: [Symbol]) - error_contents = "Did not include any key-value pairs" unless config.is_a?(Hash) - rescue Psych::Exception => e - error_contents = e.message - end - - puts "WARNING: Content was not a valid LDAP secret yml file. #{error_contents}" if error_contents - - contents + class << self + def encrypted_secrets + Gitlab::Auth::Ldap::Config.encrypted_secrets end def encrypted_file_template diff --git a/lib/gitlab/encrypted_smtp_command.rb b/lib/gitlab/encrypted_smtp_command.rb new file mode 100644 index 00000000000..51a476b143d --- /dev/null +++ b/lib/gitlab/encrypted_smtp_command.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# rubocop:disable Rails/Output +module Gitlab + class EncryptedSmtpCommand < EncryptedCommandBase + DISPLAY_NAME = "SMTP" + EDIT_COMMAND_NAME = "gitlab:smtp:secret:edit" + + class << self + def encrypted_secrets + Gitlab::Email::SmtpConfig.encrypted_secrets + end + + def encrypted_file_template + <<~YAML + # password: '123' + # user_name: 'gitlab-inst' + YAML + end + end + end +end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/restful.rb index fba4b9e433a..408a901f69d 100644 --- a/lib/gitlab/etag_caching/router/restful.rb +++ b/lib/gitlab/etag_caching/router/restful.rb @@ -71,7 +71,7 @@ module Gitlab 'continuous_delivery' ], [ - %r(#{RESERVED_WORDS_PREFIX}/environments\.json\z), + %r(#{RESERVED_WORDS_PREFIX}/-/environments\.json\z), 'environments', 'continuous_delivery' ], diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 0cf3969b490..2f78e4e5c0a 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -34,16 +34,13 @@ module Gitlab module Experimentation EXPERIMENTS = { - invite_members_empty_group_version_a: { - tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyGroupVersionA', - use_backwards_compatible_subject_index: true - }, - contact_sales_btn_in_app: { - tracking_category: 'Growth::Conversion::Experiment::ContactSalesInApp', - use_backwards_compatible_subject_index: true + remove_known_trial_form_fields_welcoming: { + tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsWelcoming', + rollout_strategy: :user }, - remove_known_trial_form_fields: { - tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFields' + remove_known_trial_form_fields_noneditable: { + tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFieldsNoneditable', + rollout_strategy: :user }, invite_members_new_dropdown: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' @@ -52,9 +49,6 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar', rollout_strategy: :group }, - trial_onboarding_issues: { - tracking_category: 'Growth::Conversion::Experiment::TrialOnboardingIssues' - }, learn_gitlab_a: { tracking_category: 'Growth::Conversion::Experiment::LearnGitLabA', rollout_strategy: :user diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index 211c0967f89..0b9a4c161ae 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -1,35 +1,51 @@ # frozen_string_literal: true -# This class extends an OpenStruct object by adding predicate methods to mimic +# Fakes ActiveRecord attribute storage by adding predicate methods to mimic # ActiveRecord access. We rely on the initial values being true or false to # determine whether to define a predicate method because for a newly-added # column that has not been migrated yet, there is no way to determine the # column type without parsing db/structure.sql. module Gitlab - class FakeApplicationSettings < OpenStruct - include ApplicationSettingImplementation - - # Mimic ActiveRecord predicate methods for boolean values - def self.define_predicate_methods(options) - options.each do |key, value| - next if key.to_s.end_with?('?') - next unless [true, false].include?(value) - - define_method "#{key}?" do - actual_key = key.to_s.chomp('?') - self[actual_key] + class FakeApplicationSettings + prepend ApplicationSettingImplementation + + def self.define_properties(settings) + settings.each do |key, value| + define_method key do + read_attribute(key) + end + + if [true, false].include?(value) + define_method "#{key}?" do + read_attribute(key) + end + end + + define_method "#{key}=" do |v| + @table[key.to_sym] = v end end end - def initialize(options = {}) - super + def initialize(settings = {}) + @table = settings.dup - FakeApplicationSettings.define_predicate_methods(options) + FakeApplicationSettings.define_properties(settings) end - alias_method :read_attribute, :[] - alias_method :has_attribute?, :[] + def read_attribute(key) + @table[key.to_sym] + end + + def has_attribute?(key) + @table.key?(key.to_sym) + end + + # Mimic behavior of OpenStruct, which absorbs any calls into undefined + # properties to return `nil`. + def method_missing(*) + nil + end end end diff --git a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb new file mode 100644 index 00000000000..a5290508e42 --- /dev/null +++ b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module FormBuilders + class GitlabUiFormBuilder < ActionView::Helpers::FormBuilder + def gitlab_ui_checkbox_component( + method, + label, + help_text: nil, + checkbox_options: {}, + checked_value: '1', + unchecked_value: '0', + label_options: {} + ) + @template.content_tag( + :div, + class: 'gl-form-checkbox custom-control custom-checkbox' + ) do + @template.check_box( + @object_name, + method, + format_options(checkbox_options, ['custom-control-input']), + checked_value, + unchecked_value + ) + + @template.label( + @object_name, method, format_options(label_options, ['custom-control-label']) + ) do + if help_text + @template.content_tag( + :span, + label + ) + + @template.content_tag( + :p, + help_text, + class: 'help-text' + ) + else + label + end + end + end + end + + private + + def format_options(options, classes) + classes << options[:class] + + objectify_options(options.merge({ class: classes.flatten.compact })) + end + end + end +end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 1c8e55ecf50..f72217dedde 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -77,8 +77,8 @@ module Gitlab end end - def raw(repository, sha) - repository.gitaly_blob_client.get_blob(oid: sha, limit: MAX_DATA_DISPLAY_SIZE) + def raw(repository, sha, limit: MAX_DATA_DISPLAY_SIZE) + repository.gitaly_blob_client.get_blob(oid: sha, limit: limit) end # Returns an array of Blob instances, specified in blob_references as diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index a863b952390..7fd4acb4179 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -111,8 +111,18 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) + # In either of these cases, we are guaranteed to return no commits, so + # shortcut the RPC call + return [] if Gitlab::Git.blank_ref?(base) || Gitlab::Git.blank_ref?(head) + wrapped_gitaly_errors do - repo.gitaly_commit_client.between(base, head) + if Feature.enabled?(:between_uses_list_commits, default_enabled: :yaml) + revisions = [head, "^#{base}"] # base..head + + repo.gitaly_commit_client.list_commits(revisions, reverse: true) + else + repo.gitaly_commit_client.between(base, head) + end end end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 8815088d23c..6a7a7032665 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -14,21 +14,23 @@ module Gitlab # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/323 def initialize(repo, commit) @id = commit.id - @additions = 0 - @deletions = 0 - @total = 0 - wrapped_gitaly_errors do - gitaly_stats(repo, commit) - end - end + additions, deletions = fetch_stats(repo, commit) - def gitaly_stats(repo, commit) - stats = repo.gitaly_commit_client.commit_stats(@id) - @additions = stats.additions - @deletions = stats.deletions + @additions = additions.to_i + @deletions = deletions.to_i @total = @additions + @deletions end + + def fetch_stats(repo, commit) + Rails.cache.fetch("commit_stats:#{repo.gl_project_path}:#{@id}") do + stats = wrapped_gitaly_errors do + repo.gitaly_commit_client.commit_stats(@id) + end + + [stats.additions, stats.deletions] + end + end end end end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index 7ffe4a7ae81..049ca5a54b3 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -6,13 +6,14 @@ module Gitlab class File UnsupportedEncoding = Class.new(StandardError) - attr_reader :their_path, :our_path, :our_mode, :repository, :commit_oid + attr_reader :ancestor_path, :their_path, :our_path, :our_mode, :repository, :commit_oid attr_accessor :raw_content def initialize(repository, commit_oid, conflict, raw_content) @repository = repository @commit_oid = commit_oid + @ancestor_path = conflict[:ancestor][:path] @their_path = conflict[:theirs][:path] @our_path = conflict[:ours][:path] @our_mode = conflict[:ours][:mode] @@ -94,6 +95,15 @@ module Gitlab resolution end + + def path + # There are conflict scenarios (e.g. file is removed on source) wherein + # our_path will be blank/nil. Since we are indexing them by path in + # `#conflicts` helper and we want to match the diff file to a conflict + # in `DiffFileEntity#highlighted_diff_lines`, we need to fallback to + # their_path (this is the path on target). + our_path.presence || their_path + end end end end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index aa5d50d1fb1..2069d26400e 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -9,15 +9,16 @@ module Gitlab ConflictSideMissing = Class.new(StandardError) ResolutionError = Class.new(StandardError) - def initialize(target_repository, our_commit_oid, their_commit_oid) + def initialize(target_repository, our_commit_oid, their_commit_oid, allow_tree_conflicts: false) @target_repository = target_repository @our_commit_oid = our_commit_oid @their_commit_oid = their_commit_oid + @allow_tree_conflicts = allow_tree_conflicts end def conflicts @conflicts ||= wrapped_gitaly_errors do - gitaly_conflicts_client(@target_repository).list_conflict_files.to_a + gitaly_conflicts_client(@target_repository).list_conflict_files(allow_tree_conflicts: @allow_tree_conflicts).to_a rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing, e.message end diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index eb368af199d..2f618294e8e 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -5,11 +5,10 @@ module Gitlab class RemoteMirror include Gitlab::Git::WrapsGitalyErrors - attr_reader :repository, :ref_name, :remote_url, :only_branches_matching, :ssh_key, :known_hosts, :keep_divergent_refs + attr_reader :repository, :remote_url, :only_branches_matching, :ssh_key, :known_hosts, :keep_divergent_refs - def initialize(repository, ref_name, remote_url, only_branches_matching: [], ssh_key: nil, known_hosts: nil, keep_divergent_refs: false) + def initialize(repository, remote_url, only_branches_matching: [], ssh_key: nil, known_hosts: nil, keep_divergent_refs: false) @repository = repository - @ref_name = ref_name @remote_url = remote_url @only_branches_matching = only_branches_matching @ssh_key = ssh_key @@ -20,7 +19,6 @@ module Gitlab def update wrapped_gitaly_errors do repository.gitaly_remote_client.update_remote_mirror( - ref_name, remote_url, only_branches_matching, ssh_key: ssh_key, diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 70d072e8082..1ab80fe2454 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -354,13 +354,9 @@ module Gitlab end end - def new_commits(newrevs) + def new_commits(newrevs, allow_quarantine: false) wrapped_gitaly_errors do - if Feature.enabled?(:list_commits) - gitaly_commit_client.list_commits(Array.wrap(newrevs) + %w[--not --all]) - else - Array.wrap(newrevs).flat_map { |newrev| gitaly_ref_client.list_new_commits(newrev) } - end + gitaly_commit_client.list_new_commits(Array.wrap(newrevs), allow_quarantine: allow_quarantine) end end @@ -703,24 +699,11 @@ module Gitlab write_ref(ref, start_point) end - # If `mirror_refmap` is present the remote is set as mirror with that mapping - def add_remote(remote_name, url, mirror_refmap: nil) - wrapped_gitaly_errors do - gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) - end - end - - def remove_remote(remote_name) - wrapped_gitaly_errors do - gitaly_remote_client.remove_remote(remote_name) - end - end - - def find_remote_root_ref(remote_name, remote_url, authorization = nil) - return unless remote_name.present? && remote_url.present? + def find_remote_root_ref(remote_url, authorization = nil) + return unless remote_url.present? wrapped_gitaly_errors do - gitaly_remote_client.find_remote_root_ref(remote_name, remote_url, authorization) + gitaly_remote_client.find_remote_root_ref(remote_url, authorization) end end @@ -820,18 +803,18 @@ module Gitlab # no_tags - should we use --no-tags flag? # prune - should we use --prune flag? # check_tags_changed - should we ask gitaly to calculate whether any tags changed? - def fetch_remote(remote, url: nil, refmap: nil, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false) + def fetch_remote(url, refmap: nil, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false, http_authorization_header: "") wrapped_gitaly_errors do gitaly_repository_client.fetch_remote( - remote, - url: url, + url, refmap: refmap, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune, check_tags_changed: check_tags_changed, - timeout: GITLAB_PROJECTS_TIMEOUT + timeout: GITLAB_PROJECTS_TIMEOUT, + http_authorization_header: http_authorization_header ) end end @@ -844,8 +827,8 @@ module Gitlab end end - def blob_at(sha, path) - Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) + def blob_at(sha, path, limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) + Gitlab::Git::Blob.find(self, sha, path, limit: limit) unless Gitlab::Git.blank_ref?(sha) end # Items should be of format [[commit_id, path], [commit_id1, path1]] @@ -922,13 +905,17 @@ module Gitlab end # rubocop:enable Metrics/ParameterLists - def write_config(full_path:) + def set_full_path(full_path:) return unless full_path.present? # This guard avoids Gitaly log/error spam raise NoRepository, 'repository does not exist' unless exists? - set_config('gitlab.fullpath' => full_path) + if Feature.enabled?(:set_full_path) + gitaly_repository_client.set_full_path(full_path) + else + set_config('gitlab.fullpath' => full_path) + end end def set_config(entries) @@ -937,12 +924,6 @@ module Gitlab end end - def delete_config(*keys) - wrapped_gitaly_errors do - gitaly_repository_client.delete_config(keys) - end - end - def disconnect_alternates wrapped_gitaly_errors do gitaly_repository_client.disconnect_alternates diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 389c9d32ccb..5993c8888d3 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -14,9 +14,12 @@ module Gitlab include Gitlab::Git::RuggedImpl::UseRugged override :tree_entries - def tree_entries(repository, sha, path, recursive) + def tree_entries(repository, sha, path, recursive, pagination_params = nil) if use_rugged?(repository, :rugged_tree_entries) - execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive) + [ + execute_rugged_call(:tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive), + nil + ] else super end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 568e894a02f..25895dc6728 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -5,6 +5,8 @@ module Gitlab class Tag < Ref extend Gitlab::EncodingHelper + delegate :id, to: :@raw_tag + attr_reader :object_sha, :repository MAX_TAG_MESSAGE_DISPLAY_SIZE = 10.megabytes @@ -24,6 +26,18 @@ module Gitlab def get_messages(repository, tag_ids) repository.gitaly_ref_client.get_tag_messages(tag_ids) end + + def extract_signature_lazily(repository, tag_id) + BatchLoader.for(tag_id).batch(key: repository) do |tag_ids, loader, args| + batch_signature_extraction(args[:key], tag_ids).each do |tag_id, signature_data| + loader.call(tag_id, signature_data) + end + end + end + + def batch_signature_extraction(repository, tag_ids) + repository.gitaly_ref_client.get_tag_signatures(tag_ids) + end end def initialize(repository, raw_tag) @@ -81,7 +95,7 @@ module Gitlab when :PGP nil # not implemented, see https://gitlab.com/gitlab-org/gitlab/issues/19260 when :X509 - X509::Tag.new(@raw_tag).signature + X509::Tag.new(@repository, self).signature else nil end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index ed02f2e92ec..eb008507397 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -15,15 +15,15 @@ module Gitlab # Uses rugged for raw objects # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 - def where(repository, sha, path = nil, recursive = false) + def where(repository, sha, path = nil, recursive = false, pagination_params = nil) path = nil if path == '' || path == '/' - tree_entries(repository, sha, path, recursive) + tree_entries(repository, sha, path, recursive, pagination_params) end - def tree_entries(repository, sha, path, recursive) + def tree_entries(repository, sha, path, recursive, pagination_params = nil) wrapped_gitaly_errors do - repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) + repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive, pagination_params) end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 642a77ced11..759c6b93d9a 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -498,21 +498,10 @@ module Gitlab end def check_changes_size - changes_size = - if Feature.enabled?(:git_access_batched_changes_size, project, default_enabled: :yaml) - revs = ['--not', '--all', '--not'] - revs += changes_list.map { |change| change[:newrev] } + revs = ['--not', '--all', '--not'] + revs += changes_list.map { |change| change[:newrev] } - repository.blobs(revs).sum(&:size) - else - changes_size = 0 - - changes_list.each do |change| - changes_size += repository.new_blobs(change[:newrev]).sum(&:size) - end - - changes_size - end + changes_size = repository.blobs(revs).sum(&:size) check_size_against_limit(changes_size) end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index b894207f0aa..fa616a252e4 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -111,17 +111,22 @@ module Gitlab nil end - def tree_entries(repository, revision, path, recursive) + def tree_entries(repository, revision, path, recursive, pagination_params) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, revision: encode_binary(revision), path: path.present? ? encode_binary(path) : '.', - recursive: recursive + recursive: recursive, + pagination_params: pagination_params ) + request.sort = Gitaly::GetTreeEntriesRequest::SortBy::TREES_FIRST if pagination_params response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) - response.flat_map do |message| + cursor = nil + + entries = response.flat_map do |message| + cursor = message.pagination_cursor if message.pagination_cursor message.entries.map do |gitaly_tree_entry| Gitlab::Git::Tree.new( id: gitaly_tree_entry.oid, @@ -135,6 +140,8 @@ module Gitlab ) end end + + [entries, cursor] end def commit_count(ref, options = {}) @@ -248,16 +255,42 @@ module Gitlab consume_commits_response(response) end - def list_commits(revisions) + def list_commits(revisions, reverse: false) request = Gitaly::ListCommitsRequest.new( repository: @gitaly_repo, - revisions: Array.wrap(revisions) + revisions: Array.wrap(revisions), + reverse: reverse ) response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end + # List all commits which are new in the repository. If commits have been pushed into the repo + def list_new_commits(revisions, allow_quarantine: false) + git_env = Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository) + if allow_quarantine && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? + # If we have a quarantine environment, then we can optimize the check + # by doing a ListAllCommitsRequest. Instead of walking through + # references, we just walk through all quarantined objects, which is + # a lot more efficient. To do so, we throw away any alternate object + # directories, which point to the main object directory of the + # repository, and only keep the object directory which points into + # the quarantine object directory. + quarantined_repo = @gitaly_repo.dup + quarantined_repo.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string) + + request = Gitaly::ListAllCommitsRequest.new( + repository: quarantined_repo + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) + consume_commits_response(response) + else + list_commits(Array.wrap(revisions) + %w[--not --all]) + end + end + def list_commits_by_oid(oids) return [] if oids.empty? diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb index 38ec910111c..b1278e3bfac 100644 --- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -43,6 +43,7 @@ module Gitlab def conflict_from_gitaly_file_header(header) { + ancestor: { path: header.ancestor_path }, ours: { path: header.our_path, mode: header.our_mode }, theirs: { path: header.their_path } } diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index 300800189f1..982454b117e 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -14,11 +14,12 @@ module Gitlab @their_commit_oid = their_commit_oid end - def list_conflict_files + def list_conflict_files(allow_tree_conflicts: false) request = Gitaly::ListConflictFilesRequest.new( repository: @gitaly_repo, our_commit_oid: @our_commit_oid, - their_commit_oid: @their_commit_oid + their_commit_oid: @their_commit_oid, + allow_tree_conflicts: allow_tree_conflicts ) response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout) GitalyClient::ConflictFilesStitcher.new(response, @gitaly_repo) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index ac2db99ee01..7097d5bd181 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -62,24 +62,6 @@ module Gitlab encode!(response.name.dup) end - def list_new_commits(newrev) - request = Gitaly::ListNewCommitsRequest.new( - repository: @gitaly_repo, - commit_id: newrev - ) - - commits = [] - - response = GitalyClient.call(@storage, :ref_service, :list_new_commits, request, timeout: GitalyClient.medium_timeout) - response.each do |msg| - msg.commits.each do |c| - commits << Gitlab::Git::Commit.new(@repository, c) - end - end - - commits - end - def list_new_blobs(newrev, limit = 0, dynamic_timeout: nil) request = Gitaly::ListNewBlobsRequest.new( repository: @gitaly_repo, @@ -196,6 +178,27 @@ module Gitlab messages end + def get_tag_signatures(tag_ids) + request = Gitaly::GetTagSignaturesRequest.new(repository: @gitaly_repo, tag_revisions: tag_ids) + response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_signatures, request, timeout: GitalyClient.fast_timeout) + + signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } + current_tag_id = nil + + response.each do |message| + message.signatures.each do |tag_signature| + current_tag_id = tag_signature.tag_id if tag_signature.tag_id.present? + + signatures[current_tag_id].first << tag_signature.signature + signatures[current_tag_id].last << tag_signature.content + end + end + + signatures + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex + end + def pack_refs request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 487127b7b74..535b987f91c 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -26,25 +26,7 @@ module Gitlab @storage = repository.storage end - def add_remote(name, url, mirror_refmaps) - request = Gitaly::AddRemoteRequest.new( - repository: @gitaly_repo, - name: name, - url: url, - mirror_refmaps: Array.wrap(mirror_refmaps).map(&:to_s) - ) - - GitalyClient.call(@storage, :remote_service, :add_remote, request, timeout: GitalyClient.fast_timeout) - end - - def remove_remote(name) - request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name) - - GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.long_timeout).result - end - - # The remote_name parameter is deprecated and will be removed soon. - def find_remote_root_ref(remote_name, remote_url, authorization) + def find_remote_root_ref(remote_url, authorization) request = Gitaly::FindRemoteRootRefRequest.new(repository: @gitaly_repo, remote_url: remote_url, http_authorization_header: authorization) @@ -55,18 +37,13 @@ module Gitlab encode_utf8(response.ref) end - def update_remote_mirror(ref_name, remote_url, only_branches_matching, ssh_key: nil, known_hosts: nil, keep_divergent_refs: false) + def update_remote_mirror(remote_url, only_branches_matching, ssh_key: nil, known_hosts: nil, keep_divergent_refs: false) req_enum = Enumerator.new do |y| first_request = Gitaly::UpdateRemoteMirrorRequest.new( repository: @gitaly_repo ) - if remote_url - first_request.remote = Gitaly::UpdateRemoteMirrorRequest::Remote.new(url: remote_url) - else - first_request.ref_name = ref_name - end - + first_request.remote = Gitaly::UpdateRemoteMirrorRequest::Remote.new(url: remote_url) first_request.ssh_key = ssh_key if ssh_key.present? first_request.known_hosts = known_hosts if known_hosts.present? first_request.keep_divergent_refs = keep_divergent_refs diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 009aeaf868a..2e26b3341a2 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -73,18 +73,21 @@ module Gitlab # rubocop: disable Metrics/ParameterLists # The `remote` parameter is going away soonish anyway, at which point the # Rubocop warning can be enabled again. - def fetch_remote(remote, url:, refmap:, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false) + def fetch_remote(url, refmap:, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false, http_authorization_header: "") request = Gitaly::FetchRemoteRequest.new( - repository: @gitaly_repo, remote: remote, force: forced, - no_tags: no_tags, timeout: timeout, no_prune: !prune, - check_tags_changed: check_tags_changed + repository: @gitaly_repo, + force: forced, + no_tags: no_tags, + timeout: timeout, + no_prune: !prune, + check_tags_changed: check_tags_changed, + remote_params: Gitaly::Remote.new( + url: url, + mirror_refmaps: Array.wrap(refmap).map(&:to_s), + http_authorization_header: http_authorization_header + ) ) - if url - request.remote_params = Gitaly::Remote.new(url: url, - mirror_refmaps: Array.wrap(refmap).map(&:to_s)) - end - if ssh_auth&.ssh_mirror_url? if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? request.ssh_key = ssh_auth.ssh_private_key @@ -263,34 +266,33 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :write_ref, request, timeout: GitalyClient.fast_timeout) end - def set_config(entries) - return if entries.empty? - - request = Gitaly::SetConfigRequest.new(repository: @gitaly_repo) - entries.each do |key, value| - request.entries << build_set_config_entry(key, value) - end - + def set_full_path(path) GitalyClient.call( @storage, :repository_service, - :set_config, - request, + :set_full_path, + Gitaly::SetFullPathRequest.new( + repository: @gitaly_repo, + path: path + ), timeout: GitalyClient.fast_timeout ) nil end - def delete_config(keys) - return if keys.empty? + def set_config(entries) + return if entries.empty? - request = Gitaly::DeleteConfigRequest.new(repository: @gitaly_repo, keys: keys) + request = Gitaly::SetConfigRequest.new(repository: @gitaly_repo) + entries.each do |key, value| + request.entries << build_set_config_entry(key, value) + end GitalyClient.call( @storage, :repository_service, - :delete_config, + :set_config, request, timeout: GitalyClient.fast_timeout ) diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb index 0d448b55104..80f8f8bfbe2 100644 --- a/lib/gitlab/github_import/bulk_importing.rb +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -3,23 +3,60 @@ module Gitlab module GithubImport module BulkImporting + attr_reader :project, :client + + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(project, client) + @project = project + @client = client + end + # Builds and returns an Array of objects to bulk insert into the # database. # # enum - An Enumerable that returns the objects to turn into database # rows. def build_database_rows(enum) - enum.each_with_object([]) do |(object, _), rows| - rows << build(object) unless already_imported?(object) + rows = enum.each_with_object([]) do |(object, _), result| + result << build(object) unless already_imported?(object) end + + log_and_increment_counter(rows.size, :fetched) + + rows end # Bulk inserts the given rows into the database. def bulk_insert(model, rows, batch_size: 100) rows.each_slice(batch_size) do |slice| - Gitlab::Database.bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert + + log_and_increment_counter(slice.size, :imported) end end + + def object_type + raise NotImplementedError + end + + private + + def log_and_increment_counter(value, operation) + Gitlab::Import::Logger.info( + import_type: :github, + project_id: project.id, + importer: self.class.name, + message: "#{value} #{object_type.to_s.pluralize} #{operation}" + ) + + Gitlab::GithubImport::ObjectCounter.increment( + project, + object_type, + operation, + value: value + ) + end end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index d2f5af63621..9bda066efcc 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -46,7 +46,7 @@ module Gitlab # To work around this we're using bulk_insert with a single row. This # allows us to efficiently insert data (even if it's just 1 row) # without having to use all sorts of hacks to disable callbacks. - Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(LegacyDiffNote.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert rescue ActiveRecord::InvalidForeignKey # It's possible the project and the issue have been deleted since # scheduling this job. In this case we'll just skip creating the note. diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 13061d2c9df..f8665676ccf 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -75,7 +75,7 @@ module Gitlab end end - Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert end end end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb index 77eb4542195..b608bb48e38 100644 --- a/lib/gitlab/github_import/importer/label_links_importer.rb +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -40,7 +40,7 @@ module Gitlab } end - Gitlab::Database.bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert end def find_target_id diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index 80246fa1b77..7293de56a9a 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -6,15 +6,9 @@ module Gitlab class LabelsImporter include BulkImporting - attr_reader :project, :client, :existing_labels - - # project - An instance of `Project`. - # client - An instance of `Gitlab::GithubImport::Client`. # rubocop: disable CodeReuse/ActiveRecord - def initialize(project, client) - @project = project - @client = client - @existing_labels = project.labels.pluck(:title).to_set + def existing_labels + @existing_labels ||= project.labels.pluck(:title).to_set end # rubocop: enable CodeReuse/ActiveRecord @@ -51,6 +45,10 @@ module Gitlab def each_label client.labels(project.import_source) end + + def object_type + :label + end end end end diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 40248ecbd31..775afd5f53a 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -35,7 +35,11 @@ module Gitlab yield object end rescue StandardError => e - error(project.id, e) + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: importer_class.name, + exception: e + ) end end end diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 71ff7465d9b..d11b151bbe2 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -6,15 +6,9 @@ module Gitlab class MilestonesImporter include BulkImporting - attr_reader :project, :client, :existing_milestones - - # project - An instance of `Project` - # client - An instance of `Gitlab::GithubImport::Client` # rubocop: disable CodeReuse/ActiveRecord - def initialize(project, client) - @project = project - @client = client - @existing_milestones = project.milestones.pluck(:iid).to_set + def existing_milestones + @existing_milestones ||= project.milestones.pluck(:iid).to_set end # rubocop: enable CodeReuse/ActiveRecord @@ -55,6 +49,10 @@ module Gitlab def each_milestone client.milestones(project.import_source, state: 'all') end + + def object_type + :milestone + end end end end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index ae9996d81ef..1fd42a69fac 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -37,7 +37,7 @@ module Gitlab # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. - Gitlab::Database.bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert + Gitlab::Database.main.bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert rescue ActiveRecord::InvalidForeignKey # It's possible the project and the issue have been deleted since # scheduling this job. In this case we'll just skip creating the note. diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index b2f099761b1..2812fbd3dfe 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -40,11 +40,7 @@ module Gitlab # updating the timestamp. project.update_column(:last_repository_updated_at, Time.zone.now) - if Feature.enabled?(:fetch_remote_params, project, default_enabled: :yaml) - project.repository.fetch_remote('github', url: project.import_url, refmap: Gitlab::GithubImport.refmap, forced: false) - else - project.repository.fetch_remote('github', forced: false) - end + project.repository.fetch_remote(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: false) pname = project.path_with_namespace diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb index e389acbf877..bd65eb5899c 100644 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb @@ -37,43 +37,6 @@ module Gitlab review.id end - def each_object_to_import(&block) - if use_github_review_importer_query_only_unimported_merge_requests? - each_merge_request_to_import(&block) - else - each_merge_request_skipping_imported(&block) - end - end - - private - - attr_reader :merge_requests_already_imported_cache_key - - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62036#note_587181108 - def use_github_review_importer_query_only_unimported_merge_requests? - Feature.enabled?( - :github_review_importer_query_only_unimported_merge_requests, - default_enabled: :yaml - ) - end - - def each_merge_request_skipping_imported - project.merge_requests.find_each do |merge_request| - next if already_imported?(merge_request) - - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - - client - .pull_request_reviews(project.import_source, merge_request.iid) - .each do |review| - review.merge_request_id = merge_request.id - yield(review) - end - - mark_as_imported(merge_request) - end - end - # The worker can be interrupted, by rate limit for instance, # in different situations. To avoid requesting already imported data, # if the worker is interrupted: @@ -82,7 +45,7 @@ module Gitlab # - before importing all merge requests reviews # Merge requests that had all the reviews imported are cached with # `mark_merge_request_reviews_imported` - def each_merge_request_to_import + def each_object_to_import(&block) each_review_page do |page, merge_request| page.objects.each do |review| next if already_imported?(review) @@ -97,6 +60,10 @@ module Gitlab end end + private + + attr_reader :merge_requests_already_imported_cache_key + def each_review_page merge_requests_to_import.find_each do |merge_request| # The page counter needs to be scoped by merge request to avoid skipping diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index a3734ccf069..c1fbd868800 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -6,15 +6,9 @@ module Gitlab class ReleasesImporter include BulkImporting - attr_reader :project, :client, :existing_tags - - # project - An instance of `Project` - # client - An instance of `Gitlab::GithubImport::Client` # rubocop: disable CodeReuse/ActiveRecord - def initialize(project, client) - @project = project - @client = client - @existing_tags = project.releases.pluck(:tag).to_set + def existing_tags + @existing_tags ||= project.releases.pluck(:tag).to_set end # rubocop: enable CodeReuse/ActiveRecord @@ -50,6 +44,10 @@ module Gitlab def description_for(release) release.body.presence || "Release for tag #{release.tag_name}" end + + def object_type + :release + end end end end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 1401c92a44e..20068a33019 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -50,7 +50,7 @@ module Gitlab project.ensure_repository refmap = Gitlab::GithubImport.refmap - project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true, remote_name: 'github') + project.repository.fetch_as_mirror(project.import_url, refmap: refmap, forced: true) project.change_head(default_branch) if default_branch @@ -59,8 +59,6 @@ module Gitlab Repositories::HousekeepingService.new(project, :gc).execute true - rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e - fail_import("Failed to import the repository: #{e.message}") end def import_wiki_repository @@ -70,7 +68,8 @@ module Gitlab rescue ::Gitlab::Git::CommandError => e if e.message !~ /repository not exported/ project.create_wiki - fail_import("Failed to import the wiki: #{e.message}") + + raise e else true end @@ -84,11 +83,6 @@ module Gitlab project.update_column(:last_repository_updated_at, Time.zone.now) end - def fail_import(message) - project.import_state.mark_as_failed(message) - false - end - private def default_branch diff --git a/lib/gitlab/github_import/logger.rb b/lib/gitlab/github_import/logger.rb new file mode 100644 index 00000000000..980aa0a7982 --- /dev/null +++ b/lib/gitlab/github_import/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class Logger < ::Gitlab::Import::Logger + def default_attributes + super.merge(import_type: :github) + end + end + end +end diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb index e4835504c2d..4c9a8da601f 100644 --- a/lib/gitlab/github_import/object_counter.rb +++ b/lib/gitlab/github_import/object_counter.rb @@ -14,11 +14,16 @@ module Gitlab CACHING = Gitlab::Cache::Import::Caching class << self - def increment(project, object_type, operation) + # Increments the project and the global counters if the given value is >= 1 + def increment(project, object_type, operation, value: 1) + integer = value.to_i + + return if integer <= 0 + validate_operation!(operation) - increment_project_counter(project, object_type, operation) - increment_global_counter(object_type, operation) + increment_project_counter(project, object_type, operation, integer) + increment_global_counter(object_type, operation, integer) end def summary(project) @@ -41,7 +46,7 @@ module Gitlab # and it's used to report the health of the Github Importer # in the Grafana Dashboard # https://dashboards.gitlab.net/d/2zgM_rImz/github-importer?orgId=1 - def increment_global_counter(object_type, operation) + def increment_global_counter(object_type, operation, value) key = GLOBAL_COUNTER_KEY % { operation: operation, object_type: object_type @@ -51,18 +56,26 @@ module Gitlab object_type: object_type.to_s.humanize } - Gitlab::Metrics.counter(key.to_sym, description).increment + Gitlab::Metrics.counter(key.to_sym, description).increment(by: value) end # Project counters are short lived, in Redis, # and it's used to report how successful a project # import was with the #summary method. - def increment_project_counter(project, object_type, operation) - counter_key = PROJECT_COUNTER_KEY % { project: project.id, operation: operation, object_type: object_type } + def increment_project_counter(project, object_type, operation, value) + counter_key = PROJECT_COUNTER_KEY % { + project: project.id, + operation: operation, + object_type: object_type + } add_counter_to_list(project, operation, counter_key) - CACHING.increment(counter_key) + if Feature.disabled?(:import_redis_increment_by, default_enabled: :yaml) + CACHING.increment(counter_key) + else + CACHING.increment_by(counter_key, value) + end end def add_counter_to_list(project, operation, key) @@ -75,7 +88,7 @@ module Gitlab def validate_operation!(operation) unless operation.to_s.presence_in(OPERATIONS) - raise ArgumentError, "Operation must be #{OPERATIONS.join(' or ')}" + raise ArgumentError, "operation must be #{OPERATIONS.join(' or ')}" end end end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 4598429d568..8c76f5a9d94 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -49,9 +49,14 @@ module Gitlab retval rescue StandardError => e - error(project.id, e) + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: self.class.name, + exception: e, + fail_import: abort_on_failure + ) - raise e + raise(e) end # Imports all the objects in sequence in the current thread. @@ -165,6 +170,10 @@ module Gitlab raise NotImplementedError end + def abort_on_failure + false + end + # Any options to be passed to the method used for retrieving the data to # import. def collection_options @@ -174,36 +183,16 @@ module Gitlab private def info(project_id, extra = {}) - logger.info(log_attributes(project_id, extra)) - end - - def error(project_id, exception) - logger.error( - log_attributes( - project_id, - message: 'importer failed', - 'error.message': exception.message - ) - ) - - Gitlab::ErrorTracking.track_exception( - exception, - log_attributes(project_id) - ) + Logger.info(log_attributes(project_id, extra)) end def log_attributes(project_id, extra = {}) extra.merge( - import_source: :github, project_id: project_id, importer: importer_class.name, parallel: parallel? ) end - - def logger - @logger ||= Gitlab::Import::Logger.build - end end end end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 058cd1ebd57..f583ef39d13 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -120,10 +120,18 @@ module Gitlab read_id_from_cache(ID_FOR_EMAIL_CACHE_KEY % email) end - # Queries and caches the GitLab user ID for a GitHub user ID, if one was - # found. + # If importing from github.com, queries and caches the GitLab user ID for + # a GitHub user ID, if one was found. + # + # When importing from Github Enterprise, do not query user by Github ID + # since we only have users' Github ID from github.com. def id_for_github_id(id) - gitlab_id = query_id_for_github_id(id) || nil + gitlab_id = + if project.github_enterprise_import? + nil + else + query_id_for_github_id(id) + end Gitlab::Cache::Import::Caching.write(ID_CACHE_KEY % id, gitlab_id) end diff --git a/lib/gitlab/graphql/copy_field_description.rb b/lib/gitlab/graphql/copy_field_description.rb index edd73083ff2..ed2273bc91a 100644 --- a/lib/gitlab/graphql/copy_field_description.rb +++ b/lib/gitlab/graphql/copy_field_description.rb @@ -11,7 +11,7 @@ module Gitlab # are always identical to the corresponding query field descriptions. # # E.g.: - # argument :name, GraphQL::STRING_TYPE, description: copy_field_description(Types::UserType, :name) + # argument :name, GraphQL::Types::String, description: copy_field_description(Types::UserType, :name) def copy_field_description(type, field_name) type.fields[field_name.to_s.camelize(:lower)].description end diff --git a/lib/gitlab/graphql/markdown_field.rb b/lib/gitlab/graphql/markdown_field.rb index 0b5bde8d8d9..6188d860aba 100644 --- a/lib/gitlab/graphql/markdown_field.rb +++ b/lib/gitlab/graphql/markdown_field.rb @@ -19,7 +19,7 @@ module Gitlab # Adding complexity to rendered notes since that could cause queries. kwargs[:complexity] ||= 5 - field name, GraphQL::STRING_TYPE, **kwargs + field name, GraphQL::Types::String, **kwargs define_method resolver_method do # We need to `dup` the context so the MarkdownHelper doesn't modify it diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index afe1554aec1..f830af68e07 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -50,11 +50,7 @@ module Gitlab attr_reader :context def self.file_size_limit - if Feature.enabled?(:one_megabyte_file_size_limit) - 1024.kilobytes - else - Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] - end + Gitlab.config.extra['maximum_text_highlight_size_kilobytes'] end private_class_method :file_size_limit diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 7e45cd216f5..8a19f208adf 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -14,7 +14,7 @@ module Gitlab Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Gitlab::HTTP::ReadTotalTimeout ].freeze HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [ - SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, + EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep ].freeze @@ -43,16 +43,29 @@ module Gitlab options end - unless options.has_key?(:use_read_total_timeout) + options[:skip_read_total_timeout] = true if options[:skip_read_total_timeout].nil? && options[:stream_body] + + if options[:skip_read_total_timeout] return httparty_perform_request(http_method, path, options_with_timeouts, &block) end start_time = Gitlab::Metrics::System.monotonic_time read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT) + tracked_timeout_error = false httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| elapsed = Gitlab::Metrics::System.monotonic_time - start_time - raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" if elapsed > read_total_timeout + + if elapsed > read_total_timeout + error = ReadTotalTimeout.new("Request timed out after #{elapsed} seconds") + + raise error if options[:use_read_total_timeout] + + unless tracked_timeout_error + Gitlab::ErrorTracking.track_exception(error) + tracked_timeout_error = true + end + end block.call fragment if block end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 7ebbe9f1c14..5f1b9873fee 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -38,26 +38,26 @@ module Gitlab # Currently monthly updated manually by ~group::import PM. # https://gitlab.com/gitlab-org/gitlab/-/issues/18923 TRANSLATION_LEVELS = { - 'bg' => 1, + 'bg' => 0, 'cs_CZ' => 1, - 'de' => 17, + 'de' => 16, 'en' => 100, - 'eo' => 1, - 'es' => 38, - 'fil_PH' => 1, + 'eo' => 0, + 'es' => 36, + 'fil_PH' => 0, 'fr' => 12, - 'gl_ES' => 1, + 'gl_ES' => 0, 'id_ID' => 0, 'it' => 2, - 'ja' => 42, - 'ko' => 13, - 'nl_NL' => 1, - 'pl_PL' => 5, - 'pt_BR' => 21, - 'ru' => 29, + 'ja' => 39, + 'ko' => 12, + 'nl_NL' => 0, + 'pl_PL' => 6, + 'pt_BR' => 36, + 'ru' => 28, 'tr_TR' => 16, - 'uk' => 41, - 'zh_CN' => 67, + 'uk' => 40, + 'zh_CN' => 74, 'zh_HK' => 2, 'zh_TW' => 3 }.freeze diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index f8ea7a7adcd..e73c3afe9bd 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -11,7 +11,7 @@ module Gitlab # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when # importing very large projects. - result = Gitlab::Database # rubocop:disable Gitlab/BulkInsert + result = Gitlab::Database.main # rubocop:disable Gitlab/BulkInsert .bulk_insert(relation.table_name, [attributes], return_ids: true) result.first diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb new file mode 100644 index 00000000000..f808ed1b6e2 --- /dev/null +++ b/lib/gitlab/import/import_failure_service.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Import + class ImportFailureService + def self.track( + exception:, + import_state: nil, + project_id: nil, + error_source: nil, + fail_import: false + ) + new( + exception: exception, + import_state: import_state, + project_id: project_id, + error_source: error_source + ).execute(fail_import: fail_import) + end + + def initialize(exception:, import_state: nil, project_id: nil, error_source: nil) + if import_state.blank? && project_id.blank? + raise ArgumentError, 'import_state OR project_id must be provided' + end + + if project_id.blank? + @import_state = import_state + @project = import_state.project + else + @project = Project.find(project_id) + @import_state = @project.import_state + end + + @exception = exception + @error_source = error_source + end + + def execute(fail_import:) + track_exception + persist_failure + + import_state.mark_as_failed(exception.message) if fail_import + end + + private + + attr_reader :exception, :import_state, :project, :error_source + + def track_exception + attributes = { + import_type: project.import_type, + project_id: project.id, + source: error_source + } + + Gitlab::Import::Logger.error( + attributes.merge( + message: 'importer failed', + 'error.message': exception.message + ) + ) + + Gitlab::ErrorTracking.track_exception(exception, attributes) + end + + def persist_failure + project.import_failures.create( + source: error_source, + exception_class: exception.class.to_s, + exception_message: exception.message.truncate(255), + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id + ) + end + end + end +end diff --git a/lib/gitlab/import/logger.rb b/lib/gitlab/import/logger.rb index ab3e822a4e9..bd34aff734a 100644 --- a/lib/gitlab/import/logger.rb +++ b/lib/gitlab/import/logger.rb @@ -6,6 +6,10 @@ module Gitlab def self.file_name_noext 'importer' end + + def default_attributes + super.merge(feature_category: :importers) + end end end end diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index 97b34088e3e..dc80c92f507 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -27,7 +27,7 @@ module Gitlab end def read_hash - ActiveSupport::JSON.decode(IO.read(@path)) + Gitlab::Json.parse(IO.read(@path)) rescue StandardError => e Gitlab::ErrorTracking.log_exception(e) raise Gitlab::ImportExport::Error, 'Incorrect JSON format' diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index 4899bd3b0ee..510da61d3ab 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -47,8 +47,8 @@ module Gitlab private def json_decode(string) - ActiveSupport::JSON.decode(string) - rescue ActiveSupport::JSON.parse_error => e + Gitlab::Json.parse(string) + rescue JSON::ParserError => e Gitlab::ErrorTracking.log_exception(e) raise Gitlab::ImportExport::Error, 'Incorrect JSON format' end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index d1e013a151c..9d28e1abeab 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -31,10 +31,12 @@ module Gitlab end def execute - serialize_root + read_from_replica_if_available do + serialize_root - includes.each do |relation_definition| - serialize_relation(relation_definition) + includes.each do |relation_definition| + serialize_relation(relation_definition) + end end end @@ -166,6 +168,13 @@ module Gitlab ) ]) end + + def read_from_replica_if_available(&block) + return yield unless ::Feature.enabled?(:load_balancing_for_export_workers, type: :development, default_enabled: :yaml) + return yield unless ::Gitlab::Database::LoadBalancing.enable? + + ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block) + end end end end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index d73ae1410a3..9931b09e9ca 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -72,7 +72,7 @@ module Gitlab @lfs_json ||= begin json = IO.read(lfs_json_path) - ActiveSupport::JSON.decode(json) + Gitlab::Json.parse(json) rescue StandardError raise Gitlab::ImportExport::Error, 'Incorrect JSON format' end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index a84978a2a80..5633194a8f8 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -230,6 +230,7 @@ excluded_attributes: - :blocking_issues_count - :service_desk_reply_to - :upvotes_count + - :work_item_type_id merge_request: - :milestone_id - :sprint_id @@ -293,6 +294,7 @@ excluded_attributes: - :encrypted_token - :encrypted_token_iv - :enabled + - :integrated service_desk_setting: - :outgoing_name priorities: @@ -328,6 +330,7 @@ excluded_attributes: - :release_id project_members: - :source_id + - :invite_email_success metrics: - :merge_request_id - :pipeline_id diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index 0753625b978..b03dceba303 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -106,7 +106,7 @@ module Gitlab end def design? - klass == DesignManagement::Design + klass == ::DesignManagement::Design end def diff_commit_user? diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 8a64abb9f62..0f21a16793d 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -5,8 +5,21 @@ module Gitlab module RedisInterceptor APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze + # These are temporary to help with investigating + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1183 + DURATION_ERROR_THRESHOLD = 1.25.seconds + + class MysteryRedisDurationError < StandardError + attr_reader :backtrace + + def initialize(backtrace) + @backtrace = backtrace + end + end + def call(*args, &block) start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined + start_real_time = Time.now instrumentation_class.instance_count_request instrumentation_class.redis_cluster_validate!(args.first) @@ -27,6 +40,13 @@ module Gitlab instrumentation_class.add_duration(duration) instrumentation_class.add_call_details(duration, args) end + + if duration > DURATION_ERROR_THRESHOLD && Feature.enabled?(:report_on_long_redis_durations, default_enabled: :yaml) + Gitlab::ErrorTracking.track_exception(MysteryRedisDurationError.new(caller), + command: command_from_args(args), + duration: duration, + timestamp: start_real_time.iso8601(5)) + end end def write(command) diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index c10c4a7bedf..23acf1e8e86 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -30,6 +30,7 @@ module Gitlab instrument_cpu(payload) instrument_thread_memory_allocations(payload) instrument_load_balancing(payload) + instrument_pid(payload) end def instrument_gitaly(payload) @@ -99,6 +100,10 @@ module Gitlab payload[:cpu_s] = cpu_s.round(DURATION_PRECISION) if cpu_s end + def instrument_pid(payload) + payload[:pid] = Process.pid + end + def instrument_thread_memory_allocations(payload) counters = ::Gitlab::Memory::Instrumentation.measure_thread_memory_allocations( ::Gitlab::RequestContext.instance.thread_memory_allocations) diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index b87c9936570..0fa9f435b5c 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -7,9 +7,13 @@ module Gitlab Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker - Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit Youtrack WebexTeams + Prometheus Pushover Redmine Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack )).freeze + def self.namespaced_integrations + NAMESPACED_INTEGRATIONS + end + def cast(value) new_cast(value) || super end @@ -32,16 +36,12 @@ module Gitlab private - def namespaced_integrations - NAMESPACED_INTEGRATIONS - end - def new_cast(value) value = prepare_value(value) return unless value stripped_name = value.delete_suffix('Service') - return unless namespaced_integrations.include?(stripped_name) + return unless self.class.namespaced_integrations.include?(stripped_name) "Integrations::#{stripped_name}" end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index 3e7659db240..13d3bb2b8dc 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -40,6 +40,14 @@ module Gitlab @authenticated = result.response.is_a?(Net::HTTPOK) store_cookies(result) if options[:use_cookies] + # This is needed to make response.to_s work. HTTParty::Response internal uses a Net::HTTPResponse as @response. + # When a block is used, Net::HTTPResponse#body will be a Net::ReadAdapter instead of a String. + # In this case HTTParty::Response.to_s will default to inspecting the Net::HTTPResponse class instead + # of returning the content of body. + # See https://github.com/jnunemaker/httparty/blob/v0.18.1/lib/httparty/response.rb#L86-L92 + # See https://github.com/ruby/net-http/blob/v0.1.1/lib/net/http/response.rb#L346-L350 + result.response.body = result.body + result end diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb index 43280606bb6..ab748d67fbf 100644 --- a/lib/gitlab/jira_import/issue_serializer.rb +++ b/lib/gitlab/jira_import/issue_serializer.rb @@ -52,9 +52,10 @@ module Gitlab end def map_user_id(jira_user) - return unless jira_user&.dig('accountId') + jira_user_identifier = jira_user&.dig('accountId') || jira_user&.dig('key') + return unless jira_user_identifier - Gitlab::JiraImport.get_user_mapping(project.id, jira_user['accountId']) + Gitlab::JiraImport.get_user_mapping(project.id, jira_user_identifier) end def reporter diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index 4314c131ada..41c18f82a4b 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -58,7 +58,7 @@ module Gitlab private def parse_value(raw, klass) - value = ActiveSupport::JSON.decode(raw.to_s) + value = Gitlab::Json.parse(raw.to_s) case value when Hash then parse_entry(value, klass) @@ -66,7 +66,7 @@ module Gitlab else value end - rescue ActiveSupport::JSON.parse_error + rescue JSON::ParserError nil end diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb index 3a74df8dc8f..d0dcd232ecc 100644 --- a/lib/gitlab/json_logger.rb +++ b/lib/gitlab/json_logger.rb @@ -7,7 +7,7 @@ module Gitlab end def format_message(severity, timestamp, progname, message) - data = {} + data = default_attributes data[:severity] = severity data[:time] = timestamp.utc.iso8601(3) data[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id @@ -21,5 +21,11 @@ module Gitlab Gitlab::Json.dump(data) + "\n" end + + protected + + def default_attributes + {} + end end end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 86c0aa2b48d..45582f19214 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -5,6 +5,7 @@ module Gitlab INTERNAL_API_REQUEST_HEADER = 'Gitlab-Kas-Api-Request' VERSION_FILE = 'GITLAB_KAS_VERSION' JWT_ISSUER = 'gitlab-kas' + K8S_PROXY_PATH = 'k8s-proxy' include JwtAuthenticatable @@ -39,6 +40,12 @@ module Gitlab Gitlab.config.gitlab_kas.external_url end + def tunnel_url + uri = URI.join(external_url, K8S_PROXY_PATH) + uri.scheme = uri.scheme.in?(%w(grpcs wss)) ? 'https' : 'http' + uri.to_s + end + # Return GitLab KAS internal_url # # @return [String] internal_url diff --git a/lib/gitlab/kubernetes/default_namespace.rb b/lib/gitlab/kubernetes/default_namespace.rb index c95362b024b..c22c2fe394d 100644 --- a/lib/gitlab/kubernetes/default_namespace.rb +++ b/lib/gitlab/kubernetes/default_namespace.rb @@ -36,14 +36,17 @@ module Gitlab end end - def default_project_namespace(slug) - namespace_slug = "#{project.path}-#{project.id}".downcase - - if cluster.namespace_per_environment? - namespace_slug += "-#{slug}" - end + def default_project_namespace(environment_slug) + maybe_environment_suffix = cluster.namespace_per_environment? ? "-#{environment_slug}" : '' + suffix = "-#{project.id}#{maybe_environment_suffix}" + namespace = project_path_slug(63 - suffix.length) + suffix + Gitlab::NamespaceSanitizer.sanitize(namespace) + end - Gitlab::NamespaceSanitizer.sanitize(namespace_slug) + def project_path_slug(max_length) + Gitlab::NamespaceSanitizer + .sanitize(project.path.downcase) + .first(max_length) end ## diff --git a/lib/gitlab/kubernetes/kubeconfig/entry/cluster.rb b/lib/gitlab/kubernetes/kubeconfig/entry/cluster.rb new file mode 100644 index 00000000000..836517d4e1f --- /dev/null +++ b/lib/gitlab/kubernetes/kubeconfig/entry/cluster.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Kubeconfig + module Entry + class Cluster + attr_reader :name + + def initialize(name:, url:, ca_pem: nil) + @name = name + @url = url + @ca_pem = ca_pem + end + + def to_h + { + name: name, + cluster: cluster + } + end + + private + + attr_reader :url, :ca_pem + + def cluster + { + server: url, + 'certificate-authority-data': certificate_authority_data + }.compact + end + + def certificate_authority_data + return unless ca_pem.present? + + Base64.strict_encode64(ca_pem) + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kubeconfig/entry/context.rb b/lib/gitlab/kubernetes/kubeconfig/entry/context.rb new file mode 100644 index 00000000000..8ff17ab9cff --- /dev/null +++ b/lib/gitlab/kubernetes/kubeconfig/entry/context.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Kubeconfig + module Entry + class Context + attr_reader :name + + def initialize(name:, cluster:, user:, namespace: nil) + @name = name + @cluster = cluster + @user = user + @namespace = namespace + end + + def to_h + { + name: name, + context: context + } + end + + private + + attr_reader :cluster, :user, :namespace + + def context + { + cluster: cluster, + namespace: namespace, + user: user + }.compact + end + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kubeconfig/entry/user.rb b/lib/gitlab/kubernetes/kubeconfig/entry/user.rb new file mode 100644 index 00000000000..784f6d67802 --- /dev/null +++ b/lib/gitlab/kubernetes/kubeconfig/entry/user.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Kubeconfig + module Entry + class User + attr_reader :name + + def initialize(name:, token:) + @name = name + @token = token + end + + def to_h + { + name: name, + user: { token: token } + } + end + + private + + attr_reader :token + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/kubeconfig/template.rb b/lib/gitlab/kubernetes/kubeconfig/template.rb new file mode 100644 index 00000000000..da0861ee86a --- /dev/null +++ b/lib/gitlab/kubernetes/kubeconfig/template.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + module Kubeconfig + class Template + ENTRIES = { + cluster: Gitlab::Kubernetes::Kubeconfig::Entry::Cluster, + user: Gitlab::Kubernetes::Kubeconfig::Entry::User, + context: Gitlab::Kubernetes::Kubeconfig::Entry::Context + }.freeze + + def initialize + @clusters = [] + @users = [] + @contexts = [] + end + + def valid? + contexts.present? + end + + def add_cluster(**args) + clusters << new_entry(:cluster, **args) + end + + def add_user(**args) + users << new_entry(:user, **args) + end + + def add_context(**args) + contexts << new_entry(:context, **args) + end + + def to_h + { + apiVersion: 'v1', + kind: 'Config', + clusters: clusters.map(&:to_h), + users: users.map(&:to_h), + contexts: contexts.map(&:to_h) + } + end + + def to_yaml + YAML.dump(to_h.deep_stringify_keys) + end + + private + + attr_reader :clusters, :users, :contexts + + def new_entry(entry, **args) + ENTRIES.fetch(entry).new(**args) + end + end + end + end +end diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb index 1e5edb79f10..fc9fb5caa09 100644 --- a/lib/gitlab/language_detection.rb +++ b/lib/gitlab/language_detection.rb @@ -18,7 +18,7 @@ module Gitlab end # Newly detected languages, returned in a structure accepted by - # Gitlab::Database.bulk_insert + # Gitlab::Database.main.bulk_insert def insertions(programming_languages) lang_to_id = programming_languages.to_h { |p| [p.name, p.id] } diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb index d0702190ac0..db39904ea23 100644 --- a/lib/gitlab/markdown_cache.rb +++ b/lib/gitlab/markdown_cache.rb @@ -2,10 +2,14 @@ module Gitlab module MarkdownCache - # Increment this number every time the renderer changes its output. + # Increment this number to invalidate cached HTML from Markdown documents. + # Even when reverting an MR, we should increment this because we only + # persist the cache when the new version is higher. + # # Changing this value puts strain on the database, as every row with - # cached markdown needs to be updated. As a result, this line should - # not be changed. + # cached markdown needs to be updated. As a result, avoid changing + # this if the change to the renderer output is a new feature or a + # minor bug fix. # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330313 CACHE_COMMONMARK_VERSION = 28 CACHE_COMMONMARK_VERSION_START = 10 diff --git a/lib/gitlab/markdown_cache/active_record/extension.rb b/lib/gitlab/markdown_cache/active_record/extension.rb index 1de890c84f9..e21268c5fff 100644 --- a/lib/gitlab/markdown_cache/active_record/extension.rb +++ b/lib/gitlab/markdown_cache/active_record/extension.rb @@ -10,7 +10,9 @@ module Gitlab # Using before_update here conflicts with elasticsearch-model somehow before_create :refresh_markdown_cache, if: :invalidated_markdown_cache? before_update :refresh_markdown_cache, if: :invalidated_markdown_cache? - after_save :store_mentions!, if: :mentionable_attributes_changed? + # The import case needs to be fixed to avoid large number of + # SQL queries: https://gitlab.com/gitlab-org/gitlab/-/issues/21801 + after_save :store_mentions!, if: :mentionable_attributes_changed?, unless: ->(obj) { obj.is_a?(Importable) && obj.importing? } end # Always exclude _html fields from attributes (including serialization). @@ -37,6 +39,7 @@ module Gitlab def save_markdown(updates) return unless persisted? && Gitlab::Database.read_write? + return if cached_markdown_version.to_i < cached_markdown_version_in_database.to_i update_columns(updates) end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index b99261b5c4d..6ba336d37cd 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -13,7 +13,7 @@ module Gitlab "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500) }.freeze - HEALTH_ENDPOINT = /^\/-\/(liveness|readiness|health|metrics)\/?$/.freeze + HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze FEATURE_CATEGORY_DEFAULT = 'unknown' @@ -66,28 +66,28 @@ module Gitlab def call(env) method = env['REQUEST_METHOD'].downcase method = 'INVALID' unless HTTP_METHODS.key?(method) - started = Gitlab::Metrics::System.monotonic_time + started = ::Gitlab::Metrics::System.monotonic_time health_endpoint = health_endpoint?(env['PATH_INFO']) status = 'undefined' begin status, headers, body = @app.call(env) - elapsed = Gitlab::Metrics::System.monotonic_time - started + elapsed = ::Gitlab::Metrics::System.monotonic_time - started - if !health_endpoint && Gitlab::Metrics.record_duration_for_status?(status) - RequestsRackMiddleware.http_request_duration_seconds.observe({ method: method }, elapsed) + if !health_endpoint && ::Gitlab::Metrics.record_duration_for_status?(status) + self.class.http_request_duration_seconds.observe({ method: method }, elapsed) end [status, headers, body] rescue StandardError - RequestsRackMiddleware.rack_uncaught_errors_count.increment + self.class.rack_uncaught_errors_count.increment raise ensure if health_endpoint - RequestsRackMiddleware.http_health_requests_total.increment(status: status.to_s, method: method) + self.class.http_health_requests_total.increment(status: status.to_s, method: method) else - RequestsRackMiddleware.http_requests_total.increment( + self.class.http_requests_total.increment( status: status.to_s, method: method, feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 258aa93be38..52d80c3c27e 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -23,7 +23,7 @@ module Gitlab def safe_sample sample rescue StandardError => e - Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") + ::Gitlab::AppLogger.warn("#{self.class}: #{e}, stopping") stop end diff --git a/lib/gitlab/metrics/subscribers/action_cable.rb b/lib/gitlab/metrics/subscribers/action_cable.rb index 631b9209f14..9f955dfe79f 100644 --- a/lib/gitlab/metrics/subscribers/action_cable.rb +++ b/lib/gitlab/metrics/subscribers/action_cable.rb @@ -28,7 +28,7 @@ module Gitlab if event.payload.present? channel = event.payload[:channel_class] operation = operation_name_from(event.payload) - data_size = ::ActiveSupport::JSON.encode(event.payload[:data]).bytesize + data_size = Gitlab::Json.generate(event.payload[:data]).bytesize transmitted_bytes_histogram.observe({ channel: channel, operation: operation }, data_size) end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index e1f1f37c905..fa129025bfe 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -40,7 +40,7 @@ module Gitlab end def current_transaction - Transaction.current + ::Gitlab::Metrics::Transaction.current end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index a8fcad9ff9f..59b2f88041f 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -8,17 +8,15 @@ module Gitlab attach_to :active_record IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze - DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze - SQL_COMMANDS_WITH_COMMENTS_REGEX = /\A(\/\*.*\*\/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i.freeze + DB_COUNTERS = %i{count write_count cached_count}.freeze + SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze - DB_LOAD_BALANCING_COUNTERS = %i{ - db_replica_count db_replica_cached_count db_replica_wal_count db_replica_wal_cached_count - db_primary_count db_primary_cached_count db_primary_wal_count db_primary_wal_cached_count - }.freeze - DB_LOAD_BALANCING_DURATIONS = %i{db_primary_duration_s db_replica_duration_s}.freeze + DB_LOAD_BALANCING_ROLES = %i{replica primary}.freeze + DB_LOAD_BALANCING_COUNTERS = %i{count cached_count wal_count wal_cached_count}.freeze + DB_LOAD_BALANCING_DURATIONS = %i{duration_s}.freeze SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze @@ -40,9 +38,10 @@ module Gitlab payload = event.payload return if ignored_query?(payload) - increment(:db_count) - increment(:db_cached_count) if cached_query?(payload) - increment(:db_write_count) unless select_sql_command?(payload) + db_config_name = db_config_name(event.payload) + increment(:count, db_config_name: db_config_name) + increment(:cached_count, db_config_name: db_config_name) if cached_query?(payload) + increment(:write_count, db_config_name: db_config_name) unless select_sql_command?(payload) observe(:gitlab_sql_duration_seconds, event) do buckets SQL_DURATION_BUCKET @@ -61,24 +60,17 @@ module Gitlab return {} unless Gitlab::SafeRequestStore.active? {}.tap do |payload| - DB_COUNTERS.each do |counter| - payload[counter] = Gitlab::SafeRequestStore[counter].to_i + db_counter_keys.each do |key| + payload[key] = Gitlab::SafeRequestStore[key].to_i end if ::Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? - DB_LOAD_BALANCING_COUNTERS.each do |counter| + load_balancing_metric_counter_keys.each do |counter| payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i end - DB_LOAD_BALANCING_DURATIONS.each do |duration| - payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) - end - if Feature.enabled?(:multiple_database_metrics, default_enabled: :yaml) - ::Gitlab::SafeRequestStore[:duration_by_database]&.each do |dbname, duration_by_role| - duration_by_role.each do |db_role, duration| - payload[:"db_#{db_role}_#{dbname}_duration_s"] = duration.to_f.round(3) - end - end + load_balancing_metric_duration_keys.each do |duration| + payload[duration] = ::Gitlab::SafeRequestStore[duration].to_f.round(3) end end end @@ -92,12 +84,15 @@ module Gitlab def increment_db_role_counters(db_role, payload) cached = cached_query?(payload) - increment("db_#{db_role}_count".to_sym) - increment("db_#{db_role}_cached_count".to_sym) if cached + + db_config_name = db_config_name(payload) + + increment(:count, db_role: db_role, db_config_name: db_config_name) + increment(:cached_count, db_role: db_role, db_config_name: db_config_name) if cached if wal_command?(payload) - increment("db_#{db_role}_wal_count".to_sym) - increment("db_#{db_role}_wal_cached_count".to_sym) if cached + increment(:wal_count, db_role: db_role, db_config_name: db_config_name) + increment(:wal_cached_count, db_role: db_role, db_config_name: db_config_name) if cached end end @@ -109,15 +104,13 @@ module Gitlab return unless ::Gitlab::SafeRequestStore.active? duration = event.duration / 1000.0 - duration_key = "db_#{db_role}_duration_s".to_sym + duration_key = compose_metric_key(:duration_s, db_role) ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration # Per database metrics - dbname = ::Gitlab::Database.dbname(event.payload[:connection]) - ::Gitlab::SafeRequestStore[:duration_by_database] ||= {} - ::Gitlab::SafeRequestStore[:duration_by_database][dbname] ||= {} - ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] ||= 0 - ::Gitlab::SafeRequestStore[:duration_by_database][dbname][db_role] += duration + db_config_name = db_config_name(event.payload) + duration_key = compose_metric_key(:duration_s, db_role, db_config_name) + ::Gitlab::SafeRequestStore[duration_key] = (::Gitlab::SafeRequestStore[duration_key].presence || 0) + duration end def ignored_query?(payload) @@ -132,19 +125,84 @@ module Gitlab payload[:sql].match(SQL_COMMANDS_WITH_COMMENTS_REGEX) end - def increment(counter) - current_transaction&.increment("gitlab_transaction_#{counter}_total".to_sym, 1) + def increment(counter, db_config_name:, db_role: nil) + log_key = compose_metric_key(counter, db_role) - Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 + prometheus_key = if db_role + :"gitlab_transaction_db_#{db_role}_#{counter}_total" + else + :"gitlab_transaction_db_#{counter}_total" + end + + if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) + else + current_transaction&.increment(prometheus_key, 1) + end + + Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 + + # To avoid confusing log keys we only log the db_config_name metrics + # when we are also logging the db_role. Otherwise it will be hard to + # tell if the log key is referring to a db_role OR a db_config_name. + if db_role.present? && db_config_name.present? + log_key = compose_metric_key(counter, db_role, db_config_name) + Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 + end end def observe(histogram, event, &block) - current_transaction&.observe(histogram, event.duration / 1000.0, &block) + db_config_name = db_config_name(event.payload) + + if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + current_transaction&.observe(histogram, event.duration / 1000.0, { db_config_name: db_config_name }, &block) + else + current_transaction&.observe(histogram, event.duration / 1000.0, &block) + end end def current_transaction ::Gitlab::Metrics::WebTransaction.current || ::Gitlab::Metrics::BackgroundTransaction.current end + + def db_config_name(payload) + ::Gitlab::Database.db_config_name(payload[:connection]) + end + + def self.db_counter_keys + DB_COUNTERS.map { |c| compose_metric_key(c) } + end + + def self.load_balancing_metric_counter_keys + load_balancing_metric_keys(DB_LOAD_BALANCING_COUNTERS) + end + + def self.load_balancing_metric_duration_keys + load_balancing_metric_keys(DB_LOAD_BALANCING_DURATIONS) + end + + def self.load_balancing_metric_keys(metrics) + [].tap do |counters| + DB_LOAD_BALANCING_ROLES.each do |role| + metrics.each do |metric| + counters << compose_metric_key(metric, role) + next unless ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + + ::Gitlab::Database.db_config_names.each do |config_name| + counters << compose_metric_key(metric, role, config_name) + end + end + end + end + end + + def compose_metric_key(metric, db_role = nil, db_config_name = nil) + self.class.compose_metric_key(metric, db_role, db_config_name) + end + + def self.compose_metric_key(metric, db_role = nil, db_config_name = nil) + [:db, db_role, db_config_name, metric].compact.join("_").to_sym + end end end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index b274d2b1079..45344e79796 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -65,7 +65,7 @@ module Gitlab private def current_transaction - Transaction.current + ::Gitlab::Metrics::Transaction.current end def metric_cache_operation_duration_seconds diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 4b65bbcc791..a1a0356ff58 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -127,23 +127,25 @@ module Gitlab def project_for_paths(paths, request) project = Project.where_full_path_in(paths).first - return unless Ability.allowed?(current_user(request, project), :read_project, project) + + return unless authentication_result(request, project).can_perform_action_on_project?(:read_project, project) project end - def current_user(request, project) - return unless has_basic_credentials?(request) + def authentication_result(request, project) + empty_result = Gitlab::Auth::Result::EMPTY + return empty_result unless has_basic_credentials?(request) login, password = user_name_and_password(request) auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip) - return unless auth_result.success? + return empty_result unless auth_result.success? - return unless auth_result.actor&.can?(:access_git) + return empty_result unless auth_result.can?(:access_git) - return unless auth_result.authentication_abilities.include?(:read_project) + return empty_result unless auth_result.authentication_abilities_include?(:read_project) - auth_result.actor + auth_result end end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 329041e3ba2..30b3fe3d893 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -177,7 +177,7 @@ module Gitlab @app.call(env) end rescue UploadedFile::InvalidPathError => e - [400, { 'Content-Type' => 'text/plain' }, e.message] + [400, { 'Content-Type' => 'text/plain' }, [e.message]] end end end diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index b29240985f1..b5e304599ab 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -11,7 +11,7 @@ module Gitlab retry_attempts = 0 begin - ActiveRecord::Base.transaction do + ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases yield(subject) end rescue ActiveRecord::StaleObjectError diff --git a/lib/gitlab/otp_key_rotator.rb b/lib/gitlab/otp_key_rotator.rb index b65c8613d00..38618a0ac06 100644 --- a/lib/gitlab/otp_key_rotator.rb +++ b/lib/gitlab/otp_key_rotator.rb @@ -36,7 +36,7 @@ module Gitlab raise ArgumentError, "New key is too short! Must be 256 bits" if new_key.size < 64 write_csv do |csv| - ActiveRecord::Base.transaction do + User.transaction do User.with_two_factor.in_batches do |relation| # rubocop: disable Cop/InBatches rows = relation.pluck(:id, :encrypted_otp_secret, :encrypted_otp_secret_iv, :encrypted_otp_secret_salt) rows.each do |row| @@ -54,7 +54,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def rollback! - ActiveRecord::Base.transaction do + User.transaction do CSV.foreach(filename, headers: HEADERS, return_headers: false) do |row| User.where(id: row['user_id']).update_all(encrypted_otp_secret: row['old_value']) end diff --git a/lib/gitlab/pagination/keyset/column_condition_builder.rb b/lib/gitlab/pagination/keyset/column_condition_builder.rb new file mode 100644 index 00000000000..ca436000abe --- /dev/null +++ b/lib/gitlab/pagination/keyset/column_condition_builder.rb @@ -0,0 +1,206 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class ColumnConditionBuilder + # This class builds the WHERE conditions for the keyset pagination library. + # It produces WHERE conditions for one column at a time. + # + # Requisite 1: Only the last column (columns.last) is non-nullable and distinct. + # Requisite 2: Only one column is distinct and non-nullable. + # + # Scenario: We want to order by columns named X, Y and Z and build the conditions + # used in the WHERE clause of a pagination query using a set of cursor values. + # X is the column definition for a nullable column + # Y is the column definition for a non-nullable but not distinct column + # Z is the column definition for a distinct, non-nullable column used as a tie breaker. + # + # Then the method is initially invoked with these arguments: + # columns = [ColumnDefinition for X, ColumnDefinition for Y, ColumnDefinition for Z] + # values = { X: x, Y: y, Z: z } => these represent cursor values for pagination + # (x could be nil since X is nullable) + # current_conditions is initialized to [] to store the result during the iteration calls + # invoked within the Order#build_where_values method. + # + # The elements of current_conditions are instances of Arel::Nodes and - + # will be concatenated using OR or UNION to be used in the WHERE clause. + # + # Example: Let's say we want to build WHERE clause conditions for + # ORDER BY X DESC NULLS LAST, Y ASC, Z DESC + # + # Iteration 1: + # columns = [X, Y, Z] + # At the end, current_conditions should be: + # [(Z < z)] + # + # Iteration 2: + # columns = [X, Y] + # At the end, current_conditions should be: + # [(Y > y) OR (Y = y AND Z < z)] + # + # Iteration 3: + # columns = [X] + # At the end, current_conditions should be: + # [((X IS NOT NULL AND Y > y) OR (X IS NOT NULL AND Y = y AND Z < z)) + # OR + # ((x IS NULL) OR (X IS NULL))] + # + # Parameters: + # + # - columns: instance of ColumnOrderDefinition + # - value: cursor value for the column + def initialize(column, value) + @column = column + @value = value + end + + def where_conditions(current_conditions) + return not_nullable_conditions(current_conditions) if column.not_nullable? + return nulls_first_conditions(current_conditions) if column.nulls_first? + + # Here we are dealing with the case of column_definition.nulls_last? + # Suppose ORDER BY X DESC NULLS FIRST, Y ASC, Z DESC is the ordering clause + # and we already have built the conditions for columns Y and Z. + # + # We first need a set of conditions to use when x (the value for X) is NULL: + # null_conds = [ + # (x IS NULL AND X IS NULL AND Y y), + # (x IS NOT NULL AND X = x AND Y = y AND Z < z), + tie_breaking_conds = current_conditions.map do |conditional| + Arel::Nodes::And.new([column_equals_to_value, conditional]) + end + + non_null_conds = [column_is_null, compare_column_with_value, *tie_breaking_conds].map do |conditional| + Arel::Nodes::And.new([value_is_not_null, conditional]) + end + + [*null_conds, *non_null_conds] + end + + private + + # WHEN THE COLUMN IS NON-NULLABLE AND DISTINCT + # Per Assumption 1, only the last column can be non-nullable and distinct + # (column Z is non-nullable/distinct and comes last in the example). + # So the Order#build_where_conditions is being called for the first time with current_conditions = []. + # + # At the end of the call, we should expect: + # current_conditions should be [(Z < z)] + # + # WHEN THE COLUMN IS NON-NULLABLE BUT NOT DISTINCT + # Let's say Z has been processed and we are about to process the column Y next. + # (per requisite 1, if a non-nullable but not distinct column is being processed, + # at the least, the conditional for the non-nullable/distinct column exists) + # + # At the start of the method call: + # current_conditions = [(Z < z)] + # comparison_node = (Y < y) + # eqaulity_node = (Y = y) + # + # We should add a comparison node for the next column Y, (Y < y) + # then break a tie using the previous conditionals, (Y = y AND Z < z) + # + # At the end of the call, we should expect: + # current_conditions = [(Y < y), (Y = y AND Z < z)] + def not_nullable_conditions(current_conditions) + tie_break_conds = current_conditions.map do |conditional| + Arel::Nodes::And.new([column_equals_to_value, conditional]) + end + + [compare_column_with_value, *tie_break_conds] + end + + def nulls_first_conditions(current_conditions) + # Using the same scenario described earlier, + # suppose the ordering clause is ORDER BY X DESC NULLS FIRST, Y ASC, Z DESC + # and we have built the conditions for columns Y and Z in previous iterations: + # + # current_conditions = [(Y > y), (Y = y AND Z < z)] + # + # In this branch of the iteration, + # we first need a set of conditions to use when m (the value for M) is NULL: + # null_conds = [ + # (x IS NULL AND X IS NULL AND Y > y), + # (x IS NULL AND X IS NULL AND Y = y AND Z < z), + # (x IS NULL AND X IS NOT NULL)] + # + # Note that when x has an actual value, say x = 3, null_conds evalutes to FALSE. + tie_breaking_conds = current_conditions.map do |conditional| + Arel::Nodes::And.new([column_is_null, conditional]) + end + + null_conds = [*tie_breaking_conds, column_is_not_null].map do |conditional| + Arel::Nodes::And.new([value_is_null, conditional]) + end + + # We then need a set of conditions to use when m has an actual value: + # non_null_conds = [ + # (x IS NOT NULL AND X < x), + # (x IS NOT NULL AND X = x AND Y > y), + # (x IS NOT NULL AND X = x AND Y = y AND Z < z)] + # + # Note again that when x IS NULL, non_null_conds evaluates to FALSE. + tie_breaking_conds = current_conditions.map do |conditional| + Arel::Nodes::And.new([column_equals_to_value, conditional]) + end + + # The combined OR condition (null_where_cond OR non_null_where_cond) will return a correct result - + # without having to account for whether x is nil or an actual value at the application level. + non_null_conds = [compare_column_with_value, *tie_breaking_conds].map do |conditional| + Arel::Nodes::And.new([value_is_not_null, conditional]) + end + + [*null_conds, *non_null_conds] + end + + def column_equals_to_value + @equality_node ||= column.column_expression.eq(value) + end + + def column_is_null + @column_is_null ||= column.column_expression.eq(nil) + end + + def column_is_not_null + @column_is_not_null ||= column.column_expression.not_eq(nil) + end + + def value_is_null + @value_is_null ||= build_quoted_value.eq(nil) + end + + def value_is_not_null + @value_is_not_null ||= build_quoted_value.not_eq(nil) + end + + def compare_column_with_value + if column.descending_order? + column.column_expression.lt(value) + else + column.column_expression.gt(value) + end + end + + # Turns the given value to an SQL literal by casting it to the proper format. + def build_quoted_value + return value if value.instance_of?(Arel::Nodes::SqlLiteral) + + Arel::Nodes.build_quoted(value, column.column_expression) + end + + attr_reader :column, :value + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index 19d44ee69dd..ccfa9334a12 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -141,24 +141,10 @@ module Gitlab return use_composite_row_comparison(values) if composite_row_comparison_possible? - where_values = [] - - reversed_column_definitions = column_definitions.reverse - reversed_column_definitions.each_with_index do |column_definition, i| - value = values[column_definition.attribute_name] - - conditions_for_column(column_definition, value).each do |condition| - column_definitions_after_index = reversed_column_definitions.last(column_definitions.reverse.size - i - 1) - - equal_conditon_for_rest = column_definitions_after_index.map do |definition| - definition.column_expression.eq(values[definition.attribute_name]) - end - - where_values << Arel::Nodes::Grouping.new(Arel::Nodes::And.new([condition, *equal_conditon_for_rest].compact)) - end - end - - where_values + column_definitions + .map { ColumnConditionBuilder.new(_1, values[_1.attribute_name]) } + .reverse + .reduce([]) { |where_conditions, column| column.where_conditions(where_conditions) } end def where_values_with_or_query(values) @@ -222,32 +208,6 @@ module Gitlab scope end - def conditions_for_column(column_definition, value) - conditions = [] - # Depending on the order, build a query condition fragment for taking the next rows - if column_definition.distinct? || (!column_definition.distinct? && value.present?) - conditions << compare_column_with_value(column_definition, value) - end - - # When the column is nullable, additional conditions for NULL a NOT NULL values are necessary. - # This depends on the position of the nulls (top or bottom of the resultset). - if column_definition.nulls_first? && value.blank? - conditions << column_definition.column_expression.not_eq(nil) - elsif column_definition.nulls_last? && value.present? - conditions << column_definition.column_expression.eq(nil) - end - - conditions - end - - def compare_column_with_value(column_definition, value) - if column_definition.descending_order? - column_definition.column_expression.lt(value) - else - column_definition.column_expression.gt(value) - end - end - def build_or_query(expressions) return [] if expressions.blank? diff --git a/lib/gitlab/pagination/keyset/simple_order_builder.rb b/lib/gitlab/pagination/keyset/simple_order_builder.rb index 76d6bbadaa4..5e79910a3e9 100644 --- a/lib/gitlab/pagination/keyset/simple_order_builder.rb +++ b/lib/gitlab/pagination/keyset/simple_order_builder.rb @@ -122,6 +122,7 @@ module Gitlab return unless attribute return unless tie_breaker_attribute + return unless attribute.respond_to?(:name) model_class.column_names.include?(attribute.name.to_s) && arel_table[primary_key].to_s == tie_breaker_attribute.to_s diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 5c9b029a107..b2179d80a18 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -119,18 +119,18 @@ module Gitlab def self.with_custom_logger(logger) original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging - original_activerecord_logger = ActiveRecord::Base.logger + original_activerecord_logger = ApplicationRecord.logger original_actioncontroller_logger = ActionController::Base.logger if logger ActiveSupport::LogSubscriber.colorize_logging = false - ActiveRecord::Base.logger = logger + ApplicationRecord.logger = logger ActionController::Base.logger = logger end yield.tap do ActiveSupport::LogSubscriber.colorize_logging = original_colorize_logging - ActiveRecord::Base.logger = original_activerecord_logger + ApplicationRecord.logger = original_activerecord_logger ActionController::Base.logger = original_actioncontroller_logger end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index e52023c4612..fb9447f9665 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -8,7 +8,8 @@ module Gitlab @project = project @repository_ref = repository_ref.presence - super(current_user, query, [project], order_by: order_by, sort: sort, filters: filters) + # use the default filter for project searches since we are already limiting by a single project + super(current_user, query, [project], order_by: order_by, sort: sort, filters: filters, default_project_filter: true) end def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) @@ -24,7 +25,7 @@ module Gitlab when 'users' users.page(page).per(per_page) else - super(scope, page: page, per_page: per_page, without_count: false) + super(scope, page: page, per_page: per_page, without_count: true) end end diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb index 138fae7b641..4bfd526914b 100644 --- a/lib/gitlab/query_limiting/active_support_subscriber.rb +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -6,10 +6,10 @@ module Gitlab attach_to :active_record def sql(event) - return if !Transaction.current || event.payload.fetch(:cached, event.payload[:name] == 'CACHE') + return if !::Gitlab::QueryLimiting::Transaction.current || event.payload.fetch(:cached, event.payload[:name] == 'CACHE') - Transaction.current.increment - Transaction.current.executed_sql(event.payload[:sql]) + ::Gitlab::QueryLimiting::Transaction.current.increment + ::Gitlab::QueryLimiting::Transaction.current.executed_sql(event.payload[:sql]) end end end diff --git a/lib/gitlab/query_limiting/middleware.rb b/lib/gitlab/query_limiting/middleware.rb index 714fe42884a..76de547b14f 100644 --- a/lib/gitlab/query_limiting/middleware.rb +++ b/lib/gitlab/query_limiting/middleware.rb @@ -13,7 +13,7 @@ module Gitlab end def call(env) - transaction, retval = Transaction.run do + transaction, retval = ::Gitlab::QueryLimiting::Transaction.run do @app.call(env) end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index efe07aa8ab2..cf5c9296d8c 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -241,8 +241,49 @@ module Gitlab "#{comment} #{TABLEFLIP}" end + desc _('Set severity') + explanation _('Sets the severity') + params '1 / S1 / Critical' + types Issue + condition do + !quick_action_target.persisted? || quick_action_target.supports_severity? + end + parse_params do |severity| + find_severity(severity) + end + command :severity do |severity| + next unless quick_action_target.supports_severity? + + if severity + if quick_action_target.persisted? + ::Issues::UpdateService.new(project: quick_action_target.project, current_user: current_user, params: { severity: severity }).execute(quick_action_target) + else + quick_action_target.build_issuable_severity(severity: severity) + end + + @execution_message[:severity] = _("Severity updated to %{severity}.") % { severity: severity.capitalize } + else + @execution_message[:severity] = _('No severity matches the provided parameter') + end + end + private + def find_severity(severity_param) + return unless severity_param + + severity_param = severity_param.downcase + severities = IssuableSeverity::SEVERITY_QUICK_ACTION_PARAMS.values.map { |vals| vals.map(&:downcase) } + + matched_severity = severities.find do |severity_values| + severity_values.include?(severity_param) + end + + return unless matched_severity + + matched_severity[0] + end + def run_label_command(labels:, command:, updates_key:) return if labels.empty? diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index 7fee1d0727f..e4a92ed5122 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -10,11 +10,6 @@ module Gitlab @expires_in = expires_in end - # NOTE Remove as part of #331319 - def old_cache_key(key) - "#{cache_namespace}:#{key}:set" - end - def cache_key(key) super(key) end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index bbcc2732e89..3c8ac07215d 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -8,6 +8,10 @@ require 'active_support/core_ext/hash/keys' require 'active_support/core_ext/module/delegation' require 'active_support/core_ext/string/inflections' +# Explicitly load Redis::Store::Factory so we can read Redis configuration in +# TestEnv +require 'redis/store/factory' + module Gitlab module Redis class Wrapper diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 0bd2ac180c3..698a417283e 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -184,19 +184,19 @@ module Gitlab # - Must not have a scheme, such as http:// or https:// # - Must not have a port number, such as :8080 or :8443 - @go_package_regex ||= / + @go_package_regex ||= %r{ \b (?# word boundary) (? [0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])? (?# first domain) (?:\.[0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])?)* (?# inner domains) \.[a-z]{2,} (?# top-level domain) ) - (?\/(?: - [-\/$_.+!*'(),0-9a-z] (?# plain URL character) + (?/(?: + [-/$_.+!*'(),0-9a-z] (?# plain URL character) | %[0-9a-f]{2})* (?# URL encoded character) )? (?# path) \b (?# word boundary) - /ix.freeze + }ix.freeze end def generic_package_version_regex @@ -416,7 +416,7 @@ module Gitlab end def base64_regex - @base64_regex ||= /(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?/.freeze + @base64_regex ||= %r{(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?}.freeze end def feature_flag_regex diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 7de53c4b3ff..3061fb96190 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -13,11 +13,6 @@ module Gitlab @expires_in = expires_in end - # NOTE Remove as part of #331319 - def old_cache_key(type) - "#{type}:#{namespace}:set" - end - def cache_key(type) super("#{type}:#{namespace}") end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index e6851af8264..90513e346f2 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -168,7 +168,7 @@ module Gitlab issues = IssuesFinder.new(current_user, issuable_params.merge(finder_params)).execute unless default_project_filter - issues = issues.where(project_id: project_ids_relation) # rubocop: disable CodeReuse/ActiveRecord + issues = issues.in_projects(project_ids_relation) end apply_sort(issues, scope: 'issues') diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 9fc7a44ec99..feb2c3c1d7d 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -10,11 +10,6 @@ module Gitlab @expires_in = expires_in end - # NOTE Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/331319 - def old_cache_key(key) - "#{key}:set" - end - def cache_key(key) "#{cache_namespace}:#{key}:set" end @@ -25,7 +20,6 @@ module Gitlab with do |redis| keys_to_expire = keys.map { |key| cache_key(key) } - keys_to_expire += keys.map { |key| old_cache_key(key) } # NOTE Remove as part of #331319 Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do redis.unlink(*keys_to_expire) diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 7ed1958a8d0..751405f1045 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -21,7 +21,7 @@ module Gitlab end rescue Errno::EEXIST puts 'Skipping config.toml generation:' - puts 'A configuration file already exists.' + puts "A configuration file for #{config_path} already exists." rescue ArgumentError => e puts 'Skipping config.toml generation:' puts e.message @@ -32,7 +32,7 @@ module Gitlab extend Gitlab::SetupHelper class << self def configuration_toml(dir, _, _) - config = { redis: { URL: redis_url } } + config = { redis: { URL: redis_url, DB: redis_db } } TomlRB.dump(config) end @@ -41,6 +41,10 @@ module Gitlab Gitlab::Redis::SharedState.url end + def redis_db + Gitlab::Redis::SharedState.params.fetch(:db, 0) + end + def get_config_path(dir, _) File.join(dir, 'config_path') end diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index e20834fa912..05319ba17a2 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -37,6 +37,7 @@ module Gitlab @logger.formatter = ::Gitlab::SidekiqLogging::JSONFormatter.new @rails_path = Dir.pwd @dryrun = false + @list_queues = false end def run(argv = ARGV) @@ -47,6 +48,11 @@ module Gitlab option_parser.parse!(argv) + if @dryrun && @list_queues + raise CommandError, + 'The --dryrun and --list-queues options are mutually exclusive' + end + worker_metadatas = SidekiqConfig::CliMethods.worker_metadatas(@rails_path) worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path) @@ -73,6 +79,12 @@ module Gitlab 'No queues found, you must select at least one queue' end + if @list_queues + puts queue_groups.map(&:sort) # rubocop:disable Rails/Output + + return + end + unless @dryrun @logger.info("Starting cluster with #{queue_groups.length} processes") end @@ -202,6 +214,10 @@ module Gitlab opt.on('-d', '--dryrun', 'Print commands that would be run without this flag, and quit') do |int| @dryrun = true end + + opt.on('--list-queues', 'List matching queues, and quit') do |int| + @list_queues = true + end end end end diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb index ef0dce0cf84..b7f53da8e00 100644 --- a/lib/gitlab/sidekiq_config/dummy_worker.rb +++ b/lib/gitlab/sidekiq_config/dummy_worker.rb @@ -5,7 +5,6 @@ module Gitlab # For queues that don't have explicit workers - default and mailers class DummyWorker ATTRIBUTE_METHODS = { - queue: :queue, name: :name, feature_category: :get_feature_category, has_external_dependencies: :worker_has_external_dependencies?, @@ -20,6 +19,10 @@ module Gitlab @attributes = attributes end + def generated_queue_name + @attributes[:queue] + end + def queue_namespace nil end diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb index aea4209f631..a343573440f 100644 --- a/lib/gitlab/sidekiq_config/worker.rb +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -6,9 +6,11 @@ module Gitlab include Comparable attr_reader :klass - delegate :feature_category_not_owned?, :get_feature_category, :get_sidekiq_options, - :get_tags, :get_urgency, :get_weight, :get_worker_resource_boundary, - :idempotent?, :queue, :queue_namespace, :worker_has_external_dependencies?, + + delegate :feature_category_not_owned?, :generated_queue_name, :get_feature_category, + :get_sidekiq_options, :get_tags, :get_urgency, :get_weight, + :get_worker_resource_boundary, :idempotent?, :queue_namespace, + :worker_has_external_dependencies?, to: :klass def initialize(klass, ee:) @@ -35,7 +37,7 @@ module Gitlab # Put namespaced queues first def to_sort - [queue_namespace ? 0 : 1, queue] + [queue_namespace ? 0 : 1, generated_queue_name] end # YAML representation @@ -45,7 +47,7 @@ module Gitlab def to_yaml { - name: queue, + name: generated_queue_name, worker_name: klass.name, feature_category: get_feature_category, has_external_dependencies: worker_has_external_dependencies?, @@ -62,7 +64,7 @@ module Gitlab end def queue_and_weight - [queue, get_weight] + [generated_queue_name, get_weight] end def retries diff --git a/lib/gitlab/signed_tag.rb b/lib/gitlab/signed_tag.rb new file mode 100644 index 00000000000..3b22cb7622d --- /dev/null +++ b/lib/gitlab/signed_tag.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + class SignedTag + include Gitlab::Utils::StrongMemoize + + def initialize(repository, tag) + @repository = repository + @tag = tag + + if Feature.enabled?(:get_tag_signatures) + @signature_data = Gitlab::Git::Tag.extract_signature_lazily(repository, tag.id) if repository + else + @signature_data = [signature_text_of_message.b, signed_text_of_message.b] + end + end + + def signature + return unless @tag.has_signature? + end + + def signature_text + @signature_data&.fetch(0) + end + + def signed_text + @signature_data&.fetch(1) + end + + private + + def signature_text_of_message + @tag.message.slice(@tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1) + rescue StandardError + nil + end + + def signed_text_of_message + %{object #{@tag.target_commit.id} +type commit +tag #{@tag.name} +tagger #{@tag.tagger.name} <#{@tag.tagger.email}> #{@tag.tagger.date.seconds} #{@tag.tagger.timezone} + +#{@tag.message.gsub(/-----BEGIN SIGNED MESSAGE-----(.*)-----END SIGNED MESSAGE-----/m, "")}} + end + end +end diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index 714ca77c3e5..71bc0dc0123 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -48,7 +48,7 @@ module Gitlab *Documentation* For more information about GitLab chatops, refer to its - documentation: https://docs.gitlab.com/ee/ci/chatops/README.html. + documentation: https://docs.gitlab.com/ee/ci/chatops/index.html. MESSAGE message diff --git a/lib/gitlab/sql/glob.rb b/lib/gitlab/sql/glob.rb index f3421bd95d2..adc6ccbd1b9 100644 --- a/lib/gitlab/sql/glob.rb +++ b/lib/gitlab/sql/glob.rb @@ -17,7 +17,7 @@ module Gitlab end def q(string) - ActiveRecord::Base.connection.quote(string) + ApplicationRecord.connection.quote(string) end end end diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb index 59a808eafa9..18275da3ef0 100644 --- a/lib/gitlab/sql/set_operator.rb +++ b/lib/gitlab/sql/set_operator.rb @@ -19,6 +19,7 @@ module Gitlab # Project.where("id IN (#{sql})") class SetOperator def initialize(relations, remove_duplicates: true, remove_order: true) + verify_select_values!(relations) if Rails.env.test? || Rails.env.development? @relations = relations @remove_duplicates = remove_duplicates @remove_order = remove_order @@ -33,7 +34,7 @@ module Gitlab # aren't incremented properly when joining relations together this way. # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. - fragments = ActiveRecord::Base.connection.unprepared_statement do + fragments = ApplicationRecord.connection.unprepared_statement do relations.map do |rel| remove_order ? rel.reorder(nil).to_sql : rel.to_sql end.reject(&:blank?) @@ -54,6 +55,34 @@ module Gitlab private attr_reader :relations, :remove_duplicates, :remove_order + + def verify_select_values!(relations) + all_select_values = relations.map do |relation| + if relation.respond_to?(:select_values) + relation.select_values + else + relation.projections # Handle Arel based subqueries + end + end + + unless all_select_values.map(&:size).uniq.one? + relation_select_sizes = all_select_values.map.with_index do |select_values, index| + if select_values.empty? + "Relation ##{index}: *, all columns" + else + "Relation ##{index}: #{select_values.size} select values" + end + end + + exception_text = <<~EOF + Relations with uneven select values were passed. The UNION query could break when the underlying table changes (add or remove columns). + + #{relation_select_sizes.join("\n")} + EOF + + raise(exception_text) + end + end end end end diff --git a/lib/gitlab/tracking/docs/helper.rb b/lib/gitlab/tracking/docs/helper.rb index 81874aac9a5..4e03858b771 100644 --- a/lib/gitlab/tracking/docs/helper.rb +++ b/lib/gitlab/tracking/docs/helper.rb @@ -16,7 +16,7 @@ module Gitlab diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb deleted file mode 100644 index bfe674b945e..00000000000 --- a/lib/gitlab/usage/docs/helper.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Docs - # Helper with functions to be used by HAML templates - module Helper - def auto_generated_comment - <<-MARKDOWN.strip_heredoc - --- - stage: Growth - group: Product Intelligence - info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers - --- - - - MARKDOWN - end - - def render_name(name) - "### `#{name}`" - end - - def render_description(object) - return 'Missing description' unless object[:description].present? - - object[:description] - end - - def render_object_schema(object) - "[Object JSON schema](#{object.json_schema_path})" - end - - def render_yaml_link(yaml_path) - "[YAML definition](#{yaml_path})" - end - - def render_status(object) - "Status: #{format(:status, object[:status])}" - end - - def render_owner(object) - "Group: `#{object[:product_group]}`" - end - - def render_tiers(object) - "Tiers:#{format(:tier, object[:tier])}" - end - - def render_data_category(object) - "Data Category: `#{object[:data_category]}`" - end - - def format(key, value) - Gitlab::Usage::Docs::ValueFormatter.format(key, value) - end - end - end - end -end diff --git a/lib/gitlab/usage/docs/renderer.rb b/lib/gitlab/usage/docs/renderer.rb deleted file mode 100644 index 7a7c58005bb..00000000000 --- a/lib/gitlab/usage/docs/renderer.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Docs - class Renderer - include Gitlab::Usage::Docs::Helper - DICTIONARY_PATH = Rails.root.join('doc', 'development', 'usage_ping') - TEMPLATE_PATH = Rails.root.join('lib', 'gitlab', 'usage', 'docs', 'templates', 'default.md.haml') - - def initialize(metrics_definitions) - @layout = Haml::Engine.new(File.read(TEMPLATE_PATH)) - @metrics_definitions = metrics_definitions.sort - end - - def contents - # Render and remove an extra trailing new line - @contents ||= @layout.render(self, metrics_definitions: @metrics_definitions).sub!(/\n(?=\Z)/, '') - end - - def write - filename = DICTIONARY_PATH.join('dictionary.md').to_s - - FileUtils.mkdir_p(DICTIONARY_PATH) - File.write(filename, contents) - - filename - end - end - end - end -end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml deleted file mode 100644 index 83a3a5b6698..00000000000 --- a/lib/gitlab/usage/docs/templates/default.md.haml +++ /dev/null @@ -1,48 +0,0 @@ -= auto_generated_comment - -:plain - # Metrics Dictionary - - This file is autogenerated, please do not edit directly. - - To generate these files from the GitLab repository, run: - - ```shell - bundle exec rake gitlab:usage_data:generate_metrics_dictionary - ``` - - The Metrics Dictionary is based on the following metrics definition YAML files: - - - [`config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics) - - [`ee/config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/metrics) - - Each table includes a `milestone`, which corresponds to the GitLab version when the metric - was released. - - - - - - ## Metrics Definitions - -\ -- metrics_definitions.each do |name, object| - - = render_name(name) - \ - = render_description(object.attributes) - - if object.has_json_schema? - \ - = render_object_schema(object) - \ - = render_yaml_link(object.yaml_path) - \ - = render_owner(object.attributes) - - if object.attributes[:data_category].present? - \ - = render_data_category(object.attributes) - \ - = render_status(object.attributes) - \ - = render_tiers(object.attributes) - \ diff --git a/lib/gitlab/usage/docs/value_formatter.rb b/lib/gitlab/usage/docs/value_formatter.rb deleted file mode 100644 index 379e5df4d52..00000000000 --- a/lib/gitlab/usage/docs/value_formatter.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Docs - class ValueFormatter - def self.format(key, value) - return '' unless value.present? - - case key - when :key_path - "**`#{value}`**" - when :data_source - value.to_s.capitalize - when :product_group, :product_category, :status - "`#{value}`" - when :introduced_by_url - "[Introduced by](#{value})" - when :distribution, :tier - Array(value).map { |tier| " `#{tier}`" }.join(',') - else - value - end - end - end - end - end -end diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index f3469209f48..5b1ac189c13 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -3,40 +3,43 @@ module Gitlab module Usage class Metric - include ActiveModel::Model + attr_reader :definition - InvalidMetricError = Class.new(RuntimeError) - - attr_accessor :key_path, :value + def initialize(definition) + @definition = definition + end - validates :key_path, presence: true + class << self + def all + @all ||= Gitlab::Usage::MetricDefinition.with_instrumentation_class.map do |definition| + self.new(definition) + end + end + end - def definition - self.class.definitions[key_path] + def with_value + unflatten_key_path(intrumentation_object.value) end - def unflatten_key_path - unflatten(key_path.split('.'), value) + def with_instrumentation + unflatten_key_path(intrumentation_object.instrumentation) end - class << self - def definitions - @definitions ||= Gitlab::Usage::MetricDefinition.definitions - end + private - def dictionary - definitions.map { |key, definition| definition.to_dictionary } - end + def unflatten_key_path(value) + ::Gitlab::Usage::Metrics::KeyPathProcessor.process(definition.key_path, value) end - private + def instrumentation_class + "Gitlab::Usage::Metrics::Instrumentations::#{definition.instrumentation_class}" + end - def unflatten(keys, value) - loop do - value = { keys.pop.to_sym => value } - break if keys.blank? - end - value + def intrumentation_object + instrumentation_class.constantize.new( + time_frame: definition.time_frame, + options: definition.attributes[:options] + ) end end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index ccd2c69e2e7..db0cb4c6326 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -7,6 +7,8 @@ module Gitlab BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze + InvalidError = Class.new(RuntimeError) + attr_reader :path attr_reader :attributes @@ -48,11 +50,15 @@ module Gitlab Metric file: #{path} ERROR_MSG - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(error_message)) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(error_message)) end end end + def category_to_lowercase + attributes[:data_category]&.downcase! + end + alias_method :to_dictionary, :to_h class << self @@ -69,6 +75,10 @@ module Gitlab @all ||= definitions.map { |_key_path, definition| definition } end + def with_instrumentation_class + all.select { |definition| definition.attributes[:instrumentation_class].present? } + end + def schemer @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) end @@ -90,9 +100,9 @@ module Gitlab definition = YAML.safe_load(definition) definition.deep_symbolize_keys! - self.new(path, definition).tap(&:validate!) + self.new(path, definition).tap(&:validate!).tap(&:category_to_lowercase) rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(e.message)) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(e.message)) end def load_all_from_path!(definitions, glob_path) @@ -100,7 +110,7 @@ module Gitlab definition = load_from_file(path) if previous = definitions[definition.key] - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'")) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'")) end definitions[definition.key] = definition @@ -114,6 +124,10 @@ module Gitlab attributes[method] || super end + def respond_to_missing?(method, *args) + attributes[method].present? || super + end + def skip_validation? !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status]) end diff --git a/lib/gitlab/usage/metrics/aggregates.rb b/lib/gitlab/usage/metrics/aggregates.rb new file mode 100644 index 00000000000..a32c413dba8 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + UNION_OF_AGGREGATED_METRICS = 'OR' + INTERSECTION_OF_AGGREGATED_METRICS = 'AND' + ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze + AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml') + AggregatedMetricError = Class.new(StandardError) + UnknownAggregationOperator = Class.new(AggregatedMetricError) + UnknownAggregationSource = Class.new(AggregatedMetricError) + DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError) + + DATABASE_SOURCE = 'database' + REDIS_SOURCE = 'redis' + + SOURCES = { + DATABASE_SOURCE => Sources::PostgresHll, + REDIS_SOURCE => Sources::RedisHll + }.freeze + end + end + end +end diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 3ec06fba5d1..2545a505984 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -4,23 +4,6 @@ module Gitlab module Usage module Metrics module Aggregates - UNION_OF_AGGREGATED_METRICS = 'OR' - INTERSECTION_OF_AGGREGATED_METRICS = 'AND' - ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze - AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml') - AggregatedMetricError = Class.new(StandardError) - UnknownAggregationOperator = Class.new(AggregatedMetricError) - UnknownAggregationSource = Class.new(AggregatedMetricError) - DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError) - - DATABASE_SOURCE = 'database' - REDIS_SOURCE = 'redis' - - SOURCES = { - DATABASE_SOURCE => Sources::PostgresHll, - REDIS_SOURCE => Sources::RedisHll - }.freeze - class Aggregate include Gitlab::Usage::TimeFrame diff --git a/lib/gitlab/usage/metrics/aggregates/sources.rb b/lib/gitlab/usage/metrics/aggregates/sources.rb new file mode 100644 index 00000000000..f782a64f3b5 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/sources.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + module Sources + UnionNotAvailable = Class.new(AggregatedMetricError) + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb index 009b8e62543..1bdf3a7f9d8 100644 --- a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb +++ b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb @@ -5,8 +5,6 @@ module Gitlab module Metrics module Aggregates module Sources - UnionNotAvailable = Class.new(AggregatedMetricError) - class RedisHll extend Calculations::Intersection def self.calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at: nil) diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index 7b5bee3f8bd..a264f9484f3 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -15,6 +15,10 @@ module Gitlab @time_frame = time_frame @options = options end + + def instrumentation + value + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb index dd1f9948815..ee51180973c 100644 --- a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb @@ -5,8 +5,8 @@ module Gitlab module Metrics module Instrumentations class CollectedDataCategoriesMetric < GenericMetric - def value - ::ServicePing::PermitDataCategoriesService.new.execute + value do + ::ServicePing::PermitDataCategoriesService.new.execute.to_a end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 7b3a545185b..d7fc798ebe2 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -33,16 +33,17 @@ module Gitlab @metric_relation = block end - def operation(symbol, column: nil) + def operation(symbol, column: nil, &block) @metric_operation = symbol @column = column + @metric_operation_block = block if block_given? end def cache_start_and_finish_as(cache_key) @cache_key = cache_key end - attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :column, :cache_key + attr_reader :metric_operation, :metric_relation, :metric_start, :metric_finish, :metric_operation_block, :column, :cache_key end def value @@ -52,13 +53,18 @@ module Gitlab .call(relation, self.class.column, start: start, - finish: finish) + finish: finish, + &self.class.metric_operation_block) end def to_sql Gitlab::Usage::Metrics::Query.for(self.class.metric_operation, relation, self.class.column) end + def instrumentation + to_sql + end + def suggested_name Gitlab::Usage::Metrics::NameSuggestion.for( self.class.metric_operation, diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 1849773e33d..0f4b903b99c 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -12,27 +12,35 @@ module Gitlab # Gitlab::CurrentSettings.uuid # end # end + FALLBACK = -1 + class << self - attr_reader :metric_operation - @metric_operation = :alt + attr_reader :metric_value + + def fallback(custom_fallback = FALLBACK) + return @metric_fallback if defined?(@metric_fallback) + + @metric_fallback = custom_fallback + end def value(&block) @metric_value = block end + end - attr_reader :metric_value + def initialize(time_frame: 'none', options: {}) + @time_frame = time_frame + @options = options end def value - alt_usage_data do + alt_usage_data(fallback: self.class.fallback) do self.class.metric_value.call end end def suggested_name - Gitlab::Usage::Metrics::NameSuggestion.for( - self.class.metric_operation - ) + Gitlab::Usage::Metrics::NameSuggestion.for(:alt) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb index a36e612a1cb..bb27cca1bb9 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_hll_metric.rb @@ -12,11 +12,6 @@ module Gitlab # events: # - g_analytics_valuestream # end - class << self - attr_reader :metric_operation - @metric_operation = :redis - end - def initialize(time_frame:, options: {}) super @@ -36,9 +31,7 @@ module Gitlab end def suggested_name - Gitlab::Usage::Metrics::NameSuggestion.for( - self.class.metric_operation - ) + Gitlab::Usage::Metrics::NameSuggestion.for(:redis) end private diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb new file mode 100644 index 00000000000..a25bad2436b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + # Usage example + # + # In metric YAML definition: + # + # instrumentation_class: RedisMetric + # options: + # event: pushes + # counter_class: SourceCodeCounter + # + class RedisMetric < BaseMetric + def initialize(time_frame:, options: {}) + super + + raise ArgumentError, "'event' option is required" unless metric_event.present? + raise ArgumentError, "'counter class' option is required" unless counter_class.present? + end + + def metric_event + options[:event] + end + + def counter_class_name + options[:counter_class] + end + + def counter_class + "Gitlab::UsageDataCounters::#{counter_class_name}".constantize + end + + def value + redis_usage_data do + counter_class.read(metric_event) + end + end + + def suggested_name + Gitlab::Usage::Metrics::NameSuggestion.for(:redis) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb index a669b43f395..b47dc5689d4 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -10,6 +10,12 @@ module Gitlab uncached_data.deep_stringify_keys.dig(*key_path.split('.')) end + def add_metric(metric, time_frame: 'none') + metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize + + metric_class.new(time_frame: time_frame).suggested_name + end + private def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index aabc706901e..910c8397f20 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -72,8 +72,8 @@ module Gitlab def license_usage_data { recorded_at: recorded_at, - uuid: alt_usage_data { Gitlab::CurrentSettings.uuid }, - hostname: alt_usage_data { Gitlab.config.gitlab.host }, + uuid: add_metric('UuidMetric'), + hostname: add_metric('HostnameMetric'), version: alt_usage_data { Gitlab::VERSION }, installation_type: alt_usage_data { installation_type }, active_user_count: count(User.active), @@ -93,7 +93,7 @@ module Gitlab { counts: { assignee_lists: count(List.assignee), - boards: count(Board), + boards: add_metric('CountBoardsMetric', time_frame: 'all'), ci_builds: count(::Ci::Build), ci_internal_pipelines: count(::Ci::Pipeline.internal), ci_external_pipelines: count(::Ci::Pipeline.external), @@ -108,6 +108,7 @@ module Gitlab deployments: deployment_count(Deployment), successful_deployments: deployment_count(Deployment.success), failed_deployments: deployment_count(Deployment.failed), + feature_flags: count(Operations::FeatureFlag), # rubocop: enable UsageData/LargeTable: environments: count(::Environment), clusters: count(::Clusters::Cluster), @@ -138,7 +139,7 @@ module Gitlab in_review_folder: count(::Environment.in_review_folder), grafana_integrated_projects: count(GrafanaIntegration.enabled), groups: count(Group), - issues: count(Issue, start: minimum_id(Issue), finish: maximum_id(Issue)), + issues: add_metric('CountIssuesMetric', time_frame: 'all'), issues_created_from_gitlab_error_tracking_ui: count(SentryIssue), issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), @@ -255,9 +256,10 @@ module Gitlab { settings: { ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, + smtp_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::SmtpConfig.encrypted_secrets.active? }, operating_system: alt_usage_data(fallback: nil) { operating_system }, gitaly_apdex: alt_usage_data { gitaly_apdex }, - collected_data_categories: alt_usage_data(fallback: []) { Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric.new(time_frame: 'none').value } + collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none') } } end @@ -328,9 +330,9 @@ module Gitlab version: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_version } }, database: { - adapter: alt_usage_data { Gitlab::Database.adapter_name }, - version: alt_usage_data { Gitlab::Database.version }, - pg_system_id: alt_usage_data { Gitlab::Database.system_id } + adapter: alt_usage_data { Gitlab::Database.main.adapter_name }, + version: alt_usage_data { Gitlab::Database.main.version }, + pg_system_id: alt_usage_data { Gitlab::Database.main.system_id } }, mail: { smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] } @@ -644,16 +646,17 @@ module Gitlab # Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active` # rubocop: disable CodeReuse/ActiveRecord def usage_activity_by_stage_plan(time_period) + time_frame = time_period.present? ? '28d' : 'none' { - issues: distinct_count(::Issue.where(time_period), :author_id), + issues: add_metric('CountUsersCreatingIssuesMetric', time_frame: time_frame), notes: distinct_count(::Note.where(time_period), :author_id), projects: distinct_count(::Project.where(time_period), :creator_id), todos: distinct_count(::Todo.where(time_period), :author_id), service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period), service_desk_issues: count(::Issue.service_desk.where(time_period)), - projects_jira_active: distinct_count(::Project.with_active_jira_integrations.where(time_period), :creator_id), - projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_jira_integrations.with_jira_dvcs_cloud.where(time_period), :creator_id), - projects_jira_dvcs_server_active: distinct_count(::Project.with_active_jira_integrations.with_jira_dvcs_server.where(time_period), :creator_id) + projects_jira_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .where(time_period), :creator_id), + projects_jira_dvcs_cloud_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .with_jira_dvcs_cloud.where(time_period), :creator_id), + projects_jira_dvcs_server_active: distinct_count(::Project.with_active_integration(::Integrations::Jira) .with_jira_dvcs_server.where(time_period), :creator_id) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index ed9dad37f3e..796cbfdb3d6 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -15,7 +15,8 @@ module Gitlab MergeRequestCounter, DesignsCounter, KubernetesAgentCounter, - StaticSiteEditorCounter + StaticSiteEditorCounter, + DiffsCounter ].freeze UsageDataCounterError = Class.new(StandardError) diff --git a/lib/gitlab/usage_data_counters/diffs_counter.rb b/lib/gitlab/usage_data_counters/diffs_counter.rb new file mode 100644 index 00000000000..b06c7e26f54 --- /dev/null +++ b/lib/gitlab/usage_data_counters/diffs_counter.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class DiffsCounter < BaseCounter + KNOWN_EVENTS = %w[searches].freeze + PREFIX = 'diff' + end + end +end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 597df9936ea..96562a44391 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -117,7 +117,7 @@ module Gitlab private def track(values, event_name, context: '', time: Time.zone.now) - return unless usage_ping_enabled? + return unless ::ServicePing::ServicePingSettings.enabled? event = event_for(event_name) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? @@ -131,10 +131,6 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end - def usage_ping_enabled? - Gitlab::CurrentSettings.usage_ping_enabled? - end - # The array of valid context on which we allow tracking def valid_context_list Plan.all_plans @@ -193,6 +189,7 @@ module Gitlab def events_in_same_slot?(events) # if we check one event then redis_slot is only one to check + return false if events.empty? return true if events.size == 1 slot = events.first[:redis_slot] diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 5023161a9dd..7ad51bfe832 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -232,3 +232,8 @@ redis_slot: code_review category: code_review aggregation: weekly +- name: i_code_review_user_searches_diff + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: diff_searching_usage_data diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index fe1eb090fa4..3db0482d38e 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -170,7 +170,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_full_code_quality_report_total - name: i_testing_web_performance_widget_total category: testing redis_slot: testing diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index c1eabb352f7..7df351859fb 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -179,6 +179,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_severity + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_shrug category: quickactions redis_slot: quickactions diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb index 2406f771fd8..591e431c871 100644 --- a/lib/gitlab/usage_data_counters/redis_counter.rb +++ b/lib/gitlab/usage_data_counters/redis_counter.rb @@ -4,13 +4,13 @@ module Gitlab module UsageDataCounters module RedisCounter def increment(redis_counter_key) - return unless Gitlab::CurrentSettings.usage_ping_enabled + return unless ::ServicePing::ServicePingSettings.enabled? Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } end def increment_by(redis_counter_key, incr) - return unless Gitlab::CurrentSettings.usage_ping_enabled + return unless ::ServicePing::ServicePingSettings.enabled? Gitlab::Redis::SharedState.with { |redis| redis.incrby(redis_counter_key, incr) } end diff --git a/lib/gitlab/usage_data_metrics.rb b/lib/gitlab/usage_data_metrics.rb index dde5dde19e0..1ef201121d9 100644 --- a/lib/gitlab/usage_data_metrics.rb +++ b/lib/gitlab/usage_data_metrics.rb @@ -5,26 +5,7 @@ module Gitlab class << self # Build the Usage Ping JSON payload from metrics YAML definitions which have instrumentation class set def uncached_data - ::Gitlab::Usage::MetricDefinition.all.map do |definition| - instrumentation_class = definition.attributes[:instrumentation_class] - options = definition.attributes[:options] - - if instrumentation_class.present? - metric_value = "Gitlab::Usage::Metrics::Instrumentations::#{instrumentation_class}".constantize.new( - time_frame: definition.attributes[:time_frame], - options: options).value - - metric_payload(definition.key_path, metric_value) - else - {} - end - end.reduce({}, :deep_merge) - end - - private - - def metric_payload(key_path, value) - ::Gitlab::Usage::Metrics::KeyPathProcessor.process(key_path, value) + ::Gitlab::Usage::Metric.all.map(&:with_value).reduce({}, :deep_merge) end end end diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index 44d5baa42f6..1ff4588d091 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -5,6 +5,16 @@ module Gitlab SQL_METRIC_DEFAULT = -3 class << self + def uncached_data + super.with_indifferent_access.deep_merge(instrumentation_metrics_queries.with_indifferent_access) + end + + def add_metric(metric, time_frame: 'none') + metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize + + metric_class.new(time_frame: time_frame).instrumentation + end + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) SQL_METRIC_DEFAULT end @@ -37,6 +47,12 @@ module Gitlab projects_jira_cloud_active: 0 } end + + private + + def instrumentation_metrics_queries + ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge) + end end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 63e6cf03d1f..f64da2fbf13 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -5,6 +5,16 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 class UsageDataQueries < UsageData class << self + def uncached_data + super.with_indifferent_access.deep_merge(instrumentation_metrics_queries.with_indifferent_access) + end + + def add_metric(metric, time_frame: 'none') + metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize + + metric_class.new(time_frame: time_frame).instrumentation + end + def count(relation, column = nil, *args, **kwargs) Gitlab::Usage::Metrics::Query.for(:count, relation, column) end @@ -58,6 +68,12 @@ module Gitlab def epics_deepest_relationship_level { epics_deepest_relationship_level: 0 } end + + private + + def instrumentation_metrics_queries + ::Gitlab::Usage::Metric.all.map(&:with_instrumentation).reduce({}, :deep_merge) + end end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 0b1acaf7dd8..cb34ed69a9c 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -13,7 +13,7 @@ module Gitlab return unless path.is_a?(String) path = decode_path(path) - path_regex = /(\A(\.{1,2})\z|\A\.\.[\/\\]|[\/\\]\.\.\z|[\/\\]\.\.[\/\\]|\n)/ + path_regex = %r{(\A(\.{1,2})\z|\A\.\.[/\\]|[/\\]\.\.\z|[/\\]\.\.[/\\]|\n)} if path.match?(path_regex) raise PathTraversalAttackError, 'Invalid path' diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index faa524d171c..d46210f6efe 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -44,6 +44,12 @@ module Gitlab DISTRIBUTED_HLL_FALLBACK = -2 MAX_BUCKET_SIZE = 100 + def add_metric(metric, time_frame: 'none') + metric_class = "Gitlab::Usage::Metrics::Instrumentations::#{metric}".constantize + + metric_class.new(time_frame: time_frame).value + end + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch Gitlab::Database::BatchCount.batch_count(relation, column, batch_size: batch_size, start: start, finish: finish) diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index abfb7e2310e..64029d4d3fe 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -12,6 +12,7 @@ module Gitlab included do scope :public_only, -> { where(visibility_level: PUBLIC) } scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } + scope :private_only, -> { where(visibility_level: PRIVATE) } scope :non_public_only, -> { where.not(visibility_level: PUBLIC) } scope :public_to_user, -> (user = nil) do diff --git a/lib/gitlab/web_ide/config/entry/terminal.rb b/lib/gitlab/web_ide/config/entry/terminal.rb index ec07023461f..3da2c3b2ced 100644 --- a/lib/gitlab/web_ide/config/entry/terminal.rb +++ b/lib/gitlab/web_ide/config/entry/terminal.rb @@ -54,7 +54,6 @@ module Gitlab def to_hash { tag_list: tags || [], - yaml_variables: yaml_variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 job_variables: yaml_variables, options: { image: image_value, diff --git a/lib/gitlab/x509/tag.rb b/lib/gitlab/x509/tag.rb index ad85b200130..cf24e6f62bd 100644 --- a/lib/gitlab/x509/tag.rb +++ b/lib/gitlab/x509/tag.rb @@ -4,37 +4,16 @@ require 'digest' module Gitlab module X509 - class Tag + class Tag < Gitlab::SignedTag include Gitlab::Utils::StrongMemoize - def initialize(raw_tag) - @raw_tag = raw_tag - end - def signature - signature = X509::Signature.new(signature_text, signed_text, @raw_tag.tagger.email, Time.at(@raw_tag.tagger.date.seconds)) - - return if signature.verified_signature.nil? - - signature - end - - private - - def signature_text - @raw_tag.message.slice(@raw_tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1) - rescue StandardError - nil - end - - def signed_text - # signed text is reconstructed as long as there is no specific gitaly function - %{object #{@raw_tag.target_commit.id} -type commit -tag #{@raw_tag.name} -tagger #{@raw_tag.tagger.name} <#{@raw_tag.tagger.email}> #{@raw_tag.tagger.date.seconds} #{@raw_tag.tagger.timezone} + strong_memoize(:signature) do + super -#{@raw_tag.message.gsub(/-----BEGIN SIGNED MESSAGE-----(.*)-----END SIGNED MESSAGE-----/m, "")}} + signature = X509::Signature.new(signature_text, signed_text, @tag.tagger.email, Time.at(@tag.tagger.date.seconds)) + signature unless signature.verified_signature.nil? + end end end end diff --git a/lib/peek/views/active_record.rb b/lib/peek/views/active_record.rb index 8e1200338c2..a3fe206c86f 100644 --- a/lib/peek/views/active_record.rb +++ b/lib/peek/views/active_record.rb @@ -66,7 +66,8 @@ module Peek backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller), cached: data[:cached] ? 'Cached' : '', transaction: data[:connection].transaction_open? ? 'In a transaction' : '', - db_role: db_role(data) + db_role: db_role(data), + db_config_name: "Config name: #{::Gitlab::Database.db_config_name(data[:connection])}" } end @@ -76,7 +77,15 @@ module Peek role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(data[:connection]) || ::Gitlab::Database::LoadBalancing::ROLE_UNKNOWN - role.to_s.capitalize + "Role: #{role.to_s.capitalize}" + end + + def format_call_details(call) + if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] + super + else + super.except(:db_config_name) + end end end end diff --git a/lib/product_analytics/tracker.rb b/lib/product_analytics/tracker.rb index d4a88b879f0..e5550c2e6ef 100644 --- a/lib/product_analytics/tracker.rb +++ b/lib/product_analytics/tracker.rb @@ -6,6 +6,6 @@ module ProductAnalytics URL = Gitlab.config.gitlab.url + '/-/sp.js' # The collector URL minus protocol and /i - COLLECTOR_URL = Gitlab.config.gitlab.url.sub(/\Ahttps?\:\/\//, '') + '/-/collector' + COLLECTOR_URL = Gitlab.config.gitlab.url.sub(%r{\Ahttps?\://}, '') + '/-/collector' end end diff --git a/lib/sidebars/concerns/has_partial.rb b/lib/sidebars/concerns/has_partial.rb new file mode 100644 index 00000000000..e89bc66bc40 --- /dev/null +++ b/lib/sidebars/concerns/has_partial.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# This module has the necessary methods to render +# menus using a custom partial +module Sidebars + module Concerns + module HasPartial + def menu_partial + nil + end + + def menu_partial_options + {} + end + + def menu_with_partial? + menu_partial.present? + end + end + end +end diff --git a/lib/sidebars/concerns/has_pill.rb b/lib/sidebars/concerns/has_pill.rb index 5082ed477e6..4bbf69bf16b 100644 --- a/lib/sidebars/concerns/has_pill.rb +++ b/lib/sidebars/concerns/has_pill.rb @@ -5,6 +5,8 @@ module Sidebars module Concerns module HasPill + include ActionView::Helpers::NumberHelper + def has_pill? false end @@ -18,6 +20,17 @@ module Sidebars def pill_html_options {} end + + def format_cached_count(count_service, count) + if count > count_service::CACHED_COUNT_THRESHOLD + number_to_human( + count, + units: { thousand: 'k', million: 'm' }, precision: 1, significant: false, format: '%n%u' + ) + else + number_with_delimiter(count) + end + end end end end diff --git a/lib/sidebars/groups/menus/ci_cd_menu.rb b/lib/sidebars/groups/menus/ci_cd_menu.rb new file mode 100644 index 00000000000..e870bbf5ebc --- /dev/null +++ b/lib/sidebars/groups/menus/ci_cd_menu.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module Menus + class CiCdMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + add_item(runners_menu_item) + + true + end + + override :link + def link + renderable_items.first.link + end + + override :title + def title + _('CI/CD') + end + + override :sprite_icon + def sprite_icon + 'rocket' + end + + private + + def runners_menu_item + return ::Sidebars::NilMenuItem.new(item_id: :runners) unless show_runners? + + ::Sidebars::MenuItem.new( + title: _('Runners'), + link: group_runners_path(context.group), + active_routes: { path: 'groups/runners#index' }, + item_id: :runners + ) + end + + # TODO Proper policies, such as `read_group_runners`, should be implemented per + # See https://gitlab.com/gitlab-org/gitlab/-/issues/334802 + def show_runners? + can?(context.current_user, :admin_group, context.group) && + Feature.enabled?(:runner_list_group_view_vue_ui, context.group, default_enabled: :yaml) + end + end + end + end +end diff --git a/lib/sidebars/groups/menus/group_information_menu.rb b/lib/sidebars/groups/menus/group_information_menu.rb new file mode 100644 index 00000000000..b28cb927ad2 --- /dev/null +++ b/lib/sidebars/groups/menus/group_information_menu.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module Menus + class GroupInformationMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + add_item(activity_menu_item) + add_item(labels_menu_item) + add_item(members_menu_item) + + true + end + + override :link + def link + renderable_items.first.link + end + + override :title + def title + context.group.subgroup? ? _('Subgroup information') : _('Group information') + end + + override :sprite_icon + def sprite_icon + 'group' + end + + override :active_routes + def active_routes + { path: 'groups#subgroups' } + end + + private + + def activity_menu_item + unless can?(context.current_user, :read_group_activity, context.group) + return ::Sidebars::NilMenuItem.new(item_id: :activity) + end + + ::Sidebars::MenuItem.new( + title: _('Activity'), + link: activity_group_path(context.group), + active_routes: { path: 'groups#activity' }, + item_id: :activity + ) + end + + def labels_menu_item + unless can?(context.current_user, :read_group_labels, context.group) + return ::Sidebars::NilMenuItem.new(item_id: :labels) + end + + ::Sidebars::MenuItem.new( + title: _('Labels'), + link: group_labels_path(context.group), + active_routes: { controller: :labels }, + item_id: :labels + ) + end + + def members_menu_item + unless can?(context.current_user, :read_group_member, context.group) + return ::Sidebars::NilMenuItem.new(item_id: :members) + end + + ::Sidebars::MenuItem.new( + title: _('Members'), + link: group_group_members_path(context.group), + active_routes: { path: 'group_members#index' }, + item_id: :members + ) + end + end + end + end +end diff --git a/lib/sidebars/groups/menus/issues_menu.rb b/lib/sidebars/groups/menus/issues_menu.rb new file mode 100644 index 00000000000..95641c09076 --- /dev/null +++ b/lib/sidebars/groups/menus/issues_menu.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module Menus + class IssuesMenu < ::Sidebars::Menu + include Gitlab::Utils::StrongMemoize + + override :configure_menu_items + def configure_menu_items + return unless can?(context.current_user, :read_group_issues, context.group) + + add_item(list_menu_item) + add_item(boards_menu_item) + add_item(milestones_menu_item) + + true + end + + override :link + def link + issues_group_path(context.group) + end + + override :title + def title + _('Issues') + end + + override :sprite_icon + def sprite_icon + 'issues' + end + + override :has_pill? + def has_pill? + true + end + + override :pill_count + def pill_count + strong_memoize(:pill_count) do + count_service = ::Groups::OpenIssuesCountService + count = count_service.new(context.group, context.current_user).count + + format_cached_count(count_service, count) + end + end + + override :pill_html_options + def pill_html_options + { + class: 'issue_counter' + } + end + + private + + def list_menu_item + ::Sidebars::MenuItem.new( + title: _('List'), + link: issues_group_path(context.group), + active_routes: { path: 'groups#issues' }, + container_html_options: { aria: { label: _('Issues') } }, + item_id: :issue_list + ) + end + + def boards_menu_item + unless can?(context.current_user, :read_group_boards, context.group) + return ::Sidebars::NilMenuItem.new(item_id: :boards) + end + + title = context.group.multiple_issue_boards_available? ? s_('IssueBoards|Boards') : s_('IssueBoards|Board') + + ::Sidebars::MenuItem.new( + title: title, + link: group_boards_path(context.group), + active_routes: { path: %w[boards#index boards#show] }, + item_id: :boards + ) + end + + def milestones_menu_item + unless can?(context.current_user, :read_group_milestones, context.group) + return ::Sidebars::NilMenuItem.new(item_id: :milestones) + end + + ::Sidebars::MenuItem.new( + title: _('Milestones'), + link: group_milestones_path(context.group), + active_routes: { path: 'milestones#index' }, + item_id: :milestones + ) + end + end + end + end +end + +Sidebars::Groups::Menus::IssuesMenu.prepend_mod_with('Sidebars::Groups::Menus::IssuesMenu') diff --git a/lib/sidebars/groups/menus/kubernetes_menu.rb b/lib/sidebars/groups/menus/kubernetes_menu.rb new file mode 100644 index 00000000000..4ea294a4837 --- /dev/null +++ b/lib/sidebars/groups/menus/kubernetes_menu.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module Menus + class KubernetesMenu < ::Sidebars::Menu + override :link + def link + group_clusters_path(context.group) + end + + override :title + def title + _('Kubernetes') + end + + override :sprite_icon + def sprite_icon + 'cloud-gear' + end + + override :render? + def render? + can?(context.current_user, :read_cluster, context.group) + end + + override :extra_container_html_options + def extra_container_html_options + { + class: 'shortcuts-kubernetes' + } + end + + override :active_routes + def active_routes + { controller: :clusters } + end + end + end + end +end diff --git a/lib/sidebars/groups/menus/merge_requests_menu.rb b/lib/sidebars/groups/menus/merge_requests_menu.rb new file mode 100644 index 00000000000..7faf50305c6 --- /dev/null +++ b/lib/sidebars/groups/menus/merge_requests_menu.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module Menus + class MergeRequestsMenu < ::Sidebars::Menu + include Gitlab::Utils::StrongMemoize + + override :link + def link + merge_requests_group_path(context.group) + end + + override :title + def title + _('Merge requests') + end + + override :sprite_icon + def sprite_icon + 'git-merge' + end + + override :render? + def render? + can?(context.current_user, :read_group_merge_requests, context.group) + end + + override :has_pill? + def has_pill? + true + end + + override :pill_count + def pill_count + strong_memoize(:pill_count) do + count_service = ::Groups::MergeRequestsCountService + count = count_service.new(context.group, context.current_user).count + + format_cached_count(count_service, count) + end + end + + override :pill_html_options + def pill_html_options + { + class: 'merge_counter js-merge-counter' + } + end + + override :active_routes + def active_routes + { path: 'groups#merge_requests' } + end + end + end + end +end diff --git a/lib/sidebars/groups/menus/packages_registries_menu.rb b/lib/sidebars/groups/menus/packages_registries_menu.rb new file mode 100644 index 00000000000..e46e2820c04 --- /dev/null +++ b/lib/sidebars/groups/menus/packages_registries_menu.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module Menus + class PackagesRegistriesMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + add_item(packages_registry_menu_item) + add_item(container_registry_menu_item) + add_item(dependency_proxy_menu_item) + + true + end + + override :link + def link + renderable_items.first.link + end + + override :title + def title + _('Packages & Registries') + end + + override :sprite_icon + def sprite_icon + 'package' + end + + private + + def packages_registry_menu_item + unless context.group.packages_feature_enabled? + return ::Sidebars::NilMenuItem.new(item_id: :packages_registry) + end + + ::Sidebars::MenuItem.new( + title: _('Package Registry'), + link: group_packages_path(context.group), + active_routes: { controller: 'groups/packages' }, + item_id: :packages_registry + ) + end + + def container_registry_menu_item + if !::Gitlab.config.registry.enabled || !can?(context.current_user, :read_container_image, context.group) + return ::Sidebars::NilMenuItem.new(item_id: :container_registry) + end + + ::Sidebars::MenuItem.new( + title: _('Container Registry'), + link: group_container_registries_path(context.group), + active_routes: { controller: 'groups/registry/repositories' }, + item_id: :container_registry + ) + end + + def dependency_proxy_menu_item + unless context.group.dependency_proxy_feature_available? + return ::Sidebars::NilMenuItem.new(item_id: :dependency_proxy) + end + + ::Sidebars::MenuItem.new( + title: _('Dependency Proxy'), + link: group_dependency_proxy_path(context.group), + active_routes: { controller: 'groups/dependency_proxies' }, + item_id: :dependency_proxy + ) + end + end + end + end +end diff --git a/lib/sidebars/groups/menus/settings_menu.rb b/lib/sidebars/groups/menus/settings_menu.rb new file mode 100644 index 00000000000..8bc6077d302 --- /dev/null +++ b/lib/sidebars/groups/menus/settings_menu.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +module Sidebars + module Groups + module Menus + class SettingsMenu < ::Sidebars::Menu + override :configure_menu_items + def configure_menu_items + return false unless can?(context.current_user, :admin_group, context.group) + + add_item(general_menu_item) + add_item(integrations_menu_item) + add_item(group_projects_menu_item) + add_item(repository_menu_item) + add_item(ci_cd_menu_item) + add_item(applications_menu_item) + add_item(packages_and_registries_menu_item) + + true + end + + override :link + def link + edit_group_path(context.group) + end + + override :title + def title + _('Settings') + end + + override :sprite_icon + def sprite_icon + 'settings' + end + + override :extra_nav_link_html_options + def extra_nav_link_html_options + { + class: 'shortcuts-settings' + } + end + + private + + def general_menu_item + ::Sidebars::MenuItem.new( + title: _('General'), + link: edit_group_path(context.group), + active_routes: { path: 'groups#edit' }, + item_id: :general + ) + end + + def integrations_menu_item + ::Sidebars::MenuItem.new( + title: _('Integrations'), + link: group_settings_integrations_path(context.group), + active_routes: { controller: :integrations }, + item_id: :integrations + ) + end + + def group_projects_menu_item + ::Sidebars::MenuItem.new( + title: _('Projects'), + link: projects_group_path(context.group), + active_routes: { path: 'groups#projects' }, + item_id: :group_projects + ) + end + + def repository_menu_item + ::Sidebars::MenuItem.new( + title: _('Repository'), + link: group_settings_repository_path(context.group), + active_routes: { controller: :repository }, + item_id: :repository + ) + end + + def ci_cd_menu_item + ::Sidebars::MenuItem.new( + title: _('CI/CD'), + link: group_settings_ci_cd_path(context.group), + active_routes: { path: %w[ci_cd#show groups/runners#show groups/runners#edit] }, + item_id: :ci_cd + ) + end + + def applications_menu_item + ::Sidebars::MenuItem.new( + title: _('Applications'), + link: group_settings_applications_path(context.group), + active_routes: { controller: :applications }, + item_id: :applications + ) + end + + def packages_and_registries_menu_item + unless context.group.packages_feature_enabled? + return ::Sidebars::NilMenuItem.new(item_id: :packages_and_registries) + end + + ::Sidebars::MenuItem.new( + title: _('Packages & Registries'), + link: group_settings_packages_and_registries_path(context.group), + active_routes: { controller: :packages_and_registries }, + item_id: :packages_and_registries + ) + end + end + end + end +end + +Sidebars::Groups::Menus::SettingsMenu.prepend_mod_with('Sidebars::Groups::Menus::SettingsMenu') diff --git a/lib/sidebars/groups/panel.rb b/lib/sidebars/groups/panel.rb index fe669bf0b29..73b943c5655 100644 --- a/lib/sidebars/groups/panel.rb +++ b/lib/sidebars/groups/panel.rb @@ -6,6 +6,14 @@ module Sidebars override :configure_menus def configure_menus set_scope_menu(Sidebars::Groups::Menus::ScopeMenu.new(context)) + + add_menu(Sidebars::Groups::Menus::GroupInformationMenu.new(context)) + add_menu(Sidebars::Groups::Menus::IssuesMenu.new(context)) + add_menu(Sidebars::Groups::Menus::MergeRequestsMenu.new(context)) + add_menu(Sidebars::Groups::Menus::CiCdMenu.new(context)) + add_menu(Sidebars::Groups::Menus::KubernetesMenu.new(context)) + add_menu(Sidebars::Groups::Menus::PackagesRegistriesMenu.new(context)) + add_menu(Sidebars::Groups::Menus::SettingsMenu.new(context)) end override :render_raw_menus_partial @@ -20,3 +28,5 @@ module Sidebars end end end + +Sidebars::Groups::Panel.prepend_mod_with('Sidebars::Groups::Panel') diff --git a/lib/sidebars/menu.rb b/lib/sidebars/menu.rb index dcdc130b0d7..3b8872fd572 100644 --- a/lib/sidebars/menu.rb +++ b/lib/sidebars/menu.rb @@ -12,6 +12,7 @@ module Sidebars include ::Sidebars::Concerns::Renderable include ::Sidebars::Concerns::ContainerWithHtmlOptions include ::Sidebars::Concerns::HasActiveRoutes + include ::Sidebars::Concerns::HasPartial attr_reader :context delegate :current_user, :container, to: :@context @@ -29,7 +30,7 @@ module Sidebars override :render? def render? - has_renderable_items? + has_renderable_items? || menu_with_partial? end # Menus might have or not a link diff --git a/lib/sidebars/projects/menus/packages_registries_menu.rb b/lib/sidebars/projects/menus/packages_registries_menu.rb index 27e318d73c5..d49bb680853 100644 --- a/lib/sidebars/projects/menus/packages_registries_menu.rb +++ b/lib/sidebars/projects/menus/packages_registries_menu.rb @@ -31,7 +31,7 @@ module Sidebars private def packages_registry_menu_item - if !::Gitlab.config.packages.enabled || !can?(context.current_user, :read_package, context.project) + if packages_registry_disabled? return ::Sidebars::NilMenuItem.new(item_id: :packages_registry) end @@ -58,7 +58,7 @@ module Sidebars end def infrastructure_registry_menu_item - if Feature.disabled?(:infrastructure_registry_page, context.current_user, default_enabled: :yaml) + if packages_registry_disabled? return ::Sidebars::NilMenuItem.new(item_id: :infrastructure_registry) end @@ -69,6 +69,10 @@ module Sidebars item_id: :infrastructure_registry ) end + + def packages_registry_disabled? + !::Gitlab.config.packages.enabled || !can?(context.current_user, :read_package, context.project) + end end end end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index ac47c5be1e8..96e3a015115 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -44,7 +44,7 @@ gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" gitlab_pages_enabled=false gitlab_pages_dir=$(cd $app_root/../gitlab-pages 2> /dev/null && pwd) gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" -gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" +gitlab_pages_options="-config $app_root/gitlab-pages/gitlab-pages.conf" gitlab_pages_log="$app_root/log/gitlab-pages.log" shell_path="/bin/bash" gitaly_enabled=true diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index 53bebe55fa3..0233c26cecc 100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -68,7 +68,7 @@ gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" # The -pages-domain must be specified the same as in `gitlab.yml > pages > host`. # Set `gitlab_pages_enabled=true` if you want to enable the Pages feature. gitlab_pages_enabled=false -gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" +gitlab_pages_options="-config $app_root/gitlab-pages/gitlab-pages.conf" gitlab_pages_log="$app_root/log/gitlab-pages.log" # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. diff --git a/lib/support/nginx/gitlab-pages-ssl b/lib/support/nginx/gitlab-pages-ssl index 62ed482e2bf..900d91e0575 100644 --- a/lib/support/nginx/gitlab-pages-ssl +++ b/lib/support/nginx/gitlab-pages-ssl @@ -31,16 +31,16 @@ server { ## Strong SSL Security ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ - ssl on; ssl_certificate /etc/nginx/ssl/gitlab-pages.crt; ssl_certificate_key /etc/nginx/ssl/gitlab-pages.key; - # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs - ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_prefer_server_ciphers on; + ssl_session_timeout 1d; ssl_session_cache shared:SSL:10m; - ssl_session_timeout 5m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; ## See app/controllers/application_controller.rb for headers set @@ -58,6 +58,9 @@ server { ## # ssl_dhparam /etc/ssl/certs/dhparam.pem; + ## [Optional] Enable HTTP Strict Transport Security + # add_header Strict-Transport-Security "max-age=63072000; includeSubDomains"; + ## Individual nginx logs for GitLab pages access_log /var/log/nginx/gitlab_pages_access.log; error_log /var/log/nginx/gitlab_pages_error.log; diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 576c13d8d10..435b9055929 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -83,23 +83,23 @@ server { ## HTTPS host server { - listen 0.0.0.0:443 ssl; - listen [::]:443 ipv6only=on ssl default_server; + listen 0.0.0.0:443 ssl http2; + listen [::]:443 ipv6only=on ssl http2 default_server; server_name YOUR_SERVER_FQDN; ## Replace this with something like gitlab.example.com server_tokens off; ## Don't show the nginx version number, a security best practice ## Strong SSL Security ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ - ssl on; ssl_certificate /etc/nginx/ssl/gitlab.crt; ssl_certificate_key /etc/nginx/ssl/gitlab.key; - # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs - ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_prefer_server_ciphers on; + ssl_session_timeout 1d; ssl_session_cache shared:SSL:10m; - ssl_session_timeout 5m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; ## See app/controllers/application_controller.rb for headers set @@ -120,7 +120,7 @@ server { # ssl_dhparam /etc/ssl/certs/dhparam.pem; ## [Optional] Enable HTTP Strict Transport Security - # add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; + # add_header Strict-Transport-Security "max-age=63072000; includeSubDomains"; ## Real IP Module Config ## http://nginx.org/en/docs/http/ngx_http_realip_module.html diff --git a/lib/support/nginx/registry-ssl b/lib/support/nginx/registry-ssl index df126919866..be16037629b 100644 --- a/lib/support/nginx/registry-ssl +++ b/lib/support/nginx/registry-ssl @@ -27,15 +27,24 @@ server { ## Strong SSL Security ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ - ssl on; ssl_certificate /etc/gitlab/ssl/registry.gitlab.example.com.crt ssl_certificate_key /etc/gitlab/ssl/registry.gitlab.example.com.key - ssl_ciphers 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4'; - ssl_protocols TLSv1 TLSv1.1 TLSv1.2; - ssl_prefer_server_ciphers on; - ssl_session_cache builtin:1000 shared:SSL:10m; - ssl_session_timeout 5m; + ssl_session_timeout 1d; + ssl_session_cache shared:SSL:10m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + ## [Optional] Generate a stronger DHE parameter: + ## sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096 + ## + # ssl_dhparam /etc/ssl/certs/dhparam.pem; + + ## [Optional] Enable HTTP Strict Transport Security + # add_header Strict-Transport-Security "max-age=63072000; includeSubDomains"; access_log /var/log/gitlab/nginx/gitlab_registry_access.log gitlab_access; error_log /var/log/gitlab/nginx/gitlab_registry_error.log; diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index ed74dd472ff..cc10d73f76a 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -91,6 +91,8 @@ namespace :gitlab do backup.cleanup end + backup.remove_tmp + puts "Warning: Your gitlab.rb and gitlab-secrets.json files contain sensitive data \n" \ "and are not included in this backup. You will need to restore these files manually.".color(:red) puts "Restore task is done." @@ -297,7 +299,7 @@ namespace :gitlab do end def repository_backup_strategy - if Feature.enabled?(:gitaly_backup) + if Feature.enabled?(:gitaly_backup, default_enabled: :yaml) max_concurrency = ENV['GITLAB_BACKUP_MAX_CONCURRENCY'].presence max_storage_concurrency = ENV['GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY'].presence Backup::GitalyBackup.new(progress, parallel: max_concurrency, parallel_storage: max_storage_concurrency) diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 2b508b341dd..51f15f5a56a 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -161,7 +161,7 @@ namespace :gitlab do exit end - indexes = Gitlab::Database::Reindexing.candidate_indexes + indexes = Gitlab::Database::PostgresIndex.reindexing_support if identifier = args[:index_name] raise ArgumentError, "Index name is not fully qualified with a schema: #{identifier}" unless identifier =~ /^\w+\.\w+$/ @@ -173,6 +173,12 @@ namespace :gitlab do ActiveRecord::Base.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) + # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any) + Gitlab::Database::Reindexing.cleanup_leftovers! + + # Hack: Before we do actual reindexing work, create async indexes + Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) + Gitlab::Database::Reindexing.perform(indexes) rescue StandardError => e Gitlab::AppLogger.error(e) @@ -217,7 +223,7 @@ namespace :gitlab do instrumentation = Gitlab::Database::Migrations::Instrumentation.new pending_migrations.each do |migration| - instrumentation.observe(migration.version) do + instrumentation.observe(version: migration.version, name: migration.name) do ActiveRecord::Migrator.new(:up, ctx.migrations, ctx.schema_migration, migration.version).run end end diff --git a/lib/tasks/gitlab/docs/redirect.rake b/lib/tasks/gitlab/docs/redirect.rake index 990ff723eeb..123a4775605 100644 --- a/lib/tasks/gitlab/docs/redirect.rake +++ b/lib/tasks/gitlab/docs/redirect.rake @@ -57,68 +57,5 @@ namespace :gitlab do post.puts "" end end - - desc 'GitLab | Docs | Clean up old redirects' - task :clean_redirects do - # - # Calculate new path from the redirect URL. - # - # If the redirect is not a full URL: - # 1. Create a new Pathname of the file - # 2. Use dirname to get all but the last component of the path - # 3. Join with the redirect_to entry - # 4. Substitute: - # - '.md' => '.html' - # - 'doc/' => '/ee/' - # - # If the redirect URL is a full URL pointing to the Docs site - # (cross-linking among the 4 products), remove the FQDN prefix: - # - # From : https://docs.gitlab.com/ee/install/requirements.html - # To : /ee/install/requirements.html - # - def new_path(redirect, filename) - if !redirect.start_with?('http') - Pathname.new(filename).dirname.join(redirect).to_s.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/') - elsif redirect.start_with?('https://docs.gitlab.com') - redirect.gsub('https://docs.gitlab.com', '') - else - redirect - end - end - - today = Time.now.utc.to_date - - # - # Find the files to be deleted. - # Exclude 'doc/development/documentation/index.md' because it - # contains an example of the YAML front matter. - # - files_to_be_deleted = `grep -Ir 'remove_date:' doc | grep -v doc/development/documentation/index.md | cut -d ":" -f 1`.split("\n") - - # - # Iterate over the files to be deleted and print the needed - # YAML entries for the Docs site redirects. - # - files_to_be_deleted.each do |filename| - frontmatter = YAML.safe_load(File.read(filename)) - remove_date = Date.parse(frontmatter['remove_date']) - old_path = filename.gsub(%r(\.md), '.html').gsub(%r(doc/), '/ee/') - - # - # Check if the removal date is before today, and delete the file and - # print the content to be pasted in - # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/master/content/_data/redirects.yaml. - # The remove_date of redirects.yaml should be nine months in the future. - # To not be confused with the remove_date of the Markdown page. - # - next unless remove_date < today - - File.delete(filename) if File.exist?(filename) - puts " - from: #{old_path}" - puts " to: #{new_path(frontmatter['redirect_to'], filename)}" - puts " remove_date: #{remove_date >> 9}" - end - end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index df75b3cf716..6675439e430 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -2,6 +2,42 @@ namespace :gitlab do namespace :gitaly do + desc 'Installs gitaly for running tests within gitlab-development-kit' + task :test_install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| + inside_gdk = Rails.env.test? && File.exist?(Rails.root.join('../GDK_ROOT')) + + if ENV['FORCE_GITALY_INSTALL'] || !inside_gdk + Rake::Task["gitlab:gitaly:install"].invoke(*args) + + next + end + + gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly')) + + # Our test setup expects a git repo, so clone rather than copy + version = Gitlab::GitalyClient.expected_server_version + checkout_or_clone_version(version: version, repo: gdk_gitaly_dir, target_dir: args.dir, clone_opts: %w[--depth 1]) + + # We assume the GDK gitaly already compiled binaries + build_dir = File.join(gdk_gitaly_dir, '_build') + FileUtils.cp_r(build_dir, args.dir) + + # We assume the GDK gitaly already ran bundle install + bundle_dir = File.join(gdk_gitaly_dir, 'ruby', '.bundle') + FileUtils.cp_r(bundle_dir, File.join(args.dir, 'ruby')) + + # For completeness we copy this for gitaly's make target + ruby_bundle_file = File.join(gdk_gitaly_dir, '.ruby-bundle') + FileUtils.cp_r(ruby_bundle_file, args.dir) + + gitaly_binary = File.join(build_dir, 'bin', 'gitaly') + warn_gitaly_out_of_date!(gitaly_binary, version) + rescue Errno::ENOENT => e + puts "Could not copy files, did you run `gdk update`? Error: #{e.message}" + + raise + end + desc 'GitLab | Gitaly | Install or upgrade gitaly' task :install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab @@ -41,5 +77,24 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]") _, status = Gitlab::Popen.popen(%w[which gmake]) status == 0 ? 'gmake' : 'make' end + + def warn_gitaly_out_of_date!(gitaly_binary, expected_version) + binary_version, exit_status = Gitlab::Popen.popen(%W[#{gitaly_binary} -version]) + + raise "Failed to run `#{gitaly_binary} -version`" unless exit_status == 0 + + binary_version = binary_version.strip + + # See help for `git describe` for format + git_describe_sha = /g([a-f0-9]{5,40})\z/ + match = binary_version.match(git_describe_sha) + + # Just skip if the version does not have a sha component + return unless match + + return if expected_version.start_with?(match[1]) + + puts "WARNING: #{binary_version.strip} does not exactly match repository version #{expected_version}" + end end end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake index b405cbd3f68..52c5c680292 100644 --- a/lib/tasks/gitlab/graphql.rake +++ b/lib/tasks/gitlab/graphql.rake @@ -56,9 +56,9 @@ namespace :gitlab do color = case complexity when 0..GitlabSchema::DEFAULT_MAX_COMPLEXITY :green - when GitlabSchema::DEFAULT_MAX_COMPLEXITY..GitlabSchema::AUTHENTICATED_COMPLEXITY + when GitlabSchema::DEFAULT_MAX_COMPLEXITY..GitlabSchema::AUTHENTICATED_MAX_COMPLEXITY :yellow - when GitlabSchema::AUTHENTICATED_COMPLEXITY..GitlabSchema::ADMIN_COMPLEXITY + when GitlabSchema::AUTHENTICATED_MAX_COMPLEXITY..GitlabSchema::ADMIN_MAX_COMPLEXITY :orange else :red diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 2826002bdc2..68395d10d24 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -68,8 +68,8 @@ namespace :gitlab do puts "Version:\t#{Gitlab::VERSION}" puts "Revision:\t#{Gitlab.revision}" puts "Directory:\t#{Rails.root}" - puts "DB Adapter:\t#{Gitlab::Database.human_adapter_name}" - puts "DB Version:\t#{Gitlab::Database.version}" + puts "DB Adapter:\t#{Gitlab::Database.main.human_adapter_name}" + puts "DB Version:\t#{Gitlab::Database.main.version}" puts "URL:\t\t#{Gitlab.config.gitlab.url}" puts "HTTP Clone URL:\t#{http_clone_url}" puts "SSH Clone URL:\t#{ssh_clone_url}" diff --git a/lib/tasks/gitlab/product_intelligence.rake b/lib/tasks/gitlab/product_intelligence.rake new file mode 100644 index 00000000000..329cd9c8c2a --- /dev/null +++ b/lib/tasks/gitlab/product_intelligence.rake @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :product_intelligence do + # @example + # bundle exec rake gitlab:product_intelligence:activate_metrics MILESTONE=14.0 + + desc 'GitLab | Product Intelligence | Update milestone metrics status to data_available' + task activate_metrics: :environment do + milestone = ENV['MILESTONE'] + raise "Please supply the MILESTONE env var".color(:red) unless milestone.present? + + Gitlab::Usage::MetricDefinition.definitions.values.each do |metric| + next if metric.attributes[:milestone] != milestone || metric.attributes[:status] != 'implemented' + + metric.attributes[:status] = 'data_available' + path = metric.path + File.open(path, "w") { |file| file << metric.to_h.deep_stringify_keys.to_yaml } + end + + puts "Task completed successfully" + end + end +end diff --git a/lib/tasks/gitlab/smtp.rake b/lib/tasks/gitlab/smtp.rake new file mode 100644 index 00000000000..23ad7577e3c --- /dev/null +++ b/lib/tasks/gitlab/smtp.rake @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +namespace :gitlab do + namespace :smtp do + namespace :secret do + desc 'GitLab | SMTP | Secret | Write SMTP secrets' + task write: [:environment] do + content = $stdin.tty? ? $stdin.gets : $stdin.read + Gitlab::EncryptedSmtpCommand.write(content) + end + + desc 'GitLab | SMTP | Secret | Edit SMTP secrets' + task edit: [:environment] do + Gitlab::EncryptedSmtpCommand.edit + end + + desc 'GitLab | SMTP | Secret | Show SMTP secrets' + task show: [:environment] do + Gitlab::EncryptedSmtpCommand.show + end + end + end +end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 6fa39a26488..fb9f9b9fe67 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -170,7 +170,7 @@ namespace :gitlab do inverval = (ENV['MAX_DATABASE_CONNECTION_CHECK_INTERVAL'] || 10).to_f attempts.to_i.times do - unless Gitlab::Database.exists? + unless Gitlab::Database.main.exists? puts "Waiting until database is ready before continuing...".color(:yellow) sleep inverval end diff --git a/lib/tasks/gitlab/usage_data.rake b/lib/tasks/gitlab/usage_data.rake index 166f08ef16a..ddd3424acda 100644 --- a/lib/tasks/gitlab/usage_data.rake +++ b/lib/tasks/gitlab/usage_data.rake @@ -24,12 +24,6 @@ namespace :gitlab do puts Gitlab::Json.pretty_generate(result.attributes) end - desc 'GitLab | UsageData | Generate metrics dictionary' - task generate_metrics_dictionary: :environment do - items = Gitlab::Usage::MetricDefinition.definitions - Gitlab::Usage::Docs::Renderer.new(items).write - end - desc 'GitLab | UsageDataMetrics | Generate usage ping from metrics definition YAML files in JSON' task generate_from_yaml: :environment do puts Gitlab::Json.pretty_generate(Gitlab::UsageDataMetrics.uncached_data) -- cgit v1.2.1