summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-05-20 14:34:42 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-05-20 14:34:42 +0000
commit9f46488805e86b1bc341ea1620b866016c2ce5ed (patch)
treef9748c7e287041e37d6da49e0a29c9511dc34768 /lib
parentdfc92d081ea0332d69c8aca2f0e745cb48ae5e6d (diff)
downloadgitlab-ce-9f46488805e86b1bc341ea1620b866016c2ce5ed.tar.gz
Add latest changes from gitlab-org/gitlab@13-0-stable-ee
Diffstat (limited to 'lib')
-rw-r--r--lib/api/admin/ci/variables.rb137
-rw-r--r--lib/api/api.rb16
-rw-r--r--lib/api/api_guard.rb9
-rw-r--r--lib/api/appearance.rb3
-rw-r--r--lib/api/branches.rb2
-rw-r--r--lib/api/deploy_tokens.rb6
-rw-r--r--lib/api/entities/appearance.rb1
-rw-r--r--lib/api/entities/branch.rb6
-rw-r--r--lib/api/entities/design_management/design.rb16
-rw-r--r--lib/api/entities/freeze_period.rb11
-rw-r--r--lib/api/entities/job_request/artifacts.rb1
-rw-r--r--lib/api/entities/merge_request_basic.rb6
-rw-r--r--lib/api/entities/metrics/user_starred_dashboard.rb11
-rw-r--r--lib/api/entities/personal_snippet.rb3
-rw-r--r--lib/api/entities/project_repository_storage_move.rb14
-rw-r--r--lib/api/entities/release.rb2
-rw-r--r--lib/api/entities/remote_mirror.rb4
-rw-r--r--lib/api/entities/runner_details.rb9
-rw-r--r--lib/api/entities/snippet.rb10
-rw-r--r--lib/api/entities/todo.rb13
-rw-r--r--lib/api/entities/user_basic.rb2
-rw-r--r--lib/api/entities/user_path.rb2
-rw-r--r--lib/api/features.rb22
-rw-r--r--lib/api/freeze_periods.rb107
-rw-r--r--lib/api/groups.rb61
-rw-r--r--lib/api/helpers.rb18
-rw-r--r--lib/api/helpers/internal_helpers.rb2
-rw-r--r--lib/api/helpers/merge_requests_helpers.rb1
-rw-r--r--lib/api/helpers/pagination_strategies.rb34
-rw-r--r--lib/api/helpers/projects_helpers.rb1
-rw-r--r--lib/api/helpers/search_helpers.rb2
-rw-r--r--lib/api/helpers/services_helpers.rb9
-rw-r--r--lib/api/helpers/snippets_helpers.rb17
-rw-r--r--lib/api/internal/base.rb19
-rw-r--r--lib/api/issues.rb2
-rw-r--r--lib/api/job_artifacts.rb4
-rw-r--r--lib/api/members.rb2
-rw-r--r--lib/api/merge_requests.rb7
-rw-r--r--lib/api/metrics/dashboard/annotations.rb45
-rw-r--r--lib/api/metrics/user_starred_dashboards.rb46
-rw-r--r--lib/api/pipelines.rb15
-rw-r--r--lib/api/project_repository_storage_moves.rb34
-rw-r--r--lib/api/project_snippets.rb34
-rw-r--r--lib/api/project_templates.rb12
-rw-r--r--lib/api/projects.rb4
-rw-r--r--lib/api/remote_mirrors.rb2
-rw-r--r--lib/api/search.rb3
-rw-r--r--lib/api/settings.rb16
-rw-r--r--lib/api/snippets.rb31
-rw-r--r--lib/api/wikis.rb14
-rw-r--r--lib/banzai/filter/issue_reference_filter.rb20
-rw-r--r--lib/banzai/filter/upload_link_filter.rb4
-rw-r--r--lib/banzai/filter/user_reference_filter.rb2
-rw-r--r--lib/banzai/pipeline.rb2
-rw-r--r--lib/banzai/reference_parser/design_parser.rb31
-rw-r--r--lib/banzai/renderer.rb7
-rw-r--r--lib/container_registry/client.rb21
-rw-r--r--lib/container_registry/config.rb2
-rw-r--r--lib/csv_builder.rb11
-rw-r--r--lib/csv_builders/single_batch.rb11
-rw-r--r--lib/declarative_policy.rb17
-rw-r--r--lib/feature.rb6
-rw-r--r--lib/gitlab/alert_management/alert_params.rb41
-rw-r--r--lib/gitlab/alert_management/alert_status_counts.rb53
-rw-r--r--lib/gitlab/alerting/alert.rb12
-rw-r--r--lib/gitlab/alerting/notification_payload_parser.rb8
-rw-r--r--lib/gitlab/analytics/cycle_analytics/median.rb6
-rw-r--r--lib/gitlab/auth.rb4
-rw-r--r--lib/gitlab/auth/auth_finders.rb27
-rw-r--r--lib/gitlab/auth/ldap/access.rb4
-rw-r--r--lib/gitlab/auth/ldap/adapter.rb4
-rw-r--r--lib/gitlab/auth/ldap/config.rb4
-rw-r--r--lib/gitlab/auth/ldap/person.rb4
-rw-r--r--lib/gitlab/auth/ldap/user.rb4
-rw-r--r--lib/gitlab/auth/o_auth/auth_hash.rb4
-rw-r--r--lib/gitlab/auth/o_auth/provider.rb14
-rw-r--r--lib/gitlab/auth/o_auth/user.rb4
-rw-r--r--lib/gitlab/auth/result.rb4
-rw-r--r--lib/gitlab/auth/saml/config.rb4
-rw-r--r--lib/gitlab/auth/saml/user.rb4
-rw-r--r--lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb47
-rw-r--r--lib/gitlab/background_migration/backfill_snippet_repositories.rb94
-rw-r--r--lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb2
-rw-r--r--lib/gitlab/background_migration/populate_user_highest_roles_table.rb2
-rw-r--r--lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb13
-rw-r--r--lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb13
-rw-r--r--lib/gitlab/blob_helper.rb16
-rw-r--r--lib/gitlab/chat/responder/mattermost.rb131
-rw-r--r--lib/gitlab/chat_name_token.rb2
-rw-r--r--lib/gitlab/checks/base_checker.rb3
-rw-r--r--lib/gitlab/checks/change_access.rb4
-rw-r--r--lib/gitlab/checks/diff_check.rb3
-rw-r--r--lib/gitlab/ci/ansi2html.rb2
-rw-r--r--lib/gitlab/ci/ansi2json/state.rb2
-rw-r--r--lib/gitlab/ci/build/artifacts/metadata.rb4
-rw-r--r--lib/gitlab/ci/build/artifacts/metadata/entry.rb2
-rw-r--r--lib/gitlab/ci/config/entry/artifacts.rb10
-rw-r--r--lib/gitlab/ci/config/entry/reports.rb4
-rw-r--r--lib/gitlab/ci/config/entry/trigger.rb9
-rw-r--r--lib/gitlab/ci/cron_parser.rb11
-rw-r--r--lib/gitlab/ci/features.rb18
-rw-r--r--lib/gitlab/ci/parsers.rb8
-rw-r--r--lib/gitlab/ci/parsers/accessibility/pa11y.rb33
-rw-r--r--lib/gitlab/ci/parsers/terraform/tfplan.rb35
-rw-r--r--lib/gitlab/ci/parsers/test/junit.rb8
-rw-r--r--lib/gitlab/ci/pipeline/chain/command.rb15
-rw-r--r--lib/gitlab/ci/pipeline/chain/sequence.rb3
-rw-r--r--lib/gitlab/ci/pipeline/seed/build/resource_group.rb1
-rw-r--r--lib/gitlab/ci/reports/accessibility_reports.rb46
-rw-r--r--lib/gitlab/ci/reports/accessibility_reports_comparer.rb55
-rw-r--r--lib/gitlab/ci/reports/terraform_reports.rb27
-rw-r--r--lib/gitlab/ci/reports/test_reports.rb6
-rw-r--r--lib/gitlab/ci/reports/test_suite.rb19
-rw-r--r--lib/gitlab/ci/status/build/failed.rb4
-rw-r--r--lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml3
-rw-r--r--lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml31
-rw-r--r--lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml14
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml5
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml11
-rw-r--r--lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml44
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml154
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml30
-rw-r--r--lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml16
-rw-r--r--lib/gitlab/ci/templates/Scala.gitlab-ci.yml8
-rw-r--r--lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml21
-rw-r--r--lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml26
-rw-r--r--lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml120
-rw-r--r--lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml34
-rw-r--r--lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml25
-rw-r--r--lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml214
-rw-r--r--lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml246
-rw-r--r--lib/gitlab/ci/templates/Terraform.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml7
-rw-r--r--lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml8
-rw-r--r--lib/gitlab/ci/yaml_processor.rb2
-rw-r--r--lib/gitlab/cleanup/orphan_lfs_file_references.rb2
-rw-r--r--lib/gitlab/code_navigation_path.rb16
-rw-r--r--lib/gitlab/config_checker/external_database_checker.rb27
-rw-r--r--lib/gitlab/cycle_analytics/group_stage_summary.rb54
-rw-r--r--lib/gitlab/cycle_analytics/stage_summary.rb5
-rw-r--r--lib/gitlab/cycle_analytics/summary/commit.rb6
-rw-r--r--lib/gitlab/cycle_analytics/summary/deploy.rb16
-rw-r--r--lib/gitlab/cycle_analytics/summary/deployment_frequency.rb3
-rw-r--r--lib/gitlab/cycle_analytics/summary/group/base.rb26
-rw-r--r--lib/gitlab/cycle_analytics/summary/group/deploy.rb31
-rw-r--r--lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb33
-rw-r--r--lib/gitlab/cycle_analytics/summary/group/issue.rb44
-rw-r--r--lib/gitlab/cycle_analytics/summary/issue.rb11
-rw-r--r--lib/gitlab/cycle_analytics/summary/value.rb42
-rw-r--r--lib/gitlab/cycle_analytics/summary_helper.rb5
-rw-r--r--lib/gitlab/danger/changelog.rb4
-rw-r--r--lib/gitlab/danger/commit_linter.rb2
-rw-r--r--lib/gitlab/danger/emoji_checker.rb6
-rw-r--r--lib/gitlab/danger/helper.rb17
-rw-r--r--lib/gitlab/danger/request_helper.rb2
-rw-r--r--lib/gitlab/danger/teammate.rb31
-rw-r--r--lib/gitlab/data_builder/wiki_page.rb3
-rw-r--r--lib/gitlab/database/batch_count.rb8
-rw-r--r--lib/gitlab/database/count/reltuples_count_strategy.rb2
-rw-r--r--lib/gitlab/database/migration_helpers.rb182
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers.rb122
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb13
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb28
-rw-r--r--lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb2
-rw-r--r--lib/gitlab/database/schema_helpers.rb52
-rw-r--r--lib/gitlab/database/with_lock_retries.rb14
-rw-r--r--lib/gitlab/dependency_linker/json_linker.rb2
-rw-r--r--lib/gitlab/diff/file.rb17
-rw-r--r--lib/gitlab/diff/formatters/text_formatter.rb3
-rw-r--r--lib/gitlab/diff/highlight_cache.rb2
-rw-r--r--lib/gitlab/diff/position.rb2
-rw-r--r--lib/gitlab/discussions_diff/highlight_cache.rb2
-rw-r--r--lib/gitlab/elasticsearch/logs/lines.rb9
-rw-r--r--lib/gitlab/email/handler.rb10
-rw-r--r--lib/gitlab/email/hook/smime_signature_interceptor.rb7
-rw-r--r--lib/gitlab/email/message/repository_push.rb2
-rw-r--r--lib/gitlab/email/smime/certificate.rb40
-rw-r--r--lib/gitlab/email/smime/signer.rb25
-rw-r--r--lib/gitlab/emoji.rb4
-rw-r--r--lib/gitlab/exclusive_lease.rb20
-rw-r--r--lib/gitlab/exclusive_lease_helpers.rb39
-rw-r--r--lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb50
-rw-r--r--lib/gitlab/experimentation.rb74
-rw-r--r--lib/gitlab/external_authorization/response.rb2
-rw-r--r--lib/gitlab/git/attributes_parser.rb2
-rw-r--r--lib/gitlab/git/commit.rb14
-rw-r--r--lib/gitlab/git/tag.rb21
-rw-r--r--lib/gitlab/git_access_design.rb28
-rw-r--r--lib/gitlab/git_access_snippet.rb16
-rw-r--r--lib/gitlab/gitaly_client.rb7
-rw-r--r--lib/gitlab/github_import/parallel_importer.rb4
-rw-r--r--lib/gitlab/gl_repository.rb11
-rw-r--r--lib/gitlab/gl_repository/repo_type.rb6
-rw-r--r--lib/gitlab/gon_helper.rb4
-rw-r--r--lib/gitlab/grape_logging/loggers/cloudflare_logger.rb18
-rw-r--r--lib/gitlab/grape_logging/loggers/context_logger.rb14
-rw-r--r--lib/gitlab/grape_logging/loggers/exception_logger.rb28
-rw-r--r--lib/gitlab/graphql/authorize/authorize_field_service.rb8
-rw-r--r--lib/gitlab/graphql/pagination/keyset/connection.rb2
-rw-r--r--lib/gitlab/graphql/pagination/keyset/order_info.rb2
-rw-r--r--lib/gitlab/graphql/query_analyzers/logger_analyzer.rb4
-rw-r--r--lib/gitlab/graphql/variables.rb2
-rw-r--r--lib/gitlab/group_search_results.rb4
-rw-r--r--lib/gitlab/health_checks/puma_check.rb2
-rw-r--r--lib/gitlab/i18n.rb40
-rw-r--r--lib/gitlab/import_export.rb8
-rw-r--r--lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb5
-rw-r--r--lib/gitlab/import_export/design_repo_restorer.rb15
-rw-r--r--lib/gitlab/import_export/design_repo_saver.rb19
-rw-r--r--lib/gitlab/import_export/group/group_restorer.rb71
-rw-r--r--lib/gitlab/import_export/group/import_export.yml2
-rw-r--r--lib/gitlab/import_export/group/legacy_import_export.yml86
-rw-r--r--lib/gitlab/import_export/group/legacy_tree_restorer.rb2
-rw-r--r--lib/gitlab/import_export/group/legacy_tree_saver.rb2
-rw-r--r--lib/gitlab/import_export/group/tree_restorer.rb140
-rw-r--r--lib/gitlab/import_export/group/tree_saver.rb72
-rw-r--r--lib/gitlab/import_export/importer.rb26
-rw-r--r--lib/gitlab/import_export/project/base_task.rb20
-rw-r--r--lib/gitlab/import_export/project/export_task.rb13
-rw-r--r--lib/gitlab/import_export/project/import_export.yml27
-rw-r--r--lib/gitlab/import_export/project/import_task.rb34
-rw-r--r--lib/gitlab/import_export/project/object_builder.rb8
-rw-r--r--lib/gitlab/import_export/project/relation_factory.rb9
-rw-r--r--lib/gitlab/import_export/project/tree_restorer.rb2
-rw-r--r--lib/gitlab/import_export/project/tree_saver.rb2
-rw-r--r--lib/gitlab/import_export/relation_tree_restorer.rb4
-rw-r--r--lib/gitlab/import_export/snippets_repo_restorer.rb1
-rw-r--r--lib/gitlab/import_export/snippets_repo_saver.rb2
-rw-r--r--lib/gitlab/instrumentation/redis.rb3
-rw-r--r--lib/gitlab/instrumentation_helper.rb9
-rw-r--r--lib/gitlab/jira_import/base_importer.rb5
-rw-r--r--lib/gitlab/jira_import/handle_labels_service.rb39
-rw-r--r--lib/gitlab/jira_import/issue_serializer.rb62
-rw-r--r--lib/gitlab/jira_import/issues_importer.rb2
-rw-r--r--lib/gitlab/jira_import/labels_importer.rb26
-rw-r--r--lib/gitlab/jira_import/metadata_collector.rb57
-rw-r--r--lib/gitlab/jira_import/user_mapper.rb53
-rw-r--r--lib/gitlab/json.rb34
-rw-r--r--lib/gitlab/kubernetes/helm.rb2
-rw-r--r--lib/gitlab/kubernetes/helm/api.rb6
-rw-r--r--lib/gitlab/kubernetes/helm/delete_command.rb2
-rw-r--r--lib/gitlab/kubernetes/helm/parsers/list_v2.rb37
-rw-r--r--lib/gitlab/kubernetes/kube_client.rb72
-rw-r--r--lib/gitlab/kubernetes/network_policy.rb91
-rw-r--r--lib/gitlab/logging/cloudflare_helper.rb23
-rw-r--r--lib/gitlab/lograge/custom_options.rb8
-rw-r--r--lib/gitlab/mail_room.rb3
-rw-r--r--lib/gitlab/markdown_cache.rb2
-rw-r--r--lib/gitlab/metrics.rb93
-rw-r--r--lib/gitlab/metrics/dashboard/url.rb2
-rw-r--r--lib/gitlab/metrics/exporter/sidekiq_exporter.rb2
-rw-r--r--lib/gitlab/metrics/influx_db.rb183
-rw-r--r--lib/gitlab/metrics/method_call.rb13
-rw-r--r--lib/gitlab/metrics/metric.rb54
-rw-r--r--lib/gitlab/metrics/rack_middleware.rb4
-rw-r--r--lib/gitlab/metrics/samplers/database_sampler.rb58
-rw-r--r--lib/gitlab/metrics/samplers/influx_sampler.rb49
-rw-r--r--lib/gitlab/metrics/samplers/puma_sampler.rb2
-rw-r--r--lib/gitlab/metrics/samplers/ruby_sampler.rb26
-rw-r--r--lib/gitlab/metrics/sidekiq_middleware.rb2
-rw-r--r--lib/gitlab/metrics/subscribers/action_view.rb8
-rw-r--r--lib/gitlab/metrics/system.rb81
-rw-r--r--lib/gitlab/metrics/transaction.rb56
-rw-r--r--lib/gitlab/middleware/multipart.rb1
-rw-r--r--lib/gitlab/middleware/read_only/controller.rb4
-rw-r--r--lib/gitlab/omniauth_initializer.rb83
-rw-r--r--lib/gitlab/pagination/keyset.rb11
-rw-r--r--lib/gitlab/patch/draw_route.rb4
-rw-r--r--lib/gitlab/performance_bar.rb2
-rw-r--r--lib/gitlab/phabricator_import/conduit/response.rb2
-rw-r--r--lib/gitlab/project_search_results.rb36
-rw-r--r--lib/gitlab/prometheus/metric_group.rb3
-rw-r--r--lib/gitlab/prometheus/queries/query_additional_metrics.rb4
-rw-r--r--lib/gitlab/prometheus_client.rb2
-rw-r--r--lib/gitlab/quick_actions/issuable_actions.rb2
-rw-r--r--lib/gitlab/regex.rb8
-rw-r--r--lib/gitlab/request_context.rb7
-rw-r--r--lib/gitlab/rugged_instrumentation.rb3
-rw-r--r--lib/gitlab/runtime.rb22
-rw-r--r--lib/gitlab/sanitizers/exif.rb2
-rw-r--r--lib/gitlab/search/parsed_query.rb4
-rw-r--r--lib/gitlab/search_results.rb13
-rw-r--r--lib/gitlab/services/logger.rb11
-rw-r--r--lib/gitlab/setup_helper.rb3
-rw-r--r--lib/gitlab/sidekiq_config/cli_methods.rb3
-rw-r--r--lib/gitlab/sidekiq_daemon/monitor.rb2
-rw-r--r--lib/gitlab/sidekiq_logging/json_formatter.rb15
-rw-r--r--lib/gitlab/sidekiq_logging/structured_logger.rb11
-rw-r--r--lib/gitlab/sidekiq_middleware.rb5
-rw-r--r--lib/gitlab/sidekiq_middleware/admin_mode/client.rb8
-rw-r--r--lib/gitlab/sidekiq_middleware/arguments_logger.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb25
-rw-r--r--lib/gitlab/slash_commands/presenters/issue_base.rb4
-rw-r--r--lib/gitlab/snippet_search_results.rb34
-rw-r--r--lib/gitlab/static_site_editor/config.rb7
-rw-r--r--lib/gitlab/task_helpers.rb8
-rw-r--r--lib/gitlab/testing/clear_process_memory_cache_middleware.rb (renamed from lib/gitlab/testing/clear_thread_memory_cache_middleware.rb)4
-rw-r--r--lib/gitlab/thread_memory_cache.rb15
-rw-r--r--lib/gitlab/tracking.rb7
-rw-r--r--lib/gitlab/tree_summary.rb36
-rw-r--r--lib/gitlab/uploads/migration_helper.rb3
-rw-r--r--lib/gitlab/url_builder.rb30
-rw-r--r--lib/gitlab/usage_data.rb203
-rw-r--r--lib/gitlab/usage_data_counters/base_counter.rb10
-rw-r--r--lib/gitlab/usage_data_counters/designs_counter.rb42
-rw-r--r--lib/gitlab/usage_data_counters/note_counter.rb10
-rw-r--r--lib/gitlab/usage_data_counters/search_counter.rb4
-rw-r--r--lib/gitlab/usage_data_counters/web_ide_counter.rb52
-rw-r--r--lib/gitlab/user_access_snippet.rb14
-rw-r--r--lib/gitlab/utils.rb6
-rw-r--r--lib/gitlab/utils/measuring.rb58
-rw-r--r--lib/gitlab/wiki_pages.rb3
-rw-r--r--lib/gitlab/with_request_store.rb14
-rw-r--r--lib/gitlab/workhorse.rb15
-rw-r--r--lib/gitlab/x509/signature.rb4
-rw-r--r--lib/gitlab/x509/tag.rb41
-rw-r--r--lib/gitlab_danger.rb2
-rw-r--r--lib/google_api/auth.rb7
-rw-r--r--lib/mattermost/client.rb2
-rw-r--r--lib/quality/helm3_client.rb6
-rw-r--r--lib/quality/helm_client.rb114
-rw-r--r--lib/quality/test_level.rb1
-rw-r--r--lib/rspec_flaky/listener.rb2
-rw-r--r--lib/static_model.rb6
-rwxr-xr-xlib/support/init.d/gitlab26
-rw-r--r--lib/support/init.d/gitlab.default.example6
-rw-r--r--lib/system_check/app/hashed_storage_all_projects_check.rb22
-rw-r--r--lib/system_check/app/hashed_storage_enabled_check.rb23
-rw-r--r--lib/system_check/rake_task/app_task.rb4
-rw-r--r--lib/tasks/file_hooks.rake5
-rw-r--r--lib/tasks/gemojione.rake6
-rw-r--r--lib/tasks/gitlab/gitaly.rake7
-rw-r--r--lib/tasks/gitlab/import_export/export.rake6
-rw-r--r--lib/tasks/gitlab/import_export/import.rake6
-rw-r--r--lib/tasks/gitlab/shell.rake2
-rw-r--r--lib/tasks/gitlab/snippets.rake91
-rw-r--r--lib/tasks/gitlab/track_deployment.rake9
-rw-r--r--lib/tasks/gitlab/workhorse.rake2
-rw-r--r--lib/tasks/sidekiq.rake38
341 files changed, 5188 insertions, 2088 deletions
diff --git a/lib/api/admin/ci/variables.rb b/lib/api/admin/ci/variables.rb
new file mode 100644
index 00000000000..df731148bac
--- /dev/null
+++ b/lib/api/admin/ci/variables.rb
@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module API
+ module Admin
+ module Ci
+ class Variables < Grape::API
+ include PaginationParams
+
+ before { authenticated_as_admin! }
+
+ namespace 'admin' do
+ namespace 'ci' do
+ namespace 'variables' do
+ desc 'Get instance-level variables' do
+ success Entities::Variable
+ end
+ params do
+ use :pagination
+ end
+ get '/' do
+ variables = ::Ci::InstanceVariable.all
+
+ present paginate(variables), with: Entities::Variable
+ end
+
+ desc 'Get a specific variable from a group' do
+ success Entities::Variable
+ end
+ params do
+ requires :key, type: String, desc: 'The key of the variable'
+ end
+ get ':key' do
+ key = params[:key]
+ variable = ::Ci::InstanceVariable.find_by_key(key)
+
+ break not_found!('InstanceVariable') unless variable
+
+ present variable, with: Entities::Variable
+ end
+
+ desc 'Create a new instance-level variable' do
+ success Entities::Variable
+ end
+ params do
+ requires :key,
+ type: String,
+ desc: 'The key of the variable'
+
+ requires :value,
+ type: String,
+ desc: 'The value of the variable'
+
+ optional :protected,
+ type: String,
+ desc: 'Whether the variable is protected'
+
+ optional :masked,
+ type: String,
+ desc: 'Whether the variable is masked'
+
+ optional :variable_type,
+ type: String,
+ values: ::Ci::InstanceVariable.variable_types.keys,
+ desc: 'The type of variable, must be one of env_var or file. Defaults to env_var'
+ end
+ post '/' do
+ variable_params = declared_params(include_missing: false)
+
+ variable = ::Ci::InstanceVariable.new(variable_params)
+
+ if variable.save
+ present variable, with: Entities::Variable
+ else
+ render_validation_error!(variable)
+ end
+ end
+
+ desc 'Update an existing instance-variable' do
+ success Entities::Variable
+ end
+ params do
+ optional :key,
+ type: String,
+ desc: 'The key of the variable'
+
+ optional :value,
+ type: String,
+ desc: 'The value of the variable'
+
+ optional :protected,
+ type: String,
+ desc: 'Whether the variable is protected'
+
+ optional :masked,
+ type: String,
+ desc: 'Whether the variable is masked'
+
+ optional :variable_type,
+ type: String,
+ values: ::Ci::InstanceVariable.variable_types.keys,
+ desc: 'The type of variable, must be one of env_var or file'
+ end
+ put ':key' do
+ variable = ::Ci::InstanceVariable.find_by_key(params[:key])
+
+ break not_found!('InstanceVariable') unless variable
+
+ variable_params = declared_params(include_missing: false).except(:key)
+
+ if variable.update(variable_params)
+ present variable, with: Entities::Variable
+ else
+ render_validation_error!(variable)
+ end
+ end
+
+ desc 'Delete an existing instance-level variable' do
+ success Entities::Variable
+ end
+ params do
+ requires :key, type: String, desc: 'The key of the variable'
+ end
+ delete ':key' do
+ variable = ::Ci::InstanceVariable.find_by_key(params[:key])
+ not_found!('InstanceVariable') unless variable
+
+ variable.destroy
+
+ no_content!
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/api.rb b/lib/api/api.rb
index de9a3120d90..b8135539cda 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -24,7 +24,8 @@ module API
Gitlab::GrapeLogging::Loggers::ExceptionLogger.new,
Gitlab::GrapeLogging::Loggers::QueueDurationLogger.new,
Gitlab::GrapeLogging::Loggers::PerfLogger.new,
- Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new
+ Gitlab::GrapeLogging::Loggers::CorrelationIdLogger.new,
+ Gitlab::GrapeLogging::Loggers::ContextLogger.new
]
allow_access_with_scope :api
@@ -97,6 +98,15 @@ module API
handle_api_exception(exception)
end
+ # This is a specific exception raised by `rack-timeout` gem when Puma
+ # requests surpass its timeout. Given it inherits from Exception, we
+ # should rescue it separately. For more info, see:
+ # - https://github.com/sharpstone/rack-timeout/blob/master/doc/exceptions.md
+ # - https://github.com/ruby-grape/grape#exception-handling
+ rescue_from Rack::Timeout::RequestTimeoutException do |exception|
+ handle_api_exception(exception)
+ end
+
format :json
content_type :txt, "text/plain"
@@ -111,6 +121,7 @@ module API
# Keep in alphabetical order
mount ::API::AccessRequests
+ mount ::API::Admin::Ci::Variables
mount ::API::Admin::Sidekiq
mount ::API::Appearance
mount ::API::Applications
@@ -131,6 +142,7 @@ module API
mount ::API::Events
mount ::API::Features
mount ::API::Files
+ mount ::API::FreezePeriods
mount ::API::GroupBoards
mount ::API::GroupClusters
mount ::API::GroupExport
@@ -153,6 +165,7 @@ module API
mount ::API::MergeRequestDiffs
mount ::API::MergeRequests
mount ::API::Metrics::Dashboard::Annotations
+ mount ::API::Metrics::UserStarredDashboards
mount ::API::Namespaces
mount ::API::Notes
mount ::API::Discussions
@@ -169,6 +182,7 @@ module API
mount ::API::ProjectImport
mount ::API::ProjectHooks
mount ::API::ProjectMilestones
+ mount ::API::ProjectRepositoryStorageMoves
mount ::API::Projects
mount ::API::ProjectSnapshots
mount ::API::ProjectSnippets
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb
index 9dd2de5c7ba..c6557fce541 100644
--- a/lib/api/api_guard.rb
+++ b/lib/api/api_guard.rb
@@ -65,7 +65,8 @@ module API
end
def find_user_from_sources
- find_user_from_access_token ||
+ deploy_token_from_request ||
+ find_user_from_access_token ||
find_user_from_job_token ||
find_user_from_warden
end
@@ -90,12 +91,16 @@ module API
end
def api_access_allowed?(user)
- Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api)
+ user_allowed_or_deploy_token?(user) && user.can?(:access_api)
end
def api_access_denied_message(user)
Gitlab::Auth::UserAccessDeniedReason.new(user).rejection_message
end
+
+ def user_allowed_or_deploy_token?(user)
+ Gitlab::UserAccess.new(user).allowed? || user.is_a?(DeployToken)
+ end
end
class_methods do
diff --git a/lib/api/appearance.rb b/lib/api/appearance.rb
index a775102e87d..71a35bb4493 100644
--- a/lib/api/appearance.rb
+++ b/lib/api/appearance.rb
@@ -27,7 +27,8 @@ module API
optional :logo, type: File, desc: 'Instance image used on the sign in / sign up page' # rubocop:disable Scalability/FileUploads
optional :header_logo, type: File, desc: 'Instance image used for the main navigation bar' # rubocop:disable Scalability/FileUploads
optional :favicon, type: File, desc: 'Instance favicon in .ico/.png format' # rubocop:disable Scalability/FileUploads
- optional :new_project_guidelines, type: String, desc: 'Markmarkdown text shown on the new project page'
+ optional :new_project_guidelines, type: String, desc: 'Markdown text shown on the new project page'
+ optional :profile_image_guidelines, type: String, desc: 'Markdown text shown on the profile page below Public Avatar'
optional :header_message, type: String, desc: 'Message within the system header bar'
optional :footer_message, type: String, desc: 'Message within the system footer bar'
optional :message_background_color, type: String, desc: 'Background color for the system header / footer bar'
diff --git a/lib/api/branches.rb b/lib/api/branches.rb
index 999bf1627c1..081e8ffe4f0 100644
--- a/lib/api/branches.rb
+++ b/lib/api/branches.rb
@@ -8,6 +8,8 @@ module API
BRANCH_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(branch: API::NO_SLASH_URL_PART_REGEX)
+ after_validation { content_type "application/json" }
+
before do
require_repository_enabled!
authorize! :download_code, user_project
diff --git a/lib/api/deploy_tokens.rb b/lib/api/deploy_tokens.rb
index f3a08ae970a..0fbbd96cf02 100644
--- a/lib/api/deploy_tokens.rb
+++ b/lib/api/deploy_tokens.rb
@@ -11,6 +11,8 @@ module API
result_hash = Hashie::Mash.new
result_hash[:read_registry] = scopes.include?('read_registry')
result_hash[:write_registry] = scopes.include?('write_registry')
+ result_hash[:read_package_registry] = scopes.include?('read_package_registry')
+ result_hash[:write_package_registry] = scopes.include?('write_package_registry')
result_hash[:read_repository] = scopes.include?('read_repository')
result_hash
end
@@ -55,7 +57,7 @@ module API
params do
requires :name, type: String, desc: "New deploy token's name"
requires :scopes, type: Array[String], values: ::DeployToken::AVAILABLE_SCOPES.map(&:to_s),
- desc: 'Indicates the deploy token scopes. Must be at least one of "read_repository", "read_registry", or "write_registry".'
+ desc: 'Indicates the deploy token scopes. Must be at least one of "read_repository", "read_registry", "write_registry", "read_package_registry", or "write_package_registry".'
optional :expires_at, type: DateTime, desc: 'Expiration date for the deploy token. Does not expire if no value is provided.'
optional :username, type: String, desc: 'Username for deploy token. Default is `gitlab+deploy-token-{n}`'
end
@@ -118,7 +120,7 @@ module API
params do
requires :name, type: String, desc: 'The name of the deploy token'
requires :scopes, type: Array[String], values: ::DeployToken::AVAILABLE_SCOPES.map(&:to_s),
- desc: 'Indicates the deploy token scopes. Must be at least one of "read_repository", "read_registry", or "write_registry".'
+ desc: 'Indicates the deploy token scopes. Must be at least one of "read_repository", "read_registry", "write_registry", "read_package_registry", or "write_package_registry".'
optional :expires_at, type: DateTime, desc: 'Expiration date for the deploy token. Does not expire if no value is provided.'
optional :username, type: String, desc: 'Username for deploy token. Default is `gitlab+deploy-token-{n}`'
end
diff --git a/lib/api/entities/appearance.rb b/lib/api/entities/appearance.rb
index c3cffc8d05c..a09faf55f48 100644
--- a/lib/api/entities/appearance.rb
+++ b/lib/api/entities/appearance.rb
@@ -19,6 +19,7 @@ module API
end
expose :new_project_guidelines
+ expose :profile_image_guidelines
expose :header_message
expose :footer_message
expose :message_background_color
diff --git a/lib/api/entities/branch.rb b/lib/api/entities/branch.rb
index 1d5017ac702..f9d06082ad6 100644
--- a/lib/api/entities/branch.rb
+++ b/lib/api/entities/branch.rb
@@ -3,6 +3,8 @@
module API
module Entities
class Branch < Grape::Entity
+ include Gitlab::Routing
+
expose :name
expose :commit, using: Entities::Commit do |repo_branch, options|
@@ -36,6 +38,10 @@ module API
expose :default do |repo_branch, options|
options[:project].default_branch == repo_branch.name
end
+
+ expose :web_url do |repo_branch|
+ project_tree_url(options[:project], repo_branch.name)
+ end
end
end
end
diff --git a/lib/api/entities/design_management/design.rb b/lib/api/entities/design_management/design.rb
new file mode 100644
index 00000000000..183fe06d8f1
--- /dev/null
+++ b/lib/api/entities/design_management/design.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module DesignManagement
+ class Design < Grape::Entity
+ expose :id
+ expose :project_id
+ expose :filename
+ expose :image_url do |design|
+ ::Gitlab::UrlBuilder.build(design)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/freeze_period.rb b/lib/api/entities/freeze_period.rb
new file mode 100644
index 00000000000..9b5f08925db
--- /dev/null
+++ b/lib/api/entities/freeze_period.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class FreezePeriod < Grape::Entity
+ expose :id
+ expose :freeze_start, :freeze_end, :cron_timezone
+ expose :created_at, :updated_at
+ end
+ end
+end
diff --git a/lib/api/entities/job_request/artifacts.rb b/lib/api/entities/job_request/artifacts.rb
index c6871fdd875..0d27f5a9189 100644
--- a/lib/api/entities/job_request/artifacts.rb
+++ b/lib/api/entities/job_request/artifacts.rb
@@ -7,6 +7,7 @@ module API
expose :name
expose :untracked
expose :paths
+ expose :exclude, expose_nil: false
expose :when
expose :expire_in
expose :artifact_type
diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb
index 4610220e4f6..1a89a83a619 100644
--- a/lib/api/entities/merge_request_basic.rb
+++ b/lib/api/entities/merge_request_basic.rb
@@ -50,8 +50,10 @@ module API
# use `MergeRequest#mergeable?` instead (boolean).
# See https://gitlab.com/gitlab-org/gitlab-foss/issues/42344 for more
# information.
- expose :merge_status do |merge_request|
- merge_request.check_mergeability(async: true)
+ #
+ # For list endpoints, we skip the recheck by default, since it's expensive
+ expose :merge_status do |merge_request, options|
+ merge_request.check_mergeability(async: true) unless options[:skip_merge_status_recheck]
merge_request.public_merge_status
end
expose :diff_head_sha, as: :sha
diff --git a/lib/api/entities/metrics/user_starred_dashboard.rb b/lib/api/entities/metrics/user_starred_dashboard.rb
new file mode 100644
index 00000000000..d774160e3ea
--- /dev/null
+++ b/lib/api/entities/metrics/user_starred_dashboard.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ module Metrics
+ class UserStarredDashboard < Grape::Entity
+ expose :id, :dashboard_path, :user_id, :project_id
+ end
+ end
+ end
+end
diff --git a/lib/api/entities/personal_snippet.rb b/lib/api/entities/personal_snippet.rb
index eb0266e61e6..03ab6c0809c 100644
--- a/lib/api/entities/personal_snippet.rb
+++ b/lib/api/entities/personal_snippet.rb
@@ -3,9 +3,6 @@
module API
module Entities
class PersonalSnippet < Snippet
- expose :raw_url do |snippet|
- Gitlab::UrlBuilder.build(snippet, raw: true)
- end
end
end
end
diff --git a/lib/api/entities/project_repository_storage_move.rb b/lib/api/entities/project_repository_storage_move.rb
new file mode 100644
index 00000000000..25643651a14
--- /dev/null
+++ b/lib/api/entities/project_repository_storage_move.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class ProjectRepositoryStorageMove < Grape::Entity
+ expose :id
+ expose :created_at
+ expose :human_state_name, as: :state
+ expose :source_storage_name
+ expose :destination_storage_name
+ expose :project, using: Entities::ProjectIdentity
+ end
+ end
+end
diff --git a/lib/api/entities/release.rb b/lib/api/entities/release.rb
index edcd9bc6505..99fa496d368 100644
--- a/lib/api/entities/release.rb
+++ b/lib/api/entities/release.rb
@@ -21,7 +21,6 @@ module API
expose :milestones, using: Entities::MilestoneWithStats, if: -> (release, _) { release.milestones.present? && can_read_milestone? }
expose :commit_path, expose_nil: false
expose :tag_path, expose_nil: false
- expose :evidence_sha, expose_nil: false, if: ->(_, _) { can_download_code? }
expose :assets do
expose :assets_count, as: :count do |release, _|
@@ -32,7 +31,6 @@ module API
expose :links, using: Entities::Releases::Link do |release, options|
release.links.sorted
end
- expose :evidence_file_path, expose_nil: false, if: ->(_, _) { can_download_code? }
end
expose :evidences, using: Entities::Releases::Evidence, expose_nil: false, if: ->(_, _) { can_download_code? }
expose :_links do
diff --git a/lib/api/entities/remote_mirror.rb b/lib/api/entities/remote_mirror.rb
index 18d51726bab..87daef9a05c 100644
--- a/lib/api/entities/remote_mirror.rb
+++ b/lib/api/entities/remote_mirror.rb
@@ -12,9 +12,7 @@ module API
expose :last_successful_update_at
expose :last_error
expose :only_protected_branches
- expose :keep_divergent_refs, if: -> (mirror, _options) do
- ::Feature.enabled?(:keep_divergent_refs, mirror.project)
- end
+ expose :keep_divergent_refs
end
end
end
diff --git a/lib/api/entities/runner_details.rb b/lib/api/entities/runner_details.rb
index 2bb143253fe..1dd8543d595 100644
--- a/lib/api/entities/runner_details.rb
+++ b/lib/api/entities/runner_details.rb
@@ -11,9 +11,12 @@ module API
expose :version, :revision, :platform, :architecture
expose :contacted_at
- # @deprecated in 12.10 https://gitlab.com/gitlab-org/gitlab/-/issues/214320
- # will be removed by 13.0 https://gitlab.com/gitlab-org/gitlab/-/issues/214322
- expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.instance_type? }
+ # Will be removed: https://gitlab.com/gitlab-org/gitlab/-/issues/217105
+ expose(:token, if: ->(runner, options) do
+ return false if ::Feature.enabled?(:hide_token_from_runners_api, default_enabled: true)
+
+ options[:current_user].admin? || !runner.instance_type?
+ end)
# rubocop: disable CodeReuse/ActiveRecord
expose :projects, with: Entities::BasicProjectDetails do |runner, options|
diff --git a/lib/api/entities/snippet.rb b/lib/api/entities/snippet.rb
index 8a13b4746a9..19c89603cbc 100644
--- a/lib/api/entities/snippet.rb
+++ b/lib/api/entities/snippet.rb
@@ -3,14 +3,20 @@
module API
module Entities
class Snippet < Grape::Entity
- expose :id, :title, :file_name, :description, :visibility
+ expose :id, :title, :description, :visibility
expose :author, using: Entities::UserBasic
expose :updated_at, :created_at
expose :project_id
expose :web_url do |snippet|
Gitlab::UrlBuilder.build(snippet)
end
- expose :ssh_url_to_repo, :http_url_to_repo, if: ->(snippet) { snippet.versioned_enabled_for?(options[:current_user]) }
+ expose :raw_url do |snippet|
+ Gitlab::UrlBuilder.build(snippet, raw: true)
+ end
+ expose :ssh_url_to_repo, :http_url_to_repo, if: ->(snippet) { snippet.repository_exists? }
+ expose :file_name do |snippet|
+ snippet.file_name_on_repo || snippet.file_name
+ end
end
end
end
diff --git a/lib/api/entities/todo.rb b/lib/api/entities/todo.rb
index abfdde89bf1..0acbb4cb704 100644
--- a/lib/api/entities/todo.rb
+++ b/lib/api/entities/todo.rb
@@ -22,6 +22,7 @@ module API
expose :body
expose :state
expose :created_at
+ expose :updated_at
def todo_target_class(target_type)
# false as second argument prevents looking up in module hierarchy
@@ -30,6 +31,8 @@ module API
end
def todo_target_url(todo)
+ return design_todo_target_url(todo) if todo.for_design?
+
target_type = todo.target_type.underscore
target_url = "#{todo.resource_parent.class.to_s.underscore}_#{target_type}_url"
@@ -41,6 +44,16 @@ module API
def todo_target_anchor(todo)
"note_#{todo.note_id}" if todo.note_id?
end
+
+ def design_todo_target_url(todo)
+ design = todo.target
+ path_options = {
+ anchor: todo_target_anchor(todo),
+ vueroute: design.filename
+ }
+
+ ::Gitlab::Routing.url_helpers.designs_project_issue_url(design.project, design.issue, path_options)
+ end
end
end
end
diff --git a/lib/api/entities/user_basic.rb b/lib/api/entities/user_basic.rb
index e063aa42855..80f3ee7b502 100644
--- a/lib/api/entities/user_basic.rb
+++ b/lib/api/entities/user_basic.rb
@@ -18,3 +18,5 @@ module API
end
end
end
+
+API::Entities::UserBasic.prepend_if_ee('EE::API::Entities::UserBasic')
diff --git a/lib/api/entities/user_path.rb b/lib/api/entities/user_path.rb
index 7d922b39654..3f007659813 100644
--- a/lib/api/entities/user_path.rb
+++ b/lib/api/entities/user_path.rb
@@ -12,3 +12,5 @@ module API
end
end
end
+
+API::Entities::UserPath.prepend_if_ee('EE::API::Entities::UserPath')
diff --git a/lib/api/features.rb b/lib/api/features.rb
index 69b751e9bdb..f507919b055 100644
--- a/lib/api/features.rb
+++ b/lib/api/features.rb
@@ -16,6 +16,15 @@ module API
end
end
+ def gate_key(params)
+ case params[:key]
+ when 'percentage_of_actors'
+ :percentage_of_actors
+ else
+ :percentage_of_time
+ end
+ end
+
def gate_targets(params)
Feature::Target.new(params).targets
end
@@ -40,15 +49,22 @@ module API
end
params do
requires :value, type: String, desc: '`true` or `false` to enable/disable, an integer for percentage of time'
+ optional :key, type: String, desc: '`percentage_of_actors` or the default `percentage_of_time`'
optional :feature_group, type: String, desc: 'A Feature group name'
optional :user, type: String, desc: 'A GitLab username'
optional :group, type: String, desc: "A GitLab group's path, such as 'gitlab-org'"
optional :project, type: String, desc: 'A projects path, like gitlab-org/gitlab-ce'
+
+ mutually_exclusive :key, :feature_group
+ mutually_exclusive :key, :user
+ mutually_exclusive :key, :group
+ mutually_exclusive :key, :project
end
post ':name' do
feature = Feature.get(params[:name])
targets = gate_targets(params)
value = gate_value(params)
+ key = gate_key(params)
case value
when true
@@ -64,7 +80,11 @@ module API
feature.disable
end
else
- feature.enable_percentage_of_time(value)
+ if key == :percentage_of_actors
+ feature.enable_percentage_of_actors(value)
+ else
+ feature.enable_percentage_of_time(value)
+ end
end
present feature, with: Entities::Feature, current_user: current_user
diff --git a/lib/api/freeze_periods.rb b/lib/api/freeze_periods.rb
new file mode 100644
index 00000000000..9c7e5a5832d
--- /dev/null
+++ b/lib/api/freeze_periods.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module API
+ class FreezePeriods < Grape::API
+ include PaginationParams
+
+ before { authenticate! }
+
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc 'Get project freeze periods' do
+ detail 'This feature was introduced in GitLab 13.0.'
+ success Entities::FreezePeriod
+ end
+ params do
+ use :pagination
+ end
+
+ get ":id/freeze_periods" do
+ authorize! :read_freeze_period, user_project
+
+ freeze_periods = ::FreezePeriodsFinder.new(user_project, current_user).execute
+
+ present paginate(freeze_periods), with: Entities::FreezePeriod, current_user: current_user
+ end
+
+ desc 'Get a single freeze period' do
+ detail 'This feature was introduced in GitLab 13.0.'
+ success Entities::FreezePeriod
+ end
+ params do
+ requires :freeze_period_id, type: Integer, desc: 'The ID of a project freeze period'
+ end
+ get ":id/freeze_periods/:freeze_period_id" do
+ authorize! :read_freeze_period, user_project
+
+ present freeze_period, with: Entities::FreezePeriod, current_user: current_user
+ end
+
+ desc 'Create a new freeze period' do
+ detail 'This feature was introduced in GitLab 13.0.'
+ success Entities::FreezePeriod
+ end
+ params do
+ requires :freeze_start, type: String, desc: 'Freeze Period start'
+ requires :freeze_end, type: String, desc: 'Freeze Period end'
+ optional :cron_timezone, type: String, desc: 'Timezone'
+ end
+ post ':id/freeze_periods' do
+ authorize! :create_freeze_period, user_project
+
+ freeze_period_params = declared(params, include_parent_namespaces: false)
+
+ freeze_period = user_project.freeze_periods.create(freeze_period_params)
+
+ if freeze_period.persisted?
+ present freeze_period, with: Entities::FreezePeriod
+ else
+ render_validation_error!(freeze_period)
+ end
+ end
+
+ desc 'Update a freeze period' do
+ detail 'This feature was introduced in GitLab 13.0.'
+ success Entities::FreezePeriod
+ end
+ params do
+ optional :freeze_start, type: String, desc: 'Freeze Period start'
+ optional :freeze_end, type: String, desc: 'Freeze Period end'
+ optional :cron_timezone, type: String, desc: 'Freeze Period Timezone'
+ end
+ put ':id/freeze_periods/:freeze_period_id' do
+ authorize! :update_freeze_period, user_project
+
+ freeze_period_params = declared(params, include_parent_namespaces: false, include_missing: false)
+
+ if freeze_period.update(freeze_period_params)
+ present freeze_period, with: Entities::FreezePeriod
+ else
+ render_validation_error!(freeze_period)
+ end
+ end
+
+ desc 'Delete a freeze period' do
+ detail 'This feature was introduced in GitLab 13.0.'
+ success Entities::FreezePeriod
+ end
+ params do
+ requires :freeze_period_id, type: Integer, desc: 'Freeze Period ID'
+ end
+ delete ':id/freeze_periods/:freeze_period_id' do
+ authorize! :destroy_freeze_period, user_project
+
+ destroy_conditionally!(freeze_period)
+ end
+ end
+
+ helpers do
+ def freeze_period
+ @freeze_period ||= user_project.freeze_periods.find(params[:freeze_period_id])
+ end
+ end
+ end
+end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index d375c35e8c0..353c8b4b242 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -60,18 +60,14 @@ module API
.execute
end
- def find_group_projects(params)
+ def find_group_projects(params, finder_options)
group = find_group!(params[:id])
- options = {
- only_owned: !params[:with_shared],
- include_subgroups: params[:include_subgroups]
- }
projects = GroupProjectsFinder.new(
group: group,
current_user: current_user,
params: project_finder_params,
- options: options
+ options: finder_options
).execute
projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled]
projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled]
@@ -80,11 +76,22 @@ module API
paginate(projects)
end
+ def present_projects(params, projects)
+ options = {
+ with: params[:simple] ? Entities::BasicProjectDetails : Entities::Project,
+ current_user: current_user
+ }
+
+ projects, options = with_custom_attributes(projects, options)
+
+ present options[:with].prepare_relation(projects), options
+ end
+
def present_groups(params, groups)
options = {
with: Entities::Group,
current_user: current_user,
- statistics: params[:statistics] && current_user.admin?
+ statistics: params[:statistics] && current_user&.admin?
}
groups = groups.with_statistics if options[:statistics]
@@ -226,16 +233,42 @@ module API
use :optional_projects_params
end
get ":id/projects" do
- projects = find_group_projects(params)
-
- options = {
- with: params[:simple] ? Entities::BasicProjectDetails : Entities::Project,
- current_user: current_user
+ finder_options = {
+ only_owned: !params[:with_shared],
+ include_subgroups: params[:include_subgroups]
}
- projects, options = with_custom_attributes(projects, options)
+ projects = find_group_projects(params, finder_options)
- present options[:with].prepare_relation(projects), options
+ present_projects(params, projects)
+ end
+
+ desc 'Get a list of shared projects in this group' do
+ success Entities::Project
+ end
+ params do
+ optional :archived, type: Boolean, default: false, desc: 'Limit by archived status'
+ optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values,
+ desc: 'Limit by visibility'
+ optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
+ optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
+ default: 'created_at', desc: 'Return projects ordered by field'
+ optional :sort, type: String, values: %w[asc desc], default: 'desc',
+ desc: 'Return projects sorted in ascending and descending order'
+ optional :simple, type: Boolean, default: false,
+ desc: 'Return only the ID, URL, name, and path of each project'
+ optional :starred, type: Boolean, default: false, desc: 'Limit by starred status'
+ optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature'
+ optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature'
+ optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Limit by minimum access level of authenticated user on projects'
+
+ use :pagination
+ use :with_custom_attributes
+ end
+ get ":id/projects/shared" do
+ projects = find_group_projects(params, { only_shared: true })
+
+ present_projects(params, projects)
end
desc 'Get a list of subgroups in this group.' do
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 42b82aac1c4..c6f6dc255d4 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -11,6 +11,7 @@ module API
SUDO_PARAM = :sudo
API_USER_ENV = 'gitlab.api.user'
API_EXCEPTION_ENV = 'gitlab.api.exception'
+ API_RESPONSE_STATUS_CODE = 'gitlab.api.response_status_code'
def declared_params(options = {})
options = { include_parent_namespaces: false }.merge(options)
@@ -178,6 +179,14 @@ module API
end
end
+ def find_tag!(tag_name)
+ if Gitlab::GitRefValidator.validate(tag_name)
+ user_project.repository.find_tag(tag_name) || not_found!('Tag')
+ else
+ render_api_error!('The tag refname is invalid', 400)
+ end
+ end
+
# rubocop: disable CodeReuse/ActiveRecord
def find_project_issue(iid, project_id = nil)
project = project_id ? find_project!(project_id) : user_project
@@ -416,6 +425,11 @@ module API
end
def render_api_error!(message, status)
+ # grape-logging doesn't pass the status code, so this is a
+ # workaround for getting that information in the loggers:
+ # https://github.com/aserafin/grape_logging/issues/71
+ env[API_RESPONSE_STATUS_CODE] = Rack::Utils.status_code(status)
+
error!({ 'message' => message }, status, header)
end
@@ -595,8 +609,8 @@ module API
header(*Gitlab::Workhorse.send_git_archive(repository, **kwargs))
end
- def send_artifacts_entry(build, entry)
- header(*Gitlab::Workhorse.send_artifacts_entry(build, entry))
+ def send_artifacts_entry(file, entry)
+ header(*Gitlab::Workhorse.send_artifacts_entry(file, entry))
end
# The Grape Error Middleware only has access to `env` but not `params` nor
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index 31272c537a3..b05e82a541d 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -51,7 +51,7 @@ module API
def parse_env
return {} if params[:env].blank?
- JSON.parse(params[:env])
+ Gitlab::Json.parse(params[:env])
rescue JSON::ParserError
{}
end
diff --git a/lib/api/helpers/merge_requests_helpers.rb b/lib/api/helpers/merge_requests_helpers.rb
index 73711a7e0ba..9dab2a88f0b 100644
--- a/lib/api/helpers/merge_requests_helpers.rb
+++ b/lib/api/helpers/merge_requests_helpers.rb
@@ -27,6 +27,7 @@ module API
coerce_with: Validations::Types::LabelsList.coerce,
desc: 'Comma-separated list of label names'
optional :with_labels_details, type: Boolean, desc: 'Return titles of labels and other details', default: false
+ optional :with_merge_status_recheck, type: Boolean, desc: 'Request that stale merge statuses be rechecked asynchronously', default: false
optional :created_after, type: DateTime, desc: 'Return merge requests created after the specified time'
optional :created_before, type: DateTime, desc: 'Return merge requests created before the specified time'
optional :updated_after, type: DateTime, desc: 'Return merge requests updated after the specified time'
diff --git a/lib/api/helpers/pagination_strategies.rb b/lib/api/helpers/pagination_strategies.rb
index 6bebb4bfeac..823891d6fe7 100644
--- a/lib/api/helpers/pagination_strategies.rb
+++ b/lib/api/helpers/pagination_strategies.rb
@@ -3,19 +3,24 @@
module API
module Helpers
module PaginationStrategies
- def paginate_with_strategies(relation)
- paginator = paginator(relation)
+ def paginate_with_strategies(relation, request_scope)
+ paginator = paginator(relation, request_scope)
yield(paginator.paginate(relation)).tap do |records, _|
paginator.finalize(records)
end
end
- def paginator(relation)
- return Gitlab::Pagination::OffsetPagination.new(self) unless keyset_pagination_enabled?
+ def paginator(relation, request_scope = nil)
+ return keyset_paginator(relation) if keyset_pagination_enabled?
- request_context = Gitlab::Pagination::Keyset::RequestContext.new(self)
+ offset_paginator(relation, request_scope)
+ end
+
+ private
+ def keyset_paginator(relation)
+ request_context = Gitlab::Pagination::Keyset::RequestContext.new(self)
unless Gitlab::Pagination::Keyset.available?(request_context, relation)
return error!('Keyset pagination is not yet available for this type of request', 405)
end
@@ -23,11 +28,28 @@ module API
Gitlab::Pagination::Keyset::Pager.new(request_context)
end
- private
+ def offset_paginator(relation, request_scope)
+ offset_limit = limit_for_scope(request_scope)
+ if Gitlab::Pagination::Keyset.available_for_type?(relation) && offset_limit_exceeded?(offset_limit)
+ return error!("Offset pagination has a maximum allowed offset of #{offset_limit} " \
+ "for requests that return objects of type #{relation.klass}. " \
+ "Remaining records can be retrieved using keyset pagination.", 405)
+ end
+
+ Gitlab::Pagination::OffsetPagination.new(self)
+ end
def keyset_pagination_enabled?
params[:pagination] == 'keyset'
end
+
+ def limit_for_scope(scope)
+ (scope || Plan.default).actual_limits.offset_pagination_limit
+ end
+
+ def offset_limit_exceeded?(offset_limit)
+ offset_limit.positive? && params[:page] * params[:per_page] > offset_limit
+ end
end
end
end
diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb
index 14c83114f32..5afdb34da97 100644
--- a/lib/api/helpers/projects_helpers.rb
+++ b/lib/api/helpers/projects_helpers.rb
@@ -31,6 +31,7 @@ module API
optional :pages_access_level, type: String, values: %w(disabled private enabled public), desc: 'Pages access level. One of `disabled`, `private`, `enabled` or `public`'
optional :emails_disabled, type: Boolean, desc: 'Disable email notifications'
+ optional :show_default_award_emojis, type: Boolean, desc: 'Show default award emojis'
optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project'
optional :resolve_outdated_diff_discussions, type: Boolean, desc: 'Automatically resolve merge request diffs discussions on lines changed with a push'
optional :remove_source_branch_after_merge, type: Boolean, desc: 'Remove the source branch by default after merge'
diff --git a/lib/api/helpers/search_helpers.rb b/lib/api/helpers/search_helpers.rb
index de8cbe62106..936684ea1f8 100644
--- a/lib/api/helpers/search_helpers.rb
+++ b/lib/api/helpers/search_helpers.rb
@@ -5,7 +5,7 @@ module API
module SearchHelpers
def self.global_search_scopes
# This is a separate method so that EE can redefine it.
- %w(projects issues merge_requests milestones snippet_titles snippet_blobs users)
+ %w(projects issues merge_requests milestones snippet_titles users)
end
def self.group_search_scopes
diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb
index 4c44aca2de4..02e60ff5db5 100644
--- a/lib/api/helpers/services_helpers.rb
+++ b/lib/api/helpers/services_helpers.rb
@@ -724,6 +724,15 @@ module API
desc: 'The Unify Circuit webhook. e.g. https://circuit.com/rest/v2/webhooks/incoming/…'
},
chat_notification_events
+ ].flatten,
+ 'webex-teams' => [
+ {
+ required: true,
+ name: :webhook,
+ type: String,
+ desc: 'The Webex Teams webhook. e.g. https://api.ciscospark.com/v1/webhooks/incoming/…'
+ },
+ chat_notification_events
].flatten
}
end
diff --git a/lib/api/helpers/snippets_helpers.rb b/lib/api/helpers/snippets_helpers.rb
new file mode 100644
index 00000000000..20aeca6a9d3
--- /dev/null
+++ b/lib/api/helpers/snippets_helpers.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module API
+ module Helpers
+ module SnippetsHelpers
+ def content_for(snippet)
+ if snippet.empty_repo?
+ snippet.content
+ else
+ blob = snippet.blobs.first
+ blob.load_all_data!
+ blob.data
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb
index 0d50a310b37..79c407b9581 100644
--- a/lib/api/internal/base.rb
+++ b/lib/api/internal/base.rb
@@ -30,10 +30,6 @@ module API
project.http_url_to_repo
end
- def ee_post_receive_response_hook(response)
- # Hook for EE to add messages
- end
-
def check_allowed(params)
# This is a separate method so that EE can alter its behaviour more
# easily.
@@ -73,13 +69,14 @@ module API
}
# Custom option for git-receive-pack command
+ if Feature.enabled?(:gitaly_upload_pack_filter, project, default_enabled: true)
+ payload[:git_config_options] << "uploadpack.allowFilter=true" << "uploadpack.allowAnySHA1InWant=true"
+ end
+
receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i
+
if receive_max_input_size > 0
payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}"
-
- if Feature.enabled?(:gitaly_upload_pack_filter, project)
- payload[:git_config_options] << "uploadpack.allowFilter=true" << "uploadpack.allowAnySHA1InWant=true"
- end
end
response_with_status(**payload)
@@ -116,10 +113,6 @@ module API
# check_ip - optional, only in EE version, may limit access to
# group resources based on its IP restrictions
post "/allowed" do
- if repo_type.snippet? && params[:protocol] != 'web' && Feature.disabled?(:version_snippets, actor.user)
- break response_with_status(code: 401, success: false, message: 'Snippet git access is disabled.')
- end
-
# It was moved to a separate method so that EE can alter its behaviour more
# easily.
check_allowed(params)
@@ -216,8 +209,6 @@ module API
response = PostReceiveService.new(actor.user, repository, project, params).execute
- ee_post_receive_response_hook(response)
-
present response, with: Entities::InternalPostReceive::Response
end
end
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index f27afd0055f..be50c3e0381 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -95,6 +95,8 @@ module API
use :issues_params
optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], default: 'created_by_me',
desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`'
+ optional :non_archived, type: Boolean, default: true,
+ desc: 'Return issues from non archived projects'
end
get do
authenticate! unless params[:scope] == 'all'
diff --git a/lib/api/job_artifacts.rb b/lib/api/job_artifacts.rb
index 920938ad453..6a82256cc96 100644
--- a/lib/api/job_artifacts.rb
+++ b/lib/api/job_artifacts.rb
@@ -54,7 +54,7 @@ module API
bad_request! unless path.valid?
- send_artifacts_entry(build, path)
+ send_artifacts_entry(build.artifacts_file, path)
end
desc 'Download the artifacts archive from a job' do
@@ -90,7 +90,7 @@ module API
bad_request! unless path.valid?
- send_artifacts_entry(build, path)
+ send_artifacts_entry(build.artifacts_file, path)
end
desc 'Keep the artifacts to prevent them from being deleted' do
diff --git a/lib/api/members.rb b/lib/api/members.rb
index 2e49b4be45c..37d4ca29b68 100644
--- a/lib/api/members.rb
+++ b/lib/api/members.rb
@@ -160,3 +160,5 @@ module API
end
end
end
+
+API::Members.prepend_if_ee('EE::API::Members')
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index d45786cdd3d..ff4ad85115b 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -26,6 +26,8 @@ module API
assignee_ids
description
labels
+ add_labels
+ remove_labels
milestone_id
remove_source_branch
state_event
@@ -91,6 +93,9 @@ module API
options[:with] = Entities::MergeRequestSimple
else
options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest', current_user)
+ if Feature.enabled?(:mr_list_api_skip_merge_status_recheck, default_enabled: true)
+ options[:skip_merge_status_recheck] = !declared_params[:with_merge_status_recheck]
+ end
end
options
@@ -180,6 +185,8 @@ module API
optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue'
optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request'
optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names'
+ optional :add_labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names'
+ optional :remove_labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names'
optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging'
optional :allow_collaboration, type: Boolean, desc: 'Allow commits from members who can merge to the target branch'
optional :allow_maintainer_to_push, type: Boolean, as: :allow_collaboration, desc: '[deprecated] See allow_collaboration'
diff --git a/lib/api/metrics/dashboard/annotations.rb b/lib/api/metrics/dashboard/annotations.rb
index 691abac863a..c8ec4d29657 100644
--- a/lib/api/metrics/dashboard/annotations.rb
+++ b/lib/api/metrics/dashboard/annotations.rb
@@ -8,30 +8,37 @@ module API
success Entities::Metrics::Dashboard::Annotation
end
- params do
- requires :starting_at, type: DateTime,
- desc: 'Date time indicating starting moment to which the annotation relates.'
- optional :ending_at, type: DateTime,
- desc: 'Date time indicating ending moment to which the annotation relates.'
- requires :dashboard_path, type: String,
- desc: 'The path to a file defining the dashboard on which the annotation should be added'
- requires :description, type: String, desc: 'The description of the annotation'
- end
+ ANNOTATIONS_SOURCES = [
+ { class: ::Environment, resource: :environments, create_service_param_key: :environment },
+ { class: Clusters::Cluster, resource: :clusters, create_service_param_key: :cluster }
+ ].freeze
+
+ ANNOTATIONS_SOURCES.each do |annotations_source|
+ resource annotations_source[:resource] do
+ params do
+ requires :starting_at, type: DateTime,
+ desc: 'Date time indicating starting moment to which the annotation relates.'
+ optional :ending_at, type: DateTime,
+ desc: 'Date time indicating ending moment to which the annotation relates.'
+ requires :dashboard_path, type: String, coerce_with: -> (val) { CGI.unescape(val) },
+ desc: 'The path to a file defining the dashboard on which the annotation should be added'
+ requires :description, type: String, desc: 'The description of the annotation'
+ end
- resource :environments do
- post ':id/metrics_dashboard/annotations' do
- environment = ::Environment.find(params[:id])
+ post ':id/metrics_dashboard/annotations' do
+ annotations_source_object = annotations_source[:class].find(params[:id])
- not_found! unless Feature.enabled?(:metrics_dashboard_annotations, environment.project)
+ forbidden! unless can?(current_user, :create_metrics_dashboard_annotation, annotations_source_object)
- forbidden! unless can?(current_user, :create_metrics_dashboard_annotation, environment)
+ create_service_params = declared(params).merge(annotations_source[:create_service_param_key] => annotations_source_object)
- result = ::Metrics::Dashboard::Annotations::CreateService.new(current_user, declared(params).merge(environment: environment)).execute
+ result = ::Metrics::Dashboard::Annotations::CreateService.new(current_user, create_service_params).execute
- if result[:status] == :success
- present result[:annotation], with: Entities::Metrics::Dashboard::Annotation
- else
- error!(result, 400)
+ if result[:status] == :success
+ present result[:annotation], with: Entities::Metrics::Dashboard::Annotation
+ else
+ error!(result, 400)
+ end
end
end
end
diff --git a/lib/api/metrics/user_starred_dashboards.rb b/lib/api/metrics/user_starred_dashboards.rb
new file mode 100644
index 00000000000..85fc0f33ed8
--- /dev/null
+++ b/lib/api/metrics/user_starred_dashboards.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module API
+ module Metrics
+ class UserStarredDashboards < Grape::API
+ resource :projects do
+ desc 'Marks selected metrics dashboard as starred' do
+ success Entities::Metrics::UserStarredDashboard
+ end
+
+ params do
+ requires :dashboard_path, type: String, allow_blank: false, coerce_with: ->(val) { CGI.unescape(val) },
+ desc: 'Url encoded path to a file defining the dashboard to which the star should be added'
+ end
+
+ post ':id/metrics/user_starred_dashboards' do
+ result = ::Metrics::UsersStarredDashboards::CreateService.new(current_user, user_project, params[:dashboard_path]).execute
+
+ if result.success?
+ present result.payload, with: Entities::Metrics::UserStarredDashboard
+ else
+ error!({ errors: result.message }, 400)
+ end
+ end
+
+ desc 'Remove star from selected metrics dashboard'
+
+ params do
+ optional :dashboard_path, type: String, allow_blank: false, coerce_with: ->(val) { CGI.unescape(val) },
+ desc: 'Url encoded path to a file defining the dashboard from which the star should be removed'
+ end
+
+ delete ':id/metrics/user_starred_dashboards' do
+ result = ::Metrics::UsersStarredDashboards::DeleteService.new(current_user, user_project, params[:dashboard_path]).execute
+
+ if result.success?
+ status :ok
+ result.payload
+ else
+ status :bad_request
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb
index 06f8920b37c..c09bca26a41 100644
--- a/lib/api/pipelines.rb
+++ b/lib/api/pipelines.rb
@@ -108,6 +108,21 @@ module API
present pipeline.variables, with: Entities::Variable
end
+ desc 'Gets the test report for a given pipeline' do
+ detail 'This feature was introduced in GitLab 13.0. Disabled by default behind feature flag `junit_pipeline_view`'
+ success TestReportEntity
+ end
+ params do
+ requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
+ end
+ get ':id/pipelines/:pipeline_id/test_report' do
+ not_found! unless Feature.enabled?(:junit_pipeline_view, user_project)
+
+ authorize! :read_build, pipeline
+
+ present pipeline.test_reports, with: TestReportEntity
+ end
+
desc 'Deletes a pipeline' do
detail 'This feature was introduced in GitLab 11.6'
http_codes [[204, 'Pipeline was deleted'], [403, 'Forbidden']]
diff --git a/lib/api/project_repository_storage_moves.rb b/lib/api/project_repository_storage_moves.rb
new file mode 100644
index 00000000000..1a63e984fbf
--- /dev/null
+++ b/lib/api/project_repository_storage_moves.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module API
+ class ProjectRepositoryStorageMoves < Grape::API
+ include PaginationParams
+
+ before { authenticated_as_admin! }
+
+ resource :project_repository_storage_moves do
+ desc 'Get a list of all project repository storage moves' do
+ detail 'This feature was introduced in GitLab 13.0.'
+ success Entities::ProjectRepositoryStorageMove
+ end
+ params do
+ use :pagination
+ end
+ get do
+ storage_moves = ProjectRepositoryStorageMove.with_projects.order_created_at_desc
+
+ present paginate(storage_moves), with: Entities::ProjectRepositoryStorageMove, current_user: current_user
+ end
+
+ desc 'Get a project repository storage move' do
+ detail 'This feature was introduced in GitLab 13.0.'
+ success Entities::ProjectRepositoryStorageMove
+ end
+ get ':id' do
+ storage_move = ProjectRepositoryStorageMove.find(params[:id])
+
+ present storage_move, with: Entities::ProjectRepositoryStorageMove, current_user: current_user
+ end
+ end
+ end
+end
diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb
index e8234a9285c..68f4a0dcb65 100644
--- a/lib/api/project_snippets.rb
+++ b/lib/api/project_snippets.rb
@@ -11,6 +11,7 @@ module API
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ helpers Helpers::SnippetsHelpers
helpers do
def check_snippets_enabled
forbidden! unless user_project.feature_available?(:snippets, current_user)
@@ -54,30 +55,27 @@ module API
success Entities::ProjectSnippet
end
params do
- requires :title, type: String, desc: 'The title of the snippet'
+ requires :title, type: String, allow_blank: false, desc: 'The title of the snippet'
requires :file_name, type: String, desc: 'The file name of the snippet'
- optional :code, type: String, allow_blank: false, desc: 'The content of the snippet (deprecated in favor of "content")'
- optional :content, type: String, allow_blank: false, desc: 'The content of the snippet'
+ requires :content, type: String, allow_blank: false, desc: 'The content of the snippet'
optional :description, type: String, desc: 'The description of a snippet'
requires :visibility, type: String,
values: Gitlab::VisibilityLevel.string_values,
desc: 'The visibility of the snippet'
- mutually_exclusive :code, :content
end
post ":id/snippets" do
authorize! :create_snippet, user_project
snippet_params = declared_params(include_missing: false).merge(request: request, api: true)
- snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present?
service_response = ::Snippets::CreateService.new(user_project, current_user, snippet_params).execute
snippet = service_response.payload[:snippet]
- render_spam_error! if snippet.spam?
-
- if snippet.persisted?
+ if service_response.success?
present snippet, with: Entities::ProjectSnippet
else
- render_validation_error!(snippet)
+ render_spam_error! if snippet.spam?
+
+ render_api_error!({ error: service_response.message }, service_response.http_status)
end
end
@@ -86,16 +84,14 @@ module API
end
params do
requires :snippet_id, type: Integer, desc: 'The ID of a project snippet'
- optional :title, type: String, desc: 'The title of the snippet'
+ optional :title, type: String, allow_blank: false, desc: 'The title of the snippet'
optional :file_name, type: String, desc: 'The file name of the snippet'
- optional :code, type: String, allow_blank: false, desc: 'The content of the snippet (deprecated in favor of "content")'
optional :content, type: String, allow_blank: false, desc: 'The content of the snippet'
optional :description, type: String, desc: 'The description of a snippet'
optional :visibility, type: String,
values: Gitlab::VisibilityLevel.string_values,
desc: 'The visibility of the snippet'
- at_least_one_of :title, :file_name, :code, :content, :visibility_level
- mutually_exclusive :code, :content
+ at_least_one_of :title, :file_name, :content, :visibility_level
end
# rubocop: disable CodeReuse/ActiveRecord
put ":id/snippets/:snippet_id" do
@@ -107,17 +103,15 @@ module API
snippet_params = declared_params(include_missing: false)
.merge(request: request, api: true)
- snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present?
-
service_response = ::Snippets::UpdateService.new(user_project, current_user, snippet_params).execute(snippet)
snippet = service_response.payload[:snippet]
- render_spam_error! if snippet.spam?
-
- if snippet.valid?
+ if service_response.success?
present snippet, with: Entities::ProjectSnippet
else
- render_validation_error!(snippet)
+ render_spam_error! if snippet.spam?
+
+ render_api_error!({ error: service_response.message }, service_response.http_status)
end
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -155,7 +149,7 @@ module API
env['api.format'] = :txt
content_type 'text/plain'
- present snippet.content
+ present content_for(snippet)
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb
index 119902a189c..cfcc7f5212d 100644
--- a/lib/api/project_templates.rb
+++ b/lib/api/project_templates.rb
@@ -5,6 +5,10 @@ module API
include PaginationParams
TEMPLATE_TYPES = %w[dockerfiles gitignores gitlab_ci_ymls licenses].freeze
+ # The regex is needed to ensure a period (e.g. agpl-3.0)
+ # isn't confused with a format type. We also need to allow encoded
+ # values (e.g. C%2B%2B for C++), so allow % and + as well.
+ TEMPLATE_NAMES_ENDPOINT_REQUIREMENTS = API::NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(name: /[\w%.+-]+/)
before { authenticate_non_get! }
@@ -12,7 +16,7 @@ module API
requires :id, type: String, desc: 'The ID of a project'
requires :type, type: String, values: TEMPLATE_TYPES, desc: 'The type (dockerfiles|gitignores|gitlab_ci_ymls|licenses) of the template'
end
- resource :projects do
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
desc 'Get a list of templates available to this project' do
detail 'This endpoint was introduced in GitLab 11.4'
end
@@ -36,10 +40,8 @@ module API
optional :project, type: String, desc: 'The project name to use when expanding placeholders in the template. Only affects licenses'
optional :fullname, type: String, desc: 'The full name of the copyright holder to use when expanding placeholders in the template. Only affects licenses'
end
- # The regex is needed to ensure a period (e.g. agpl-3.0)
- # isn't confused with a format type. We also need to allow encoded
- # values (e.g. C%2B%2B for C++), so allow % and + as well.
- get ':id/templates/:type/:name', requirements: { name: /[\w%.+-]+/ } do
+
+ get ':id/templates/:type/:name', requirements: TEMPLATE_NAMES_ENDPOINT_REQUIREMENTS do
template = TemplateFinder
.build(params[:type], user_project, name: params[:name])
.execute
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index ee0731a331f..732453cf1c4 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -95,7 +95,7 @@ module API
projects = reorder_projects(projects)
projects = apply_filters(projects)
- records, options = paginate_with_strategies(projects) do |projects|
+ records, options = paginate_with_strategies(projects, options[:request_scope]) do |projects|
projects, options = with_custom_attributes(projects, options)
options = options.reverse_merge(
@@ -313,7 +313,7 @@ module API
get ':id/forks' do
forks = ForkProjectsFinder.new(user_project, params: project_finder_params, current_user: current_user).execute
- present_projects forks
+ present_projects forks, request_scope: user_project
end
desc 'Check pages access of this project'
diff --git a/lib/api/remote_mirrors.rb b/lib/api/remote_mirrors.rb
index 7e484eb8885..0808541d3c7 100644
--- a/lib/api/remote_mirrors.rb
+++ b/lib/api/remote_mirrors.rb
@@ -34,7 +34,6 @@ module API
end
post ':id/remote_mirrors' do
create_params = declared_params(include_missing: false)
- create_params.delete(:keep_divergent_refs) unless ::Feature.enabled?(:keep_divergent_refs, user_project)
new_mirror = user_project.remote_mirrors.create(create_params)
@@ -59,7 +58,6 @@ module API
mirror_params = declared_params(include_missing: false)
mirror_params[:id] = mirror_params.delete(:mirror_id)
- mirror_params.delete(:keep_divergent_refs) unless ::Feature.enabled?(:keep_divergent_refs, user_project)
update_params = { remote_mirrors_attributes: mirror_params }
diff --git a/lib/api/search.rb b/lib/api/search.rb
index ed52a4fc8f2..3d2d4527e30 100644
--- a/lib/api/search.rb
+++ b/lib/api/search.rb
@@ -17,7 +17,6 @@ module API
blobs: Entities::Blob,
wiki_blobs: Entities::Blob,
snippet_titles: Entities::Snippet,
- snippet_blobs: Entities::Snippet,
users: Entities::UserBasic
}.freeze
@@ -36,7 +35,7 @@ module API
end
def snippets?
- %w(snippet_blobs snippet_titles).include?(params[:scope]).to_s
+ %w(snippet_titles).include?(params[:scope]).to_s
end
def entity
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 5362b3060c1..e3a8f0671ef 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -84,16 +84,7 @@ module API
optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts"
optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB'
optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB'
- optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics'
- given metrics_enabled: ->(val) { val } do
- requires :metrics_host, type: String, desc: 'The InfluxDB host'
- requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.'
- requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet'
- requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open'
- requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB'
- requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds'
- requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out'
- end
+ optional :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.'
optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5
optional :password_authentication_enabled_for_web, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface'
mutually_exclusive :password_authentication_enabled_for_web, :password_authentication_enabled, :signin_enabled
@@ -153,6 +144,8 @@ module API
optional :snowplow_cookie_domain, type: String, desc: 'The Snowplow cookie domain'
optional :snowplow_app_id, type: String, desc: 'The Snowplow site name / application id'
end
+ optional :issues_create_limit, type: Integer, desc: "Maximum number of issue creation requests allowed per minute per user. Set to 0 for unlimited requests per minute."
+ optional :raw_blob_request_limit, type: Integer, desc: "Maximum number of requests per minute for each raw path. Set to 0 for unlimited requests per minute."
ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type|
optional :"#{type}_key_restriction",
@@ -192,6 +185,9 @@ module API
attrs[:allow_local_requests_from_web_hooks_and_services] = attrs.delete(:allow_local_requests_from_hooks_and_services)
end
+ # since 13.0 it's not possible to disable hashed storage - support can be removed in 14.0
+ attrs.delete(:hashed_storage_enabled) if attrs.has_key?(:hashed_storage_enabled)
+
attrs = filter_attributes_using_license(attrs)
if ApplicationSettings::UpdateService.new(current_settings, current_user, attrs).execute
diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb
index 0aaab9a812f..be58b832f97 100644
--- a/lib/api/snippets.rb
+++ b/lib/api/snippets.rb
@@ -8,6 +8,7 @@ module API
before { authenticate! }
resource :snippets do
+ helpers Helpers::SnippetsHelpers
helpers do
def snippets_for_current_user
SnippetsFinder.new(current_user, author: current_user).execute
@@ -24,13 +25,13 @@ module API
desc 'Get a snippets list for authenticated user' do
detail 'This feature was introduced in GitLab 8.15.'
- success Entities::PersonalSnippet
+ success Entities::Snippet
end
params do
use :pagination
end
get do
- present paginate(snippets_for_current_user), with: Entities::PersonalSnippet
+ present paginate(snippets_for_current_user), with: Entities::Snippet
end
desc 'List all public personal snippets current_user has access to' do
@@ -64,9 +65,9 @@ module API
success Entities::PersonalSnippet
end
params do
- requires :title, type: String, desc: 'The title of a snippet'
+ requires :title, type: String, allow_blank: false, desc: 'The title of a snippet'
requires :file_name, type: String, desc: 'The name of a snippet file'
- requires :content, type: String, desc: 'The content of a snippet'
+ requires :content, type: String, allow_blank: false, desc: 'The content of a snippet'
optional :description, type: String, desc: 'The description of a snippet'
optional :visibility, type: String,
values: Gitlab::VisibilityLevel.string_values,
@@ -80,12 +81,12 @@ module API
service_response = ::Snippets::CreateService.new(nil, current_user, attrs).execute
snippet = service_response.payload[:snippet]
- render_spam_error! if snippet.spam?
-
- if snippet.persisted?
+ if service_response.success?
present snippet, with: Entities::PersonalSnippet
else
- render_validation_error!(snippet)
+ render_spam_error! if snippet.spam?
+
+ render_api_error!({ error: service_response.message }, service_response.http_status)
end
end
@@ -95,9 +96,9 @@ module API
end
params do
requires :id, type: Integer, desc: 'The ID of a snippet'
- optional :title, type: String, desc: 'The title of a snippet'
+ optional :title, type: String, allow_blank: false, desc: 'The title of a snippet'
optional :file_name, type: String, desc: 'The name of a snippet file'
- optional :content, type: String, desc: 'The content of a snippet'
+ optional :content, type: String, allow_blank: false, desc: 'The content of a snippet'
optional :description, type: String, desc: 'The description of a snippet'
optional :visibility, type: String,
values: Gitlab::VisibilityLevel.string_values,
@@ -114,12 +115,12 @@ module API
service_response = ::Snippets::UpdateService.new(nil, current_user, attrs).execute(snippet)
snippet = service_response.payload[:snippet]
- render_spam_error! if snippet.spam?
-
- if snippet.persisted?
+ if service_response.success?
present snippet, with: Entities::PersonalSnippet
else
- render_validation_error!(snippet)
+ render_spam_error! if snippet.spam?
+
+ render_api_error!({ error: service_response.message }, service_response.http_status)
end
end
@@ -159,7 +160,7 @@ module API
env['api.format'] = :txt
content_type 'text/plain'
header['Content-Disposition'] = 'attachment'
- present snippet.content
+ present content_for(snippet)
end
desc 'Get the user agent details for a snippet' do
diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb
index a2146406690..884e3019a2d 100644
--- a/lib/api/wikis.rb
+++ b/lib/api/wikis.rb
@@ -70,7 +70,7 @@ module API
post ':id/wikis' do
authorize! :create_wiki, user_project
- page = WikiPages::CreateService.new(user_project, current_user, params).execute
+ page = WikiPages::CreateService.new(container: user_project, current_user: current_user, params: params).execute
if page.valid?
present page, with: Entities::WikiPage
@@ -91,7 +91,7 @@ module API
put ':id/wikis/:slug' do
authorize! :create_wiki, user_project
- page = WikiPages::UpdateService.new(user_project, current_user, params).execute(wiki_page)
+ page = WikiPages::UpdateService.new(container: user_project, current_user: current_user, params: params).execute(wiki_page)
if page.valid?
present page, with: Entities::WikiPage
@@ -107,7 +107,7 @@ module API
delete ':id/wikis/:slug' do
authorize! :admin_wiki, user_project
- WikiPages::DestroyService.new(user_project, current_user).execute(wiki_page)
+ WikiPages::DestroyService.new(container: user_project, current_user: current_user).execute(wiki_page)
no_content!
end
@@ -123,9 +123,11 @@ module API
post ":id/wikis/attachments" do
authorize! :create_wiki, user_project
- result = ::Wikis::CreateAttachmentService.new(user_project,
- current_user,
- commit_params(declared_params(include_missing: false))).execute
+ result = ::Wikis::CreateAttachmentService.new(
+ container: user_project,
+ current_user: current_user,
+ params: commit_params(declared_params(include_missing: false))
+ ).execute
if result[:status] == :success
status(201)
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 09a4d71b5f6..37e66387f2e 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -28,8 +28,24 @@ module Banzai
def parent_records(parent, ids)
parent.issues.where(iid: ids.to_a)
end
+
+ def object_link_text_extras(issue, matches)
+ super + design_link_extras(issue, matches.named_captures['path'])
+ end
+
+ private
+
+ def design_link_extras(issue, path)
+ if path == '/designs' && read_designs?(issue)
+ ['designs']
+ else
+ []
+ end
+ end
+
+ def read_designs?(issue)
+ Ability.allowed?(current_user, :read_design, issue)
+ end
end
end
end
-
-Banzai::Filter::IssueReferenceFilter.prepend_if_ee('EE::Banzai::Filter::IssueReferenceFilter')
diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb
index 023c1288367..762371e1418 100644
--- a/lib/banzai/filter/upload_link_filter.rb
+++ b/lib/banzai/filter/upload_link_filter.rb
@@ -50,6 +50,10 @@ module Banzai
Addressable::URI.join(Gitlab.config.gitlab.base_url, path).to_s
end
+ if html_attr.name == 'href'
+ html_attr.parent.set_attribute('data-link', 'true')
+ end
+
html_attr.parent.add_class('gfm')
end
diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb
index 8cda67867a8..9268ff1a827 100644
--- a/lib/banzai/filter/user_reference_filter.rb
+++ b/lib/banzai/filter/user_reference_filter.rb
@@ -106,7 +106,7 @@ module Banzai
end
def link_class
- reference_class(:project_member, tooltip: false)
+ [reference_class(:project_member, tooltip: false), "js-user-link"].join(" ")
end
def link_to_all(link_content: nil)
diff --git a/lib/banzai/pipeline.rb b/lib/banzai/pipeline.rb
index 8fdbc044861..01cadb11e83 100644
--- a/lib/banzai/pipeline.rb
+++ b/lib/banzai/pipeline.rb
@@ -9,7 +9,7 @@ module Banzai
# Examples:
# Pipeline[nil] # => Banzai::Pipeline::FullPipeline
# Pipeline[:label] # => Banzai::Pipeline::LabelPipeline
- # Pipeline[StatusPage::PostProcessPipeline] # => StatusPage::PostProcessPipeline
+ # Pipeline[StatusPage::Pipeline::PostProcessPipeline] # => StatusPage::Pipeline::PostProcessPipeline
#
# Pipeline['label'] # => raises ArgumentError - unsupport type
# Pipeline[Project] # => raises ArgumentError - not a subclass of BasePipeline
diff --git a/lib/banzai/reference_parser/design_parser.rb b/lib/banzai/reference_parser/design_parser.rb
new file mode 100644
index 00000000000..04e878756d8
--- /dev/null
+++ b/lib/banzai/reference_parser/design_parser.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Banzai
+ module ReferenceParser
+ class DesignParser < BaseParser
+ self.reference_type = :design
+
+ def references_relation
+ DesignManagement::Design
+ end
+
+ def nodes_visible_to_user(user, nodes)
+ issues = issues_for_nodes(nodes)
+ issue_attr = 'data-issue'
+
+ nodes.select do |node|
+ if node.has_attribute?(issue_attr)
+ can?(user, :read_design, issues[node])
+ else
+ true
+ end
+ end
+ end
+
+ def issues_for_nodes(nodes)
+ relation = Issue.includes(project: [:project_feature])
+ grouped_objects_for_nodes(nodes, relation, 'data-issue')
+ end
+ end
+ end
+end
diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb
index 3cb9ec21e8f..fbbd6135959 100644
--- a/lib/banzai/renderer.rb
+++ b/lib/banzai/renderer.rb
@@ -138,15 +138,18 @@ module Banzai
#
# html - String to process
# context - Hash of options to customize output
- # :pipeline - Symbol pipeline type
+ # :pipeline - Symbol pipeline type - for context transform only, defaults to :full
# :project - Project
# :user - User object
+ # :post_process_pipeline - pipeline to use for post_processing - defaults to PostProcessPipeline
#
# Returns an HTML-safe String
def self.post_process(html, context)
context = Pipeline[context[:pipeline]].transform_context(context)
- pipeline = Pipeline[:post_process]
+ # Use a passed class for the pipeline or default to PostProcessPipeline
+ pipeline = context.delete(:post_process_pipeline) || ::Banzai::Pipeline::PostProcessPipeline
+
if context[:xhtml]
pipeline.to_document(html, context).to_html(save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML)
else
diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb
index 56f556c229a..118eb8e2d7c 100644
--- a/lib/container_registry/client.rb
+++ b/lib/container_registry/client.rb
@@ -13,6 +13,8 @@ module ContainerRegistry
DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE = 'application/vnd.docker.distribution.manifest.v2+json'
OCI_MANIFEST_V1_TYPE = 'application/vnd.oci.image.manifest.v1+json'
CONTAINER_IMAGE_V1_TYPE = 'application/vnd.docker.container.image.v1+json'
+ REGISTRY_VERSION_HEADER = 'gitlab-container-registry-version'
+ REGISTRY_FEATURES_HEADER = 'gitlab-container-registry-features'
ACCEPTED_TYPES = [DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE, OCI_MANIFEST_V1_TYPE].freeze
@@ -24,6 +26,21 @@ module ContainerRegistry
@options = options
end
+ def registry_info
+ response = faraday.get("/v2/")
+
+ return {} unless response&.success?
+
+ version = response.headers[REGISTRY_VERSION_HEADER]
+ features = response.headers.fetch(REGISTRY_FEATURES_HEADER, '')
+
+ {
+ version: version,
+ features: features.split(',').map(&:strip),
+ vendor: version ? 'gitlab' : 'other'
+ }
+ end
+
def repository_tags(name)
response_body faraday.get("/v2/#{name}/tags/list")
end
@@ -83,7 +100,7 @@ module ContainerRegistry
image = {
config: {}
}
- image, image_digest = upload_raw_blob(path, JSON.pretty_generate(image))
+ image, image_digest = upload_raw_blob(path, Gitlab::Json.pretty_generate(image))
return unless image
{
@@ -109,7 +126,7 @@ module ContainerRegistry
def put_tag(name, reference, manifest)
response = faraday.put("/v2/#{name}/manifests/#{reference}") do |req|
req.headers['Content-Type'] = DOCKER_DISTRIBUTION_MANIFEST_V2_TYPE
- req.body = JSON.pretty_generate(manifest)
+ req.body = Gitlab::Json.pretty_generate(manifest)
end
response.headers['docker-content-digest'] if response.success?
diff --git a/lib/container_registry/config.rb b/lib/container_registry/config.rb
index 740c0e13da0..40dd92befd2 100644
--- a/lib/container_registry/config.rb
+++ b/lib/container_registry/config.rb
@@ -6,7 +6,7 @@ module ContainerRegistry
def initialize(tag, blob)
@tag, @blob = tag, blob
- @data = JSON.parse(blob.data)
+ @data = Gitlab::Json.parse(blob.data)
end
def [](key)
diff --git a/lib/csv_builder.rb b/lib/csv_builder.rb
index 7df4e3bf85d..a9ef5a83ae8 100644
--- a/lib/csv_builder.rb
+++ b/lib/csv_builder.rb
@@ -14,6 +14,9 @@
# CsvBuilder.new(@posts, columns).render
#
class CsvBuilder
+ DEFAULT_ORDER_BY = 'id'.freeze
+ DEFAULT_BATCH_SIZE = 1000
+
attr_reader :rows_written
#
@@ -68,6 +71,12 @@ class CsvBuilder
}
end
+ protected
+
+ def each(&block)
+ @collection.find_each(&block) # rubocop: disable CodeReuse/ActiveRecord
+ end
+
private
def headers
@@ -91,7 +100,7 @@ class CsvBuilder
def write_csv(csv, until_condition:)
csv << headers
- @collection.find_each do |object|
+ each do |object|
csv << row(object)
@rows_written += 1
diff --git a/lib/csv_builders/single_batch.rb b/lib/csv_builders/single_batch.rb
new file mode 100644
index 00000000000..bed6b7424b3
--- /dev/null
+++ b/lib/csv_builders/single_batch.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module CsvBuilders
+ class SingleBatch < CsvBuilder
+ protected
+
+ def each(&block)
+ @collection.each(&block)
+ end
+ end
+end
diff --git a/lib/declarative_policy.rb b/lib/declarative_policy.rb
index e51f30af581..bd1c121fe79 100644
--- a/lib/declarative_policy.rb
+++ b/lib/declarative_policy.rb
@@ -72,18 +72,17 @@ module DeclarativePolicy
end
def compute_class_for_class(subject_class)
+ if subject_class.respond_to?(:declarative_policy_class)
+ return subject_class.declarative_policy_class.constantize
+ end
+
subject_class.ancestors.each do |klass|
- next unless klass.name
+ name = klass.name
+
+ next unless name
begin
- klass_name =
- if subject_class.respond_to?(:declarative_policy_class)
- subject_class.declarative_policy_class
- else
- "#{klass.name}Policy"
- end
-
- policy_class = klass_name.constantize
+ policy_class = "#{name}Policy".constantize
# NOTE: the < operator here tests whether policy_class
# inherits from Base. We can't use #is_a? because that
diff --git a/lib/feature.rb b/lib/feature.rb
index 60a5c03a839..dc7e8da8f35 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -134,11 +134,7 @@ class Feature
end
def l1_cache_backend
- if Gitlab::Utils.to_boolean(ENV['USE_THREAD_MEMORY_CACHE'])
- Gitlab::ThreadMemoryCache.cache_backend
- else
- Gitlab::ProcessMemoryCache.cache_backend
- end
+ Gitlab::ProcessMemoryCache.cache_backend
end
def l2_cache_backend
diff --git a/lib/gitlab/alert_management/alert_params.rb b/lib/gitlab/alert_management/alert_params.rb
new file mode 100644
index 00000000000..982479784a9
--- /dev/null
+++ b/lib/gitlab/alert_management/alert_params.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module AlertManagement
+ class AlertParams
+ MONITORING_TOOLS = {
+ prometheus: 'Prometheus'
+ }.freeze
+
+ def self.from_generic_alert(project:, payload:)
+ parsed_payload = Gitlab::Alerting::NotificationPayloadParser.call(payload).with_indifferent_access
+ annotations = parsed_payload[:annotations]
+
+ {
+ project_id: project.id,
+ title: annotations[:title],
+ description: annotations[:description],
+ monitoring_tool: annotations[:monitoring_tool],
+ service: annotations[:service],
+ hosts: Array(annotations[:hosts]),
+ payload: payload,
+ started_at: parsed_payload['startsAt'],
+ severity: annotations[:severity]
+ }
+ end
+
+ def self.from_prometheus_alert(project:, parsed_alert:)
+ {
+ project_id: project.id,
+ title: parsed_alert.title,
+ description: parsed_alert.description,
+ monitoring_tool: MONITORING_TOOLS[:prometheus],
+ payload: parsed_alert.payload,
+ started_at: parsed_alert.starts_at,
+ ended_at: parsed_alert.ends_at,
+ fingerprint: parsed_alert.gitlab_fingerprint
+ }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/alert_management/alert_status_counts.rb b/lib/gitlab/alert_management/alert_status_counts.rb
new file mode 100644
index 00000000000..382026236e0
--- /dev/null
+++ b/lib/gitlab/alert_management/alert_status_counts.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module AlertManagement
+ # Represents counts of each status or category of statuses
+ class AlertStatusCounts
+ include Gitlab::Utils::StrongMemoize
+
+ STATUSES = ::AlertManagement::Alert::STATUSES
+
+ attr_reader :project
+
+ def self.declarative_policy_class
+ 'AlertManagement::AlertPolicy'
+ end
+
+ def initialize(current_user, project, params)
+ @project = project
+ @current_user = current_user
+ @params = params
+ end
+
+ # Define method for each status
+ STATUSES.each_key do |status|
+ define_method(status) { counts[status] }
+ end
+
+ def open
+ counts[:triggered] + counts[:acknowledged]
+ end
+
+ def all
+ counts.values.sum # rubocop:disable CodeReuse/ActiveRecord
+ end
+
+ private
+
+ attr_reader :current_user, :params
+
+ def counts
+ strong_memoize(:counts) do
+ Hash.new(0).merge(counts_by_status)
+ end
+ end
+
+ def counts_by_status
+ ::AlertManagement::AlertsFinder
+ .counts_by_status(current_user, project, params)
+ .transform_keys { |status_id| STATUSES.key(status_id) }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb
index 7d97bd1bb52..d859ca89418 100644
--- a/lib/gitlab/alerting/alert.rb
+++ b/lib/gitlab/alerting/alert.rb
@@ -105,6 +105,10 @@ module Gitlab
metric_id.present?
end
+ def gitlab_fingerprint
+ Digest::SHA1.hexdigest(plain_gitlab_fingerprint)
+ end
+
def valid?
payload.respond_to?(:dig) && project && title && starts_at
end
@@ -115,6 +119,14 @@ module Gitlab
private
+ def plain_gitlab_fingerprint
+ if gitlab_managed?
+ [metric_id, starts_at].join('/')
+ else # self managed
+ [starts_at, title, full_query].join('/')
+ end
+ end
+
def parse_environment_from_payload
environment_name = payload&.dig('labels', 'gitlab_environment_name')
diff --git a/lib/gitlab/alerting/notification_payload_parser.rb b/lib/gitlab/alerting/notification_payload_parser.rb
index a54bb44d66a..c79d69613f3 100644
--- a/lib/gitlab/alerting/notification_payload_parser.rb
+++ b/lib/gitlab/alerting/notification_payload_parser.rb
@@ -6,6 +6,7 @@ module Gitlab
BadPayloadError = Class.new(StandardError)
DEFAULT_TITLE = 'New: Incident'
+ DEFAULT_SEVERITY = 'critical'
def initialize(payload)
@payload = payload.to_h.with_indifferent_access
@@ -30,6 +31,10 @@ module Gitlab
payload[:title].presence || DEFAULT_TITLE
end
+ def severity
+ payload[:severity].presence || DEFAULT_SEVERITY
+ end
+
def annotations
primary_params
.reverse_merge(flatten_secondary_params)
@@ -43,7 +48,8 @@ module Gitlab
'description' => payload[:description],
'monitoring_tool' => payload[:monitoring_tool],
'service' => payload[:service],
- 'hosts' => hosts.presence
+ 'hosts' => hosts.presence,
+ 'severity' => severity
}
end
diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb
index 41883a80338..9fcaeadf351 100644
--- a/lib/gitlab/analytics/cycle_analytics/median.rb
+++ b/lib/gitlab/analytics/cycle_analytics/median.rb
@@ -15,7 +15,11 @@ module Gitlab
@query = @query.select(median_duration_in_seconds.as('median'))
result = execute_query(@query).first || {}
- result['median'] ? result['median'].to_i : nil
+ result['median'] || nil
+ end
+
+ def days
+ seconds ? seconds.fdiv(1.day) : nil
end
private
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index 8e14d21f591..44e8c9c04b9 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -337,6 +337,10 @@ module Gitlab
REGISTRY_SCOPES
end
+ def resource_bot_scopes
+ Gitlab::Auth::API_SCOPES + Gitlab::Auth::REPOSITORY_SCOPES + Gitlab::Auth.registry_scopes - [:read_user]
+ end
+
private
def non_admin_available_scopes
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index f0ca6491bd0..b7e78189d37 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -18,8 +18,6 @@ module Gitlab
end
module AuthFinders
- prepend_if_ee('::EE::Gitlab::Auth::AuthFinders') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
include Gitlab::Utils::StrongMemoize
include ActionController::HttpAuthentication::Basic
@@ -27,6 +25,7 @@ module Gitlab
PRIVATE_TOKEN_PARAM = :private_token
JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze
JOB_TOKEN_PARAM = :job_token
+ DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN'.freeze
RUNNER_TOKEN_PARAM = :token
RUNNER_JOB_TOKEN_PARAM = :token
@@ -103,6 +102,25 @@ module Gitlab
access_token.user || raise(UnauthorizedError)
end
+ # This returns a deploy token, not a user since a deploy token does not
+ # belong to a user.
+ #
+ # deploy tokens are accepted with deploy token headers and basic auth headers
+ def deploy_token_from_request
+ return unless route_authentication_setting[:deploy_token_allowed]
+
+ token = current_request.env[DEPLOY_TOKEN_HEADER].presence || parsed_oauth_token
+
+ if has_basic_credentials?(current_request)
+ _, token = user_name_and_password(current_request)
+ end
+
+ deploy_token = DeployToken.active.find_by_token(token)
+ @current_authenticated_deploy_token = deploy_token # rubocop:disable Gitlab/ModuleWithInstanceVariables
+
+ deploy_token
+ end
+
def find_runner_from_token
return unless api_request?
@@ -113,6 +131,9 @@ module Gitlab
end
def validate_access_token!(scopes: [])
+ # return early if we've already authenticated via a deploy token
+ return if @current_authenticated_deploy_token.present? # rubocop:disable Gitlab/ModuleWithInstanceVariables
+
return unless access_token
case AccessTokenValidationService.new(access_token, request: request).validate(scopes: scopes)
@@ -249,3 +270,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::AuthFinders.prepend_if_ee('::EE::Gitlab::Auth::AuthFinders')
diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb
index 98eec0e4a7b..66d20ee2b59 100644
--- a/lib/gitlab/auth/ldap/access.rb
+++ b/lib/gitlab/auth/ldap/access.rb
@@ -8,8 +8,6 @@ module Gitlab
module Auth
module Ldap
class Access
- prepend_if_ee('::EE::Gitlab::Auth::Ldap::Access') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
attr_reader :provider, :user, :ldap_identity
def self.open(user, &block)
@@ -118,3 +116,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::Ldap::Access.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Access')
diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb
index c5ec4e1981b..f64fcd822c6 100644
--- a/lib/gitlab/auth/ldap/adapter.rb
+++ b/lib/gitlab/auth/ldap/adapter.rb
@@ -4,8 +4,6 @@ module Gitlab
module Auth
module Ldap
class Adapter
- prepend_if_ee('::EE::Gitlab::Auth::Ldap::Adapter') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze
MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze
@@ -142,3 +140,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::Ldap::Adapter.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Adapter')
diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb
index b8874e18a0b..7677189eb9f 100644
--- a/lib/gitlab/auth/ldap/config.rb
+++ b/lib/gitlab/auth/ldap/config.rb
@@ -5,8 +5,6 @@ module Gitlab
module Auth
module Ldap
class Config
- prepend_if_ee('::EE::Gitlab::Auth::Ldap::Config') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
NET_LDAP_ENCRYPTION_METHOD = {
simple_tls: :simple_tls,
start_tls: :start_tls,
@@ -288,3 +286,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::Ldap::Config.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Config')
diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb
index 430f94a9a28..e4a4900c37a 100644
--- a/lib/gitlab/auth/ldap/person.rb
+++ b/lib/gitlab/auth/ldap/person.rb
@@ -4,8 +4,6 @@ module Gitlab
module Auth
module Ldap
class Person
- prepend_if_ee('::EE::Gitlab::Auth::Ldap::Person') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
# Active Directory-specific LDAP filter that checks if bit 2 of the
# userAccountControl attribute is set.
# Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/
@@ -122,3 +120,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::Ldap::Person.prepend_if_ee('::EE::Gitlab::Auth::Ldap::Person')
diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb
index df14e5fc3dc..1405fb4ab95 100644
--- a/lib/gitlab/auth/ldap/user.rb
+++ b/lib/gitlab/auth/ldap/user.rb
@@ -11,8 +11,6 @@ module Gitlab
module Ldap
class User < Gitlab::Auth::OAuth::User
extend ::Gitlab::Utils::Override
- prepend_if_ee('::EE::Gitlab::Auth::Ldap::User') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
class << self
# rubocop: disable CodeReuse/ActiveRecord
def find_by_uid_and_provider(uid, provider)
@@ -64,3 +62,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::Ldap::User.prepend_if_ee('::EE::Gitlab::Auth::Ldap::User')
diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb
index b37a9225dd7..46ff6b2ccab 100644
--- a/lib/gitlab/auth/o_auth/auth_hash.rb
+++ b/lib/gitlab/auth/o_auth/auth_hash.rb
@@ -6,8 +6,6 @@ module Gitlab
module Auth
module OAuth
class AuthHash
- prepend_if_ee('::EE::Gitlab::Auth::OAuth::AuthHash') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
attr_reader :auth_hash
def initialize(auth_hash)
@auth_hash = auth_hash
@@ -93,3 +91,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::OAuth::AuthHash.prepend_if_ee('::EE::Gitlab::Auth::OAuth::AuthHash')
diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb
index f0811098b15..6d699d37a8c 100644
--- a/lib/gitlab/auth/o_auth/provider.rb
+++ b/lib/gitlab/auth/o_auth/provider.rb
@@ -66,7 +66,10 @@ module Gitlab
nil
end
else
- Gitlab.config.omniauth.providers.find { |provider| provider.name == name }
+ provider = Gitlab.config.omniauth.providers.find { |provider| provider.name == name }
+ merge_provider_args_with_defaults!(provider)
+
+ provider
end
end
@@ -81,6 +84,15 @@ module Gitlab
config = config_for(name)
config && config['icon']
end
+
+ def self.merge_provider_args_with_defaults!(provider)
+ return unless provider
+
+ provider['args'] ||= {}
+
+ defaults = Gitlab::OmniauthInitializer.default_arguments_for(provider['name'])
+ provider['args'].deep_merge!(defaults.deep_stringify_keys)
+ end
end
end
end
diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb
index df595da1536..8a60d6ef482 100644
--- a/lib/gitlab/auth/o_auth/user.rb
+++ b/lib/gitlab/auth/o_auth/user.rb
@@ -9,8 +9,6 @@ module Gitlab
module Auth
module OAuth
class User
- prepend_if_ee('::EE::Gitlab::Auth::OAuth::User') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
SignupDisabledError = Class.new(StandardError)
SigninDisabledForProviderError = Class.new(StandardError)
@@ -275,3 +273,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::OAuth::User.prepend_if_ee('::EE::Gitlab::Auth::OAuth::User')
diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb
index 0fe91f9f3c8..757a0e671c3 100644
--- a/lib/gitlab/auth/result.rb
+++ b/lib/gitlab/auth/result.rb
@@ -3,8 +3,6 @@
module Gitlab
module Auth
Result = Struct.new(:actor, :project, :type, :authentication_abilities) do
- prepend_if_ee('::EE::Gitlab::Auth::Result') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
def ci?(for_project)
type == :ci &&
project &&
@@ -26,3 +24,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::Result.prepend_if_ee('::EE::Gitlab::Auth::Result')
diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb
index ed2f3f158c1..67a53fa3205 100644
--- a/lib/gitlab/auth/saml/config.rb
+++ b/lib/gitlab/auth/saml/config.rb
@@ -4,8 +4,6 @@ module Gitlab
module Auth
module Saml
class Config
- prepend_if_ee('::EE::Gitlab::Auth::Saml::Config') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
class << self
def options
Gitlab::Auth::OAuth::Provider.config_for('saml')
@@ -31,3 +29,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::Saml::Config.prepend_if_ee('::EE::Gitlab::Auth::Saml::Config')
diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb
index 1ba36ad95b4..37bc3f9bed0 100644
--- a/lib/gitlab/auth/saml/user.rb
+++ b/lib/gitlab/auth/saml/user.rb
@@ -9,8 +9,6 @@ module Gitlab
module Auth
module Saml
class User < Gitlab::Auth::OAuth::User
- prepend_if_ee('::EE::Gitlab::Auth::Saml::User') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
extend ::Gitlab::Utils::Override
def save
@@ -63,3 +61,5 @@ module Gitlab
end
end
end
+
+Gitlab::Auth::Saml::User.prepend_if_ee('::EE::Gitlab::Auth::Saml::User')
diff --git a/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb b/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb
new file mode 100644
index 00000000000..4fd3b81fda3
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_environment_id_deployment_merge_requests.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # BackfillEnvironmentIdDeploymentMergeRequests deletes duplicates
+ # from deployment_merge_requests table and backfills environment_id
+ class BackfillEnvironmentIdDeploymentMergeRequests
+ def perform(_start_mr_id, _stop_mr_id)
+ # no-op
+
+ # Background migration removed due to
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/217191
+ end
+
+ def backfill_range(start_mr_id, stop_mr_id)
+ start_mr_id = Integer(start_mr_id)
+ stop_mr_id = Integer(stop_mr_id)
+
+ ActiveRecord::Base.connection.execute(<<~SQL)
+ DELETE FROM deployment_merge_requests
+ WHERE (deployment_id, merge_request_id) in (
+ SELECT t.deployment_id, t.merge_request_id FROM (
+ SELECT mrd.merge_request_id, mrd.deployment_id, ROW_NUMBER() OVER w AS rnum
+ FROM deployment_merge_requests as mrd
+ INNER JOIN "deployments" ON "deployments"."id" = "mrd"."deployment_id"
+ WHERE mrd.merge_request_id BETWEEN #{start_mr_id} AND #{stop_mr_id}
+ WINDOW w AS (
+ PARTITION BY merge_request_id, deployments.environment_id
+ ORDER BY deployments.id
+ )
+ ) t
+ WHERE t.rnum > 1
+ );
+ SQL
+
+ ActiveRecord::Base.connection.execute(<<~SQL)
+ UPDATE deployment_merge_requests
+ SET environment_id = deployments.environment_id
+ FROM deployments
+ WHERE deployments.id = "deployment_merge_requests".deployment_id
+ AND "deployment_merge_requests".environment_id IS NULL
+ AND "deployment_merge_requests".merge_request_id BETWEEN #{start_mr_id} AND #{stop_mr_id}
+ SQL
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
index fa6453abefb..21538000fec 100644
--- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb
@@ -8,22 +8,41 @@ module Gitlab
MAX_RETRIES = 2
def perform(start_id, stop_id)
- Snippet.includes(:author, snippet_repository: :shard).where(id: start_id..stop_id).find_each do |snippet|
+ snippets = snippet_relation.where(id: start_id..stop_id)
+
+ migrate_snippets(snippets)
+ end
+
+ def perform_by_ids(snippet_ids)
+ snippets = snippet_relation.where(id: snippet_ids)
+
+ migrate_snippets(snippets)
+ end
+
+ private
+
+ def migrate_snippets(snippets)
+ snippets.find_each do |snippet|
# We need to expire the exists? value for the cached method in case it was cached
snippet.repository.expire_exists_cache
next if repository_present?(snippet)
retry_index = 0
+ @invalid_path_error = false
+ @invalid_signature_error = false
begin
create_repository_and_files(snippet)
logger.info(message: 'Snippet Migration: repository created and migrated', snippet: snippet.id)
rescue => e
+ set_file_path_error(e)
+ set_signature_error(e)
+
retry_index += 1
- retry if retry_index < MAX_RETRIES
+ retry if retry_index < max_retries
logger.error(message: "Snippet Migration: error migrating snippet. Reason: #{e.message}", snippet: snippet.id)
@@ -33,7 +52,9 @@ module Gitlab
end
end
- private
+ def snippet_relation
+ @snippet_relation ||= Snippet.includes(:author, snippet_repository: :shard)
+ end
def repository_present?(snippet)
snippet.snippet_repository && !snippet.empty_repo?
@@ -44,16 +65,19 @@ module Gitlab
create_commit(snippet)
end
+ # Removing the db record
def destroy_snippet_repository(snippet)
- # Removing the db record
- snippet.snippet_repository&.destroy
+ snippet.snippet_repository&.delete
rescue => e
logger.error(message: "Snippet Migration: error destroying snippet repository. Reason: #{e.message}", snippet: snippet.id)
end
+ # Removing the repository in disk
def delete_repository(snippet)
- # Removing the repository in disk
- snippet.repository.remove if snippet.repository_exists?
+ return unless snippet.repository_exists?
+
+ snippet.repository.remove
+ snippet.repository.expire_exists_cache
rescue => e
logger.error(message: "Snippet Migration: error deleting repository. Reason: #{e.message}", snippet: snippet.id)
end
@@ -70,7 +94,10 @@ module Gitlab
end
def filename(snippet)
- snippet.file_name.presence || empty_file_name
+ file_name = snippet.file_name
+ file_name = file_name.parameterize if @invalid_path_error
+
+ file_name.presence || empty_file_name
end
def empty_file_name
@@ -82,7 +109,56 @@ module Gitlab
end
def create_commit(snippet)
- snippet.snippet_repository.multi_files_action(snippet.author, snippet_action(snippet), commit_attrs)
+ snippet.snippet_repository.multi_files_action(commit_author(snippet), snippet_action(snippet), commit_attrs)
+ end
+
+ # If the user is not allowed to access git or update the snippet
+ # because it is blocked, internal, ghost, ... we cannot commit
+ # files because these users are not allowed to, but we need to
+ # migrate their snippets as well.
+ # In this scenario the migration bot user will be the one that will commit the files.
+ def commit_author(snippet)
+ return migration_bot_user if snippet_content_size_over_limit?(snippet)
+ return migration_bot_user if @invalid_signature_error
+
+ if Gitlab::UserAccessSnippet.new(snippet.author, snippet: snippet).can_do_action?(:update_snippet)
+ snippet.author
+ else
+ migration_bot_user
+ end
+ end
+
+ def migration_bot_user
+ @migration_bot_user ||= User.migration_bot
+ end
+
+ # We sometimes receive invalid path errors from Gitaly if the Snippet filename
+ # cannot be parsed into a valid git path.
+ # In this situation, we need to parameterize the file name of the Snippet so that
+ # the migration can succeed, to achieve that, we'll identify in migration retries
+ # that the path is invalid
+ def set_file_path_error(error)
+ @invalid_path_error ||= error.is_a?(SnippetRepository::InvalidPathError)
+ end
+
+ # We sometimes receive invalid signature from Gitaly if the commit author
+ # name or email is invalid to create the commit signature.
+ # In this situation, we set the error and use the migration_bot since
+ # the information used to build it is valid
+ def set_signature_error(error)
+ @invalid_signature_error ||= error.is_a?(SnippetRepository::InvalidSignatureError)
+ end
+
+ # In the case where the snippet file_name is invalid and also the
+ # snippet author has invalid commit info, we need to increase the
+ # number of retries by 1, because we will receive two errors
+ # from Gitaly and, in the third one, we will commit successfully.
+ def max_retries
+ MAX_RETRIES + (@invalid_signature_error && @invalid_path_error ? 1 : 0)
+ end
+
+ def snippet_content_size_over_limit?(snippet)
+ snippet.content.size > Gitlab::CurrentSettings.snippet_size_limit
end
end
end
diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
index 14e14f28439..956f9daa493 100644
--- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
+++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
@@ -79,7 +79,7 @@ module Gitlab
data = { 'jira_tracker_data' => [], 'issue_tracker_data' => [] }
select_all(query).each do |service|
begin
- properties = JSON.parse(service['properties'])
+ properties = Gitlab::Json.parse(service['properties'])
rescue JSON::ParserError
logger.warn(
message: 'Properties data not parsed - invalid json',
diff --git a/lib/gitlab/background_migration/populate_user_highest_roles_table.rb b/lib/gitlab/background_migration/populate_user_highest_roles_table.rb
index 0c9e15b5a80..16386ebf9c3 100644
--- a/lib/gitlab/background_migration/populate_user_highest_roles_table.rb
+++ b/lib/gitlab/background_migration/populate_user_highest_roles_table.rb
@@ -20,6 +20,8 @@ module Gitlab
end
def perform(from_id, to_id)
+ return unless User.column_names.include?('bot_type')
+
(from_id..to_id).each_slice(BATCH_SIZE) do |ids|
execute(
<<-EOF
diff --git a/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb
new file mode 100644
index 00000000000..3920e8dc2de
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+ module BackgroundMigration
+ class RemoveUndefinedOccurrenceConfidenceLevel
+ def perform(start_id, stop_id)
+ end
+ end
+ end
+end
+
+Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel')
diff --git a/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb
new file mode 100644
index 00000000000..f6ea61f4502
--- /dev/null
+++ b/lib/gitlab/background_migration/remove_undefined_vulnerability_confidence_level.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+ module BackgroundMigration
+ class RemoveUndefinedVulnerabilityConfidenceLevel
+ def perform(start_id, stop_id)
+ end
+ end
+ end
+end
+
+Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedVulnerabilityConfidenceLevel')
diff --git a/lib/gitlab/blob_helper.rb b/lib/gitlab/blob_helper.rb
index fc579ad8d2a..57d632afd74 100644
--- a/lib/gitlab/blob_helper.rb
+++ b/lib/gitlab/blob_helper.rb
@@ -3,6 +3,8 @@
# This has been extracted from https://github.com/github/linguist/blob/master/lib/linguist/blob_helper.rb
module Gitlab
module BlobHelper
+ include Gitlab::Utils::StrongMemoize
+
def extname
File.extname(name.to_s)
end
@@ -120,8 +122,18 @@ module Gitlab
end
def encoded_newlines_re
- @encoded_newlines_re ||=
- Regexp.union(["\r\n", "\r", "\n"].map { |nl| nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data.encoding) })
+ strong_memoize(:encoded_newlines_re) do
+ newlines = ["\r\n", "\r", "\n"]
+ data_encoding = data&.encoding
+
+ if ruby_encoding && data_encoding
+ newlines.map! do |nl|
+ nl.encode(ruby_encoding, "ASCII-8BIT").force_encoding(data_encoding)
+ end
+ end
+
+ Regexp.union(newlines)
+ end
end
def ruby_encoding
diff --git a/lib/gitlab/chat/responder/mattermost.rb b/lib/gitlab/chat/responder/mattermost.rb
new file mode 100644
index 00000000000..0488c98e422
--- /dev/null
+++ b/lib/gitlab/chat/responder/mattermost.rb
@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Chat
+ module Responder
+ class Mattermost < Responder::Base
+ SUCCESS_COLOR = '#00c100'
+ FAILURE_COLOR = '#e40303'
+
+ # Slack breaks messages apart if they're around 4 KB in size. We use a
+ # slightly smaller limit here to account for user mentions.
+ MESSAGE_SIZE_LIMIT = 3.5.kilobytes
+
+ # Sends a response back to Mattermost
+ #
+ # body - The message payload to send back to Mattermost, as a Hash.
+ def send_response(body)
+ Gitlab::HTTP.post(
+ pipeline.chat_data.response_url,
+ {
+ headers: { 'Content-Type': 'application/json' },
+ body: body.to_json
+ }
+ )
+ end
+
+ # Sends the output for a build that completed successfully.
+ #
+ # output - The output produced by the chat command.
+ def success(output)
+ return if output.empty?
+
+ send_response(
+ response_type: :in_channel,
+ attachments: [
+ {
+ color: SUCCESS_COLOR,
+ text: "ChatOps job started by #{user_ref} completed successfully",
+ fields: [
+ {
+ short: true,
+ title: "ID",
+ value: "#{build_ref}"
+ },
+ {
+ short: true,
+ title: "Name",
+ value: build.name
+ },
+ {
+ short: false,
+ title: "Output",
+ value: success_message(output)
+ }
+ ]
+ }
+ ]
+ )
+ end
+
+ # Sends the output for a build that failed.
+ def failure
+ send_response(
+ response_type: :in_channel,
+ attachments: [
+ {
+ color: FAILURE_COLOR,
+ text: "ChatOps job started by #{user_ref} failed!",
+ fields: [
+ {
+ short: true,
+ title: "ID",
+ value: "#{build_ref}"
+ },
+ {
+ short: true,
+ title: "Name",
+ value: build.name
+ }
+ ]
+ }
+ ]
+ )
+ end
+
+ # Returns the output to send back after a command has been scheduled.
+ def scheduled_output
+ {
+ response_type: :ephemeral,
+ text: "Your ChatOps job #{build_ref} has been created!"
+ }
+ end
+
+ private
+
+ def success_message(output)
+ <<~HEREDOC.chomp
+ ```shell
+ #{strip_ansi_colorcodes(limit_output(output))}
+ ```
+ HEREDOC
+ end
+
+ def limit_output(output)
+ if output.bytesize <= MESSAGE_SIZE_LIMIT
+ output
+ else
+ "The output is too large to be sent back directly!"
+ end
+ end
+
+ def strip_ansi_colorcodes(output)
+ output.gsub(/\x1b\[[0-9;]*m/, '')
+ end
+
+ def user_ref
+ user = pipeline.chat_data.chat_name.user
+ user_url = ::Gitlab::Routing.url_helpers.user_url(user)
+
+ "[#{user.name}](#{user_url})"
+ end
+
+ def build_ref
+ build_url = ::Gitlab::Routing.url_helpers.project_build_url(project, build)
+
+ "[##{build.id}](#{build_url})"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb
index 8b3c5dc9e8b..9b4cb9d0134 100644
--- a/lib/gitlab/chat_name_token.rb
+++ b/lib/gitlab/chat_name_token.rb
@@ -16,7 +16,7 @@ module Gitlab
def get
Gitlab::Redis::SharedState.with do |redis|
data = redis.get(redis_shared_state_key)
- JSON.parse(data, symbolize_names: true) if data
+ Gitlab::Json.parse(data, symbolize_names: true) if data
end
end
diff --git a/lib/gitlab/checks/base_checker.rb b/lib/gitlab/checks/base_checker.rb
index a14fa02c2a4..0045d8a4113 100644
--- a/lib/gitlab/checks/base_checker.rb
+++ b/lib/gitlab/checks/base_checker.rb
@@ -3,7 +3,6 @@
module Gitlab
module Checks
class BaseChecker
- prepend_if_ee('EE::Gitlab::Checks::BaseChecker') # rubocop: disable Cop/InjectEnterpriseEditionModule
include Gitlab::Utils::StrongMemoize
attr_reader :change_access
@@ -57,3 +56,5 @@ module Gitlab
end
end
end
+
+Gitlab::Checks::BaseChecker.prepend_if_ee('EE::Gitlab::Checks::BaseChecker')
diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb
index 14a445fcb96..8bb5ac94e45 100644
--- a/lib/gitlab/checks/change_access.rb
+++ b/lib/gitlab/checks/change_access.rb
@@ -3,8 +3,6 @@
module Gitlab
module Checks
class ChangeAccess
- prepend_if_ee('EE::Gitlab::Checks::ChangeAccess') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
ATTRIBUTES = %i[user_access project skip_authorization
skip_lfs_integrity_check protocol oldrev newrev ref
branch_name tag_name logger commits].freeze
@@ -55,3 +53,5 @@ module Gitlab
end
end
end
+
+Gitlab::Checks::ChangeAccess.prepend_if_ee('EE::Gitlab::Checks::ChangeAccess')
diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb
index a73f243e946..8780b410a07 100644
--- a/lib/gitlab/checks/diff_check.rb
+++ b/lib/gitlab/checks/diff_check.rb
@@ -4,7 +4,6 @@ module Gitlab
module Checks
class DiffCheck < BaseChecker
include Gitlab::Utils::StrongMemoize
- prepend_if_ee('EE::Gitlab::Checks::DiffCheck') # rubocop: disable Cop/InjectEnterpriseEditionModule
LOG_MESSAGES = {
validate_file_paths: "Validating diffs' file paths...",
@@ -97,3 +96,5 @@ module Gitlab
end
end
end
+
+Gitlab::Checks::DiffCheck.prepend_if_ee('EE::Gitlab::Checks::DiffCheck')
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb
index 3a05feee156..e145bd2e9df 100644
--- a/lib/gitlab/ci/ansi2html.rb
+++ b/lib/gitlab/ci/ansi2html.rb
@@ -353,7 +353,7 @@ module Gitlab
def restore_state(new_state, stream)
state = Base64.urlsafe_decode64(new_state)
- state = JSON.parse(state, symbolize_names: true)
+ state = Gitlab::Json.parse(state, symbolize_names: true)
return if state[:offset].to_i > stream.size
STATE_PARAMS.each do |param|
diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb
index 7e1a8102a35..38d36e6950c 100644
--- a/lib/gitlab/ci/ansi2json/state.rb
+++ b/lib/gitlab/ci/ansi2json/state.rb
@@ -90,7 +90,7 @@ module Gitlab
decoded_state = Base64.urlsafe_decode64(state)
return unless decoded_state.present?
- JSON.parse(decoded_state)
+ Gitlab::Json.parse(decoded_state)
end
end
end
diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb
index 1c3ce08be76..c5afb16ab1a 100644
--- a/lib/gitlab/ci/build/artifacts/metadata.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata.rb
@@ -32,7 +32,7 @@ module Gitlab
raise ParserError, 'Errors field not found!' unless errors
begin
- JSON.parse(errors)
+ Gitlab::Json.parse(errors)
rescue JSON::ParserError
raise ParserError, 'Invalid errors field!'
end
@@ -71,7 +71,7 @@ module Gitlab
next unless path =~ match_pattern
next if path =~ INVALID_PATH_PATTERN
- entries[path] = JSON.parse(meta, symbolize_names: true)
+ entries[path] = Gitlab::Json.parse(meta, symbolize_names: true)
rescue JSON::ParserError, Encoding::CompatibilityError
next
end
diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
index 80e69cdcc95..ef354832e8e 100644
--- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb
+++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb
@@ -50,7 +50,7 @@ module Gitlab
end
def basename
- (directory? && !blank_node?) ? name + '/' : name
+ directory? && !blank_node? ? name + '/' : name
end
def name
diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb
index 241c73db3bb..a9a9636637f 100644
--- a/lib/gitlab/ci/config/entry/artifacts.rb
+++ b/lib/gitlab/ci/config/entry/artifacts.rb
@@ -12,7 +12,7 @@ module Gitlab
include ::Gitlab::Config::Entry::Validatable
include ::Gitlab::Config::Entry::Attributable
- ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as].freeze
+ ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude].freeze
EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze
EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces"
@@ -35,6 +35,8 @@ module Gitlab
}, if: :expose_as_present?
validates :expose_as, type: String, length: { maximum: 100 }, if: :expose_as_present?
validates :expose_as, format: { with: EXPOSE_AS_REGEX, message: EXPOSE_AS_ERROR_MESSAGE }, if: :expose_as_present?
+ validates :exclude, array_of_strings: true, if: :exclude_enabled?
+ validates :exclude, absence: { message: 'feature is disabled' }, unless: :exclude_enabled?
validates :reports, type: Hash
validates :when,
inclusion: { in: %w[on_success on_failure always],
@@ -50,8 +52,6 @@ module Gitlab
end
def expose_as_present?
- return false unless Feature.enabled?(:ci_expose_arbitrary_artifacts_in_mr, default_enabled: true)
-
# This duplicates the `validates :config, type: Hash` above,
# but Validatable currently doesn't halt the validation
# chain if it encounters a validation error.
@@ -59,6 +59,10 @@ module Gitlab
!@config[:expose_as].nil?
end
+
+ def exclude_enabled?
+ ::Gitlab::Ci::Features.artifacts_exclude_enabled?
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb
index 8ccee3b5b2b..1a871e043a6 100644
--- a/lib/gitlab/ci/config/entry/reports.rb
+++ b/lib/gitlab/ci/config/entry/reports.rb
@@ -14,7 +14,7 @@ module Gitlab
ALLOWED_KEYS =
%i[junit codequality sast dependency_scanning container_scanning
dast performance license_management license_scanning metrics lsif
- dotenv cobertura terraform].freeze
+ dotenv cobertura terraform accessibility cluster_applications].freeze
attributes ALLOWED_KEYS
@@ -37,6 +37,8 @@ module Gitlab
validates :dotenv, array_of_strings_or_string: true
validates :cobertura, array_of_strings_or_string: true
validates :terraform, array_of_strings_or_string: true
+ validates :accessibility, array_of_strings_or_string: true
+ validates :cluster_applications, array_of_strings_or_string: true
end
end
diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb
index 7202784842a..c6ba53adfd7 100644
--- a/lib/gitlab/ci/config/entry/trigger.rb
+++ b/lib/gitlab/ci/config/entry/trigger.rb
@@ -25,8 +25,7 @@ module Gitlab
strategy :CrossProjectTrigger, if: -> (config) { !config.key?(:include) }
strategy :SameProjectTrigger, if: -> (config) do
- ::Feature.enabled?(:ci_parent_child_pipeline, default_enabled: true) &&
- config.key?(:include)
+ config.key?(:include)
end
class CrossProjectTrigger < ::Gitlab::Config::Entry::Node
@@ -72,11 +71,7 @@ module Gitlab
class UnknownStrategy < ::Gitlab::Config::Entry::Node
def errors
- if ::Feature.enabled?(:ci_parent_child_pipeline, default_enabled: true)
- ['config must specify either project or include']
- else
- ['config must specify project']
- end
+ ['config must specify either project or include']
end
end
end
diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb
index 1d7e7ea0f9a..efd48a9b29f 100644
--- a/lib/gitlab/ci/cron_parser.rb
+++ b/lib/gitlab/ci/cron_parser.rb
@@ -12,8 +12,11 @@ module Gitlab
end
def next_time_from(time)
- @cron_line ||= try_parse_cron(@cron, @cron_timezone)
- @cron_line.next_time(time).utc.in_time_zone(Time.zone) if @cron_line.present?
+ cron_line.next_time(time).utc.in_time_zone(Time.zone) if cron_line.present?
+ end
+
+ def previous_time_from(time)
+ cron_line.previous_time(time).utc.in_time_zone(Time.zone) if cron_line.present?
end
def cron_valid?
@@ -49,6 +52,10 @@ module Gitlab
def try_parse_cron(cron, cron_timezone)
Fugit::Cron.parse("#{cron} #{cron_timezone}")
end
+
+ def cron_line
+ @cron_line ||= try_parse_cron(@cron, @cron_timezone)
+ end
end
end
end
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
new file mode 100644
index 00000000000..48f3d4fdd2f
--- /dev/null
+++ b/lib/gitlab/ci/features.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ ##
+ # Ci::Features is a class that aggregates all CI/CD feature flags in one place.
+ #
+ module Features
+ def self.artifacts_exclude_enabled?
+ ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: false)
+ end
+
+ def self.ensure_scheduling_type_enabled?
+ ::Feature.enabled?(:ci_ensure_scheduling_type, default_enabled: true)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb
index a44105d53c2..0e44475607b 100644
--- a/lib/gitlab/ci/parsers.rb
+++ b/lib/gitlab/ci/parsers.rb
@@ -3,14 +3,14 @@
module Gitlab
module Ci
module Parsers
- prepend_if_ee('::EE::Gitlab::Ci::Parsers') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
ParserNotFoundError = Class.new(ParserError)
def self.parsers
{
junit: ::Gitlab::Ci::Parsers::Test::Junit,
- cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura
+ cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura,
+ terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan,
+ accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y
}
end
@@ -22,3 +22,5 @@ module Gitlab
end
end
end
+
+Gitlab::Ci::Parsers.prepend_if_ee('::EE::Gitlab::Ci::Parsers')
diff --git a/lib/gitlab/ci/parsers/accessibility/pa11y.rb b/lib/gitlab/ci/parsers/accessibility/pa11y.rb
new file mode 100644
index 00000000000..953b5a91258
--- /dev/null
+++ b/lib/gitlab/ci/parsers/accessibility/pa11y.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Accessibility
+ class Pa11y
+ def parse!(json_data, accessibility_report)
+ root = Gitlab::Json.parse(json_data).with_indifferent_access
+
+ parse_all(root, accessibility_report)
+ rescue JSON::ParserError => e
+ accessibility_report.set_error_message("JSON parsing failed: #{e}")
+ rescue StandardError => e
+ accessibility_report.set_error_message("Pa11y parsing failed: #{e}")
+ end
+
+ private
+
+ def parse_all(root, accessibility_report)
+ return unless root.present?
+
+ root.dig("results").each do |url, value|
+ accessibility_report.add_url(url, value)
+ end
+
+ accessibility_report
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb
new file mode 100644
index 00000000000..26a18c6603e
--- /dev/null
+++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Terraform
+ class Tfplan
+ TfplanParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
+
+ def parse!(json_data, terraform_reports, artifact:)
+ tfplan = Gitlab::Json.parse(json_data).tap do |parsed_data|
+ parsed_data['job_path'] = Gitlab::Routing.url_helpers.project_job_path(
+ artifact.job.project, artifact.job
+ )
+ end
+
+ raise TfplanParserError, 'Tfplan missing required key' unless valid_supported_keys?(tfplan)
+
+ terraform_reports.add_plan(artifact.filename, tfplan)
+ rescue JSON::ParserError
+ raise TfplanParserError, 'JSON parsing failed'
+ rescue
+ raise TfplanParserError, 'Tfplan parsing failed'
+ end
+
+ private
+
+ def valid_supported_keys?(tfplan)
+ tfplan.keys == %w[create update delete job_path]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb
index 33140b4c7fd..5746f38ae5b 100644
--- a/lib/gitlab/ci/parsers/test/junit.rb
+++ b/lib/gitlab/ci/parsers/test/junit.rb
@@ -15,10 +15,10 @@ module Gitlab
test_case = create_test_case(test_case, args)
test_suite.add_test_case(test_case)
end
- rescue Nokogiri::XML::SyntaxError
- raise JunitParserError, "XML parsing failed"
- rescue
- raise JunitParserError, "JUnit parsing failed"
+ rescue Nokogiri::XML::SyntaxError => e
+ test_suite.set_suite_error("JUnit XML parsing failed: #{e}")
+ rescue StandardError => e
+ test_suite.set_suite_error("JUnit data parsing failed: #{e}")
end
private
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index fa46114615c..73187401903 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -76,6 +76,21 @@ module Gitlab
def parent_pipeline
bridge&.parent_pipeline
end
+
+ def duration_histogram
+ strong_memoize(:duration_histogram) do
+ name = :gitlab_ci_pipeline_creation_duration_seconds
+ comment = 'Pipeline creation duration'
+ labels = {}
+ buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0]
+
+ Gitlab::Metrics.histogram(name, comment, labels, buckets)
+ end
+ end
+
+ def observe_creation_duration(duration)
+ duration_histogram.observe({}, duration.seconds)
+ end
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index 99780409085..a7c671e76d3 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -10,6 +10,7 @@ module Gitlab
@command = command
@sequence = sequence
@completed = []
+ @start = Time.now
end
def build!
@@ -24,6 +25,8 @@ module Gitlab
@pipeline.tap do
yield @pipeline, self if block_given?
+
+ @command.observe_creation_duration(Time.now - @start)
end
end
diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb
index 3bec6d1e8b6..c0641d9ff0a 100644
--- a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb
+++ b/lib/gitlab/ci/pipeline/seed/build/resource_group.rb
@@ -16,7 +16,6 @@ module Gitlab
end
def to_resource
- return unless Feature.enabled?(:ci_resource_group, build.project, default_enabled: true)
return unless resource_group_key.present?
resource_group = build.project.resource_groups
diff --git a/lib/gitlab/ci/reports/accessibility_reports.rb b/lib/gitlab/ci/reports/accessibility_reports.rb
new file mode 100644
index 00000000000..1901ba3b102
--- /dev/null
+++ b/lib/gitlab/ci/reports/accessibility_reports.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ class AccessibilityReports
+ attr_reader :urls, :error_message
+
+ def initialize
+ @urls = {}
+ @error_message = nil
+ end
+
+ def add_url(url, data)
+ if url.empty?
+ set_error_message("Empty URL detected in gl-accessibility.json")
+ else
+ urls[url] = data
+ end
+ end
+
+ def scans_count
+ @urls.size
+ end
+
+ def passes_count
+ @urls.count { |url, errors| errors.empty? }
+ end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def errors_count
+ @urls.sum { |url, errors| errors.size }
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
+ def set_error_message(error)
+ @error_message = error
+ end
+
+ def all_errors
+ @urls.values.flatten
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/accessibility_reports_comparer.rb b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb
new file mode 100644
index 00000000000..fa6337166d5
--- /dev/null
+++ b/lib/gitlab/ci/reports/accessibility_reports_comparer.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ class AccessibilityReportsComparer
+ include Gitlab::Utils::StrongMemoize
+
+ STATUS_SUCCESS = 'success'
+ STATUS_FAILED = 'failed'
+
+ attr_reader :base_reports, :head_reports
+
+ def initialize(base_reports, head_reports)
+ @base_reports = base_reports || AccessibilityReports.new
+ @head_reports = head_reports
+ end
+
+ def status
+ head_reports.errors_count.positive? ? STATUS_FAILED : STATUS_SUCCESS
+ end
+
+ def existing_errors
+ strong_memoize(:existing_errors) do
+ base_reports.all_errors
+ end
+ end
+
+ def new_errors
+ strong_memoize(:new_errors) do
+ head_reports.all_errors - base_reports.all_errors
+ end
+ end
+
+ def resolved_errors
+ strong_memoize(:resolved_errors) do
+ base_reports.all_errors - head_reports.all_errors
+ end
+ end
+
+ def errors_count
+ head_reports.errors_count
+ end
+
+ def resolved_count
+ resolved_errors.size
+ end
+
+ def total_count
+ existing_errors.size + new_errors.size
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/terraform_reports.rb b/lib/gitlab/ci/reports/terraform_reports.rb
new file mode 100644
index 00000000000..f955d007daf
--- /dev/null
+++ b/lib/gitlab/ci/reports/terraform_reports.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ class TerraformReports
+ attr_reader :plans
+
+ def initialize
+ @plans = {}
+ end
+
+ def pick(keys)
+ terraform_plans = plans.select do |key|
+ keys.include?(key)
+ end
+
+ { plans: terraform_plans }
+ end
+
+ def add_plan(name, plan)
+ plans[name] = plan
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb
index 72323c4343d..86ba725c71e 100644
--- a/lib/gitlab/ci/reports/test_reports.rb
+++ b/lib/gitlab/ci/reports/test_reports.rb
@@ -42,6 +42,12 @@ module Gitlab
self
end
+ def suite_errors
+ test_suites.each_with_object({}) do |(name, suite), errors|
+ errors[suite.name] = suite.suite_error if suite.suite_error
+ end
+ end
+
TestCase::STATUS_TYPES.each do |status_type|
define_method("#{status_type}_count") do
# rubocop: disable CodeReuse/ActiveRecord
diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb
index cf43c5313c0..8bbf2e0f6cf 100644
--- a/lib/gitlab/ci/reports/test_suite.rb
+++ b/lib/gitlab/ci/reports/test_suite.rb
@@ -7,6 +7,7 @@ module Gitlab
attr_reader :name
attr_reader :test_cases
attr_reader :total_time
+ attr_reader :suite_error
def initialize(name = nil)
@name = name
@@ -25,12 +26,16 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def total_count
+ return 0 if suite_error
+
test_cases.values.sum(&:count)
end
# rubocop: enable CodeReuse/ActiveRecord
def total_status
- if failed_count > 0 || error_count > 0
+ if suite_error
+ TestCase::STATUS_ERROR
+ elsif failed_count > 0 || error_count > 0
TestCase::STATUS_FAILED
else
TestCase::STATUS_SUCCESS
@@ -49,14 +54,22 @@ module Gitlab
TestCase::STATUS_TYPES.each do |status_type|
define_method("#{status_type}") do
- test_cases[status_type] || {}
+ return {} if suite_error || test_cases[status_type].nil?
+
+ test_cases[status_type]
end
define_method("#{status_type}_count") do
- test_cases[status_type]&.length.to_i
+ return 0 if suite_error || test_cases[status_type].nil?
+
+ test_cases[status_type].length
end
end
+ def set_suite_error(msg)
+ @suite_error = msg
+ end
+
private
def existing_key?(test_case)
diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb
index b0b01538a30..76ad113aad9 100644
--- a/lib/gitlab/ci/status/build/failed.rb
+++ b/lib/gitlab/ci/status/build/failed.rb
@@ -29,8 +29,6 @@ module Gitlab
private_constant :REASONS
- prepend_if_ee('::EE::Gitlab::Ci::Status::Build::Failed') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
def status_tooltip
base_message
end
@@ -65,3 +63,5 @@ module Gitlab
end
end
end
+
+Gitlab::Ci::Status::Build::Failed.prepend_if_ee('::EE::Gitlab::Ci::Status::Build::Failed')
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index a9f29bda9b9..5017037fb5a 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -48,7 +48,6 @@ variables:
POSTGRES_PASSWORD: testing-password
POSTGRES_ENABLED: "true"
POSTGRES_DB: $CI_ENVIRONMENT_SLUG
- POSTGRES_VERSION: 9.6.2
DOCKER_DRIVER: overlay2
@@ -159,5 +158,5 @@ include:
- template: Security/DAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
- template: Security/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
- template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
- - template: Security/License-Management.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml
+ - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
- template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml
index a41b399032f..82b2f5c035e 100644
--- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml
@@ -5,32 +5,9 @@ stages:
- deploy
- production
+variables:
+ AUTO_DEVOPS_PLATFORM_TARGET: ECS
+
include:
- template: Jobs/Build.gitlab-ci.yml
-
-.deploy_to_ecs:
- image: registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest
- script:
- - ecs update-task-definition
-
-review:
- extends: .deploy_to_ecs
- stage: review
- environment:
- name: review/$CI_COMMIT_REF_NAME
- only:
- refs:
- - branches
- - tags
- except:
- refs:
- - master
-
-production:
- extends: .deploy_to_ecs
- stage: production
- environment:
- name: production
- only:
- refs:
- - master
+ - template: Jobs/Deploy/ECS.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
index d85078c0a40..adbf9731e43 100644
--- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
@@ -30,11 +30,9 @@ performance:
paths:
- performance.json
- sitespeed-results/
- only:
- refs:
- - branches
- - tags
- kubernetes: active
- except:
- variables:
- - $PERFORMANCE_DISABLED
+ rules:
+ - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+ when: never
+ - if: '$PERFORMANCE_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index 3949b87bbda..787f07521e0 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -15,6 +15,5 @@ build:
export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG}
fi
- /build/build.sh
- only:
- - branches
- - tags
+ rules:
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 9c4699f1f44..24e75c56a75 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -26,10 +26,7 @@ code_quality:
codequality: gl-code-quality-report.json
expire_in: 1 week
dependencies: []
- only:
- refs:
- - branches
- - tags
- except:
- variables:
- - $CODE_QUALITY_DISABLED
+ rules:
+ - if: '$CODE_QUALITY_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index 3cf4910fe86..5174aed04ba 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.dast-auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.10.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0"
dast_environment_deploy:
extends: .dast-auto-deploy
@@ -18,17 +18,16 @@ dast_environment_deploy:
on_stop: stop_dast_environment
artifacts:
paths: [environment_url.txt]
- only:
- refs:
- - branches
- variables:
- - $GITLAB_FEATURES =~ /\bdast\b/
- kubernetes: active
- except:
- variables:
- - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME
- - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH
- - $DAST_WEBSITE # we don't need to create a review app if a URL is already given
+ rules:
+ - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME
+ when: never
+ - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH
+ when: never
+ - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $CI_KUBERNETES_ACTIVE &&
+ $GITLAB_FEATURES =~ /\bdast\b/
stop_dast_environment:
extends: .dast-auto-deploy
@@ -42,14 +41,13 @@ stop_dast_environment:
name: dast-default
action: stop
needs: ["dast"]
- only:
- refs:
- - branches
- variables:
- - $GITLAB_FEATURES =~ /\bdast\b/
- kubernetes: active
- except:
- variables:
- - $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME
- - $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH
- - $DAST_WEBSITE
+ rules:
+ - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME
+ when: never
+ - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH
+ when: never
+ - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $CI_KUBERNETES_ACTIVE &&
+ $GITLAB_FEATURES =~ /\bdast\b/
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 9bf0d31409a..b4e5a41a34d 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,8 @@
.auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.13.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0"
+
+include:
+ - template: Jobs/Deploy/ECS.gitlab-ci.yml
review:
extends: .auto-deploy
@@ -18,16 +21,14 @@ review:
on_stop: stop_review
artifacts:
paths: [environment_url.txt]
- only:
- refs:
- - branches
- - tags
- kubernetes: active
- except:
- refs:
- - master
- variables:
- - $REVIEW_DISABLED
+ rules:
+ - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+ when: never
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ when: never
+ - if: '$REVIEW_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
stop_review:
extends: .auto-deploy
@@ -41,18 +42,16 @@ stop_review:
name: review/$CI_COMMIT_REF_NAME
action: stop
dependencies: []
- when: manual
allow_failure: true
- only:
- refs:
- - branches
- - tags
- kubernetes: active
- except:
- refs:
- - master
- variables:
- - $REVIEW_DISABLED
+ rules:
+ - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+ when: never
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ when: never
+ - if: '$REVIEW_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
+ when: manual
# Staging deploys are disabled by default since
# continuous deployment to production is enabled by default
@@ -73,12 +72,12 @@ staging:
environment:
name: staging
url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN
- only:
- refs:
- - master
- kubernetes: active
- variables:
- - $STAGING_ENABLED
+ rules:
+ - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+ when: never
+ - if: '$CI_COMMIT_BRANCH != "master"'
+ when: never
+ - if: '$STAGING_ENABLED'
# Canaries are disabled by default, but if you want them,
# and know what the downsides are, you can enable this by setting
@@ -97,13 +96,13 @@ canary:
environment:
name: production
url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
- when: manual
- only:
- refs:
- - master
- kubernetes: active
- variables:
- - $CANARY_ENABLED
+ rules:
+ - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+ when: never
+ - if: '$CI_COMMIT_BRANCH != "master"'
+ when: never
+ - if: '$CANARY_ENABLED'
+ when: manual
.production: &production_template
extends: .auto-deploy
@@ -126,32 +125,33 @@ canary:
production:
<<: *production_template
- only:
- refs:
- - master
- kubernetes: active
- except:
- variables:
- - $STAGING_ENABLED
- - $CANARY_ENABLED
- - $INCREMENTAL_ROLLOUT_ENABLED
- - $INCREMENTAL_ROLLOUT_MODE
+ rules:
+ - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+ when: never
+ - if: '$STAGING_ENABLED'
+ when: never
+ - if: '$CANARY_ENABLED'
+ when: never
+ - if: '$INCREMENTAL_ROLLOUT_ENABLED'
+ when: never
+ - if: '$INCREMENTAL_ROLLOUT_MODE'
+ when: never
+ - if: '$CI_COMMIT_BRANCH == "master"'
production_manual:
<<: *production_template
- when: manual
allow_failure: false
- only:
- refs:
- - master
- kubernetes: active
- variables:
- - $STAGING_ENABLED
- - $CANARY_ENABLED
- except:
- variables:
- - $INCREMENTAL_ROLLOUT_ENABLED
- - $INCREMENTAL_ROLLOUT_MODE
+ rules:
+ - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+ when: never
+ - if: '$INCREMENTAL_ROLLOUT_ENABLED'
+ when: never
+ - if: '$INCREMENTAL_ROLLOUT_MODE'
+ when: never
+ - if: '$CI_COMMIT_BRANCH == "master" && $STAGING_ENABLED'
+ when: manual
+ - if: '$CI_COMMIT_BRANCH == "master" && $CANARY_ENABLED'
+ when: manual
# This job implements incremental rollout on for every push to `master`.
@@ -176,29 +176,29 @@ production_manual:
.manual_rollout_template: &manual_rollout_template
<<: *rollout_template
stage: production
- when: manual
- # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4)
- only:
- refs:
- - master
- kubernetes: active
- variables:
- - $INCREMENTAL_ROLLOUT_MODE == "manual"
- - $INCREMENTAL_ROLLOUT_ENABLED
- except:
- variables:
- - $INCREMENTAL_ROLLOUT_MODE == "timed"
+ rules:
+ - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+ when: never
+ - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"'
+ when: never
+ - if: '$CI_COMMIT_BRANCH != "master"'
+ when: never
+ # $INCREMENTAL_ROLLOUT_ENABLED is for compamtibilty with pre-GitLab 11.4 syntax
+ - if: '$INCREMENTAL_ROLLOUT_MODE == "manual" || $INCREMENTAL_ROLLOUT_ENABLED'
+ when: manual
.timed_rollout_template: &timed_rollout_template
<<: *rollout_template
- when: delayed
- start_in: 5 minutes
- only:
- refs:
- - master
- kubernetes: active
- variables:
- - $INCREMENTAL_ROLLOUT_MODE == "timed"
+ rules:
+ - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
+ when: never
+ - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"'
+ when: never
+ - if: '$CI_COMMIT_BRANCH != "master"'
+ when: never
+ - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"'
+ when: delayed
+ start_in: 5 minutes
timed rollout 10%:
<<: *timed_rollout_template
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
new file mode 100644
index 00000000000..642f0ebeaf7
--- /dev/null
+++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
@@ -0,0 +1,30 @@
+.deploy_to_ecs:
+ image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest'
+ script:
+ - ecs update-task-definition
+
+review_ecs:
+ extends: .deploy_to_ecs
+ stage: review
+ environment:
+ name: review/$CI_COMMIT_REF_NAME
+ rules:
+ - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"'
+ when: never
+ - if: '$CI_KUBERNETES_ACTIVE'
+ when: never
+ - if: '$REVIEW_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_BRANCH != "master"'
+
+production_ecs:
+ extends: .deploy_to_ecs
+ stage: production
+ environment:
+ name: production
+ rules:
+ - if: '$AUTO_DEVOPS_PLATFORM_TARGET != "ECS"'
+ when: never
+ - if: '$CI_KUBERNETES_ACTIVE'
+ when: never
+ - if: '$CI_COMMIT_BRANCH == "master"'
diff --git a/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml
index a0ddd273552..3b87d53f165 100644
--- a/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml
@@ -1,10 +1,12 @@
test:
- services:
- - "postgres:${POSTGRES_VERSION}"
variables:
+ POSTGRES_VERSION: 9.6.16
POSTGRES_DB: test
+ services:
+ - "postgres:${POSTGRES_VERSION}"
stage: test
image: gliderlabs/herokuish:latest
+ needs: []
script:
- |
if [ -z ${KUBERNETES_PORT+x} ]; then
@@ -15,9 +17,7 @@ test:
- export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}"
- cp -R . /tmp/app
- /bin/herokuish buildpack test
- only:
- - branches
- - tags
- except:
- variables:
- - $TEST_DISABLED
+ rules:
+ - if: '$TEST_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
index b4208ed9d7d..e081e20564a 100644
--- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
@@ -1,7 +1,7 @@
-# Official Java image. Look for the different tagged releases at
-# https://hub.docker.com/r/library/java/tags/ . A Java image is not required
+# Official OpenJDK Java image. Look for the different tagged releases at
+# https://hub.docker.com/_/openjdk/ . A Java image is not required
# but an image with a JVM speeds up the build a bit.
-image: java:8
+image: openjdk:8
before_script:
# Enable the usage of sources over https
@@ -14,7 +14,7 @@ before_script:
- apt-get update -yqq
- apt-get install sbt -yqq
# Log the sbt version
- - sbt sbt-version
+ - sbt sbtVersion
test:
script:
diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
index 6efb6b4e273..21bcdd8d9b5 100644
--- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
@@ -1,16 +1,20 @@
# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/
variables:
+ # Setting this variable will affect all Security templates
+ # (SAST, Dependency Scanning, ...)
+ SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+
CS_MAJOR_VERSION: 2
container_scanning:
stage: test
- image: registry.gitlab.com/gitlab-org/security-products/analyzers/klar:$CS_MAJOR_VERSION
+ image: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION
variables:
# By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image
# to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes
CLAIR_DB_IMAGE_TAG: "latest"
- CLAIR_DB_IMAGE: "arminc/clair-db:$CLAIR_DB_IMAGE_TAG"
+ CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG"
# Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml`
# file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template
# for details
@@ -25,11 +29,8 @@ container_scanning:
reports:
container_scanning: gl-container-scanning-report.json
dependencies: []
- only:
- refs:
- - branches
- variables:
- - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
- except:
- variables:
- - $CONTAINER_SCANNING_DISABLED
+ rules:
+ - if: $CONTAINER_SCANNING_DISABLED
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
index 0e3d7660bdf..07399216597 100644
--- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
@@ -12,11 +12,14 @@ stages:
variables:
DAST_VERSION: 1
+ # Setting this variable will affect all Security templates
+ # (SAST, Dependency Scanning, ...)
+ SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
dast:
stage: dast
image:
- name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
+ name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION"
variables:
GIT_STRATEGY: none
allow_failure: true
@@ -27,12 +30,15 @@ dast:
artifacts:
reports:
dast: gl-dast-report.json
- only:
- refs:
- - branches
- variables:
- - $GITLAB_FEATURES =~ /\bdast\b/
- except:
- variables:
- - $DAST_DISABLED
- - $DAST_DISABLED_FOR_DEFAULT_BRANCH && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ rules:
+ - if: $DAST_DISABLED
+ when: never
+ - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH &&
+ $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+ when: never
+ - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME &&
+ $REVIEW_DISABLED && $DAST_WEBSITE == null &&
+ $DAST_API_SPECIFICATION == null
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdast\b/
diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
index 0ecf37b37a3..616966b4f04 100644
--- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
@@ -5,11 +5,16 @@
# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
variables:
- SECURITY_SCANNER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products"
- DS_ANALYZER_IMAGE_PREFIX: "$SECURITY_SCANNER_IMAGE_PREFIX/analyzers"
+ # Setting this variable will affect all Security templates
+ # (SAST, Dependency Scanning, ...)
+ SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+
+ # Deprecated, use SECURE_ANALYZERS_PREFIX instead
+ DS_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX"
+
DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python"
DS_MAJOR_VERSION: 2
- DS_DISABLE_DIND: "false"
+ DS_DISABLE_DIND: "true"
dependency_scanning:
stage: test
@@ -21,7 +26,6 @@ dependency_scanning:
services:
- docker:stable-dind
script:
- - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
- |
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
@@ -68,28 +72,25 @@ dependency_scanning:
) \
--volume "$PWD:/code" \
--volume /var/run/docker.sock:/var/run/docker.sock \
- "$SECURITY_SCANNER_IMAGE_PREFIX/dependency-scanning:$DS_VERSION" /code
+ "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$DS_MAJOR_VERSION" /code
artifacts:
reports:
dependency_scanning: gl-dependency-scanning-report.json
dependencies: []
- only:
- refs:
- - branches
- variables:
- - $GITLAB_FEATURES =~ /\bdependency_scanning\b/
- except:
- variables:
- - $DEPENDENCY_SCANNING_DISABLED
- - $DS_DISABLE_DIND == 'true'
+ rules:
+ - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'true'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdependency_scanning\b/
.ds-analyzer:
extends: dependency_scanning
services: []
- except:
- variables:
- - $DEPENDENCY_SCANNING_DISABLED
- - $DS_DISABLE_DIND == 'false'
+ rules:
+ - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdependency_scanning\b/
script:
- /analyzer run
@@ -97,48 +98,81 @@ gemnasium-dependency_scanning:
extends: .ds-analyzer
image:
name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium:$DS_MAJOR_VERSION"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
- $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby|javascript|php|\bgo\b/
+ rules:
+ - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
+ $DS_DEFAULT_ANALYZERS =~ /gemnasium([^-]|$)/
+ exists:
+ - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'
+ - '{composer.lock,*/composer.lock,*/*/composer.lock}'
+ - '{gems.locked,*/gems.locked,*/*/gems.locked}'
+ - '{go.sum,*/go.sum,*/*/go.sum}'
+ - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}'
+ - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}'
+ - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}'
gemnasium-maven-dependency_scanning:
extends: .ds-analyzer
image:
name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-maven:$DS_MAJOR_VERSION"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
- $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(java|scala)\b/
+ rules:
+ - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
+ $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/
+ exists:
+ - '{build.gradle,*/build.gradle,*/*/build.gradle}'
+ - '{build.sbt,*/build.sbt,*/*/build.sbt}'
+ - '{pom.xml,*/pom.xml,*/*/pom.xml}'
gemnasium-python-dependency_scanning:
extends: .ds-analyzer
image:
name: "$DS_ANALYZER_IMAGE_PREFIX/gemnasium-python:$DS_MAJOR_VERSION"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
- $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /python/
+ rules:
+ - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
+ $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/
+ exists:
+ - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}'
+ - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}'
+ - '{Pipfile,*/Pipfile,*/*/Pipfile}'
+ - '{requires.txt,*/requires.txt,*/*/requires.txt}'
+ - '{setup.py,*/setup.py,*/*/setup.py}'
+ # Support passing of $PIP_REQUIREMENTS_FILE
+ # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
+ $DS_DEFAULT_ANALYZERS =~ /gemnasium-python/ &&
+ $PIP_REQUIREMENTS_FILE
bundler-audit-dependency_scanning:
extends: .ds-analyzer
image:
name: "$DS_ANALYZER_IMAGE_PREFIX/bundler-audit:$DS_MAJOR_VERSION"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
- $DS_DEFAULT_ANALYZERS =~ /bundler-audit/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /ruby/
+ rules:
+ - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
+ $DS_DEFAULT_ANALYZERS =~ /bundler-audit/
+ exists:
+ - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}'
retire-js-dependency_scanning:
extends: .ds-analyzer
image:
name: "$DS_ANALYZER_IMAGE_PREFIX/retire.js:$DS_MAJOR_VERSION"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
- $DS_DEFAULT_ANALYZERS =~ /retire.js/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /javascript/
+ rules:
+ - if: $DEPENDENCY_SCANNING_DISABLED || $DS_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bdependency_scanning\b/ &&
+ $DS_DEFAULT_ANALYZERS =~ /retire.js/
+ exists:
+ - '{package.json,*/package.json,*/*/package.json}'
diff --git a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml
index 58fd018a82d..87f78d0c887 100644
--- a/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/License-Management.gitlab-ci.yml
@@ -1,29 +1,13 @@
# Deprecated: https://gitlab.com/gitlab-org/gitlab/issues/14624
# Please, use License-Scanning.gitlab-ci.yml template instead
-variables:
- LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager.
+include:
+ - template: License-Scanning.gitlab-ci.yml
-license_management:
- stage: test
- image:
- name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable"
- entrypoint: [""]
- variables:
- SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD
- allow_failure: true
- script:
- - echo "This template is deprecated, please use License-Scanning.gitlab-ci.yml template instead."
- - /run.sh analyze .
- artifacts:
- reports:
- license_management: gl-license-management-report.json
- dependencies: []
- only:
- refs:
- - branches
- variables:
- - $GITLAB_FEATURES =~ /\blicense_management\b/
- except:
- variables:
- - $LICENSE_MANAGEMENT_DISABLED
+license_scanning:
+ before_script:
+ - |
+ echo "As of GitLab 12.8, we deprecated the License-Management.gitlab.ci.yml template.
+ Please replace it with the License-Scanning.gitlab-ci.yml template instead.
+ For more details visit
+ https://docs.gitlab.com/ee/user/compliance/license_compliance/#migration-from-license_management-to-license_scanning"
diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
index 2333fb4e947..b86014c1ebc 100644
--- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
@@ -5,29 +5,30 @@
# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
variables:
+ # Setting this variable will affect all Security templates
+ # (SAST, Dependency Scanning, ...)
+ SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+
LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager.
+ LICENSE_MANAGEMENT_VERSION: 3
license_scanning:
stage: test
image:
- name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable"
+ name: "$SECURE_ANALYZERS_PREFIX/license-finder:$LICENSE_MANAGEMENT_VERSION"
entrypoint: [""]
variables:
+ LM_REPORT_FILE: gl-license-scanning-report.json
SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD
allow_failure: true
script:
- /run.sh analyze .
- after_script:
- - mv gl-license-management-report.json gl-license-scanning-report.json
artifacts:
reports:
- license_scanning: gl-license-scanning-report.json
+ license_scanning: $LM_REPORT_FILE
dependencies: []
- only:
- refs:
- - branches
- variables:
- - $GITLAB_FEATURES =~ /\blicense_scanning\b/
- except:
- variables:
- - $LICENSE_MANAGEMENT_DISABLED
+ rules:
+ - if: $LICENSE_MANAGEMENT_DISABLED
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\blicense_scanning\b/
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 03b9720747d..47f68118ee0 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -5,10 +5,16 @@
# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
variables:
- SAST_ANALYZER_IMAGE_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+ # Setting this variable will affect all Security templates
+ # (SAST, Dependency Scanning, ...)
+ SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+
+ # Deprecated, use SECURE_ANALYZERS_PREFIX instead
+ SAST_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX"
+
SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec"
SAST_ANALYZER_IMAGE_TAG: 2
- SAST_DISABLE_DIND: "false"
+ SAST_DISABLE_DIND: "true"
SCAN_KUBERNETES_MANIFESTS: "false"
sast:
@@ -17,19 +23,18 @@ sast:
artifacts:
reports:
sast: gl-sast-report.json
- only:
- refs:
- - branches
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'true'
+ when: never
+ - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bsast\b/
image: docker:stable
variables:
+ SEARCH_MAX_DEPTH: 4
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
services:
- docker:stable-dind
script:
- - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
- |
if ! docker info &>/dev/null; then
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
@@ -41,19 +46,16 @@ sast:
$(awk 'BEGIN{for(v in ENVIRON) print v}' | grep -v -E '^(DOCKER_|CI|GITLAB_|FF_|HOME|PWD|OLDPWD|PATH|SHLVL|HOSTNAME)' | awk '{printf " -e %s", $0}') \
--volume "$PWD:/code" \
--volume /var/run/docker.sock:/var/run/docker.sock \
- "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_VERSION" /app/bin/run /code
- except:
- variables:
- - $SAST_DISABLED
- - $SAST_DISABLE_DIND == 'true'
+ "registry.gitlab.com/gitlab-org/security-products/sast:$SAST_ANALYZER_IMAGE_TAG" /app/bin/run /code
.sast-analyzer:
extends: sast
services: []
- except:
- variables:
- - $SAST_DISABLED
- - $SAST_DISABLE_DIND == 'false'
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/
script:
- /analyzer run
@@ -61,49 +63,65 @@ bandit-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /bandit/&&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bpython\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /bandit/
+ exists:
+ - '**/*.py'
brakeman-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /brakeman/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bruby\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /brakeman/
+ exists:
+ - '**/*.rb'
eslint-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /eslint/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjavascript\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /eslint/
+ exists:
+ - '**/*.html'
+ - '**/*.js'
flawfinder-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /(c(\+\+)?,)|(c(\+\+)?$)/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /flawfinder/
+ exists:
+ - '**/*.c'
+ - '**/*.cpp'
kubesec-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /kubesec/ &&
$SCAN_KUBERNETES_MANIFESTS == 'true'
@@ -111,87 +129,117 @@ gosec-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /gosec/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bgo\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /gosec/
+ exists:
+ - '**/*.go'
nodejs-scan-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bjavascript\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/
+ exists:
+ - '**/*.js'
phpcs-security-audit-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bphp\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/
+ exists:
+ - '**/*.php'
pmd-apex-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\bapex\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/
+ exists:
+ - '**/*.cls'
secrets-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/secrets:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /secrets/
security-code-scan-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(c\#|visual basic\b)/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/
+ exists:
+ - '**/*.csproj'
+ - '**/*.vbproj'
sobelow-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /sobelow/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\belixir\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /sobelow/
+ exists:
+ - '**/*.ex'
+ - '**/*.exs'
spotbugs-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /spotbugs/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\b(groovy|java|scala)\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /spotbugs/
+ exists:
+ - '**/*.groovy'
+ - '**/*.java'
+ - '**/*.scala'
tslint-sast:
extends: .sast-analyzer
image:
name: "$SAST_ANALYZER_IMAGE_PREFIX/tslint:$SAST_ANALYZER_IMAGE_TAG"
- only:
- variables:
- - $GITLAB_FEATURES =~ /\bsast\b/ &&
- $SAST_DEFAULT_ANALYZERS =~ /tslint/ &&
- $CI_PROJECT_REPOSITORY_LANGUAGES =~ /\btypescript\b/
+ rules:
+ - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
+ when: never
+ - if: $CI_COMMIT_BRANCH &&
+ $GITLAB_FEATURES =~ /\bsast\b/ &&
+ $SAST_DEFAULT_ANALYZERS =~ /tslint/
+ exists:
+ - '**/*.ts'
diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
new file mode 100644
index 00000000000..b6c05c61db1
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
@@ -0,0 +1,246 @@
+# This template should be used when Security Products (https://about.gitlab.com/handbook/engineering/development/secure/#security-products)
+# have to be downloaded and stored locally.
+#
+# Usage:
+#
+# ```
+# include:
+# - template: Secure-Binaries.gitlab-ci.yml
+# ```
+#
+# Docs: https://docs.gitlab.com/ee/topics/airgap/
+
+
+variables:
+ SECURE_BINARIES_ANALYZERS: >-
+ bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec,
+ bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python,
+ klar, clair-vulnerabilities-db,
+ license-finder,
+ dast
+
+ SECURE_BINARIES_DOWNLOAD_IMAGES: "true"
+ SECURE_BINARIES_PUSH_IMAGES: "true"
+ SECURE_BINARIES_SAVE_ARTIFACTS: "false"
+
+ SECURE_BINARIES_ANALYZER_VERSION: "2"
+
+.download_images:
+ allow_failure: true
+ image: docker:stable
+ only:
+ refs:
+ - branches
+ variables:
+ DOCKER_DRIVER: overlay2
+ DOCKER_TLS_CERTDIR: ""
+ services:
+ - docker:stable-dind
+ script:
+ - docker info
+ - env
+ - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"registry.gitlab.com/gitlab-org/security-products/analyzers/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi
+ - docker pull ${SECURE_BINARIES_IMAGE}
+ - mkdir -p output/$(dirname ${CI_JOB_NAME})
+ - |
+ if [ "$SECURE_BINARIES_SAVE_ARTIFACTS" = "true" ]; then
+ docker save ${SECURE_BINARIES_IMAGE} | gzip > output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz
+ sha256sum output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz > output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz.sha256sum
+ fi
+ - |
+ if [ "$SECURE_BINARIES_PUSH_IMAGES" = "true" ]; then
+ docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+ docker tag ${SECURE_BINARIES_IMAGE} ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}
+ docker push ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}
+ fi
+
+ artifacts:
+ paths:
+ - output/
+
+#
+# SAST jobs
+#
+
+bandit:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bbandit\b/
+
+brakeman:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bbrakeman\b/
+
+gosec:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bgosec\b/
+
+spotbugs:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bspotbugs\b/
+
+flawfinder:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bflawfinder\b/
+
+phpcs-security-audit:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bphpcs-security-audit\b/
+
+security-code-scan:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bsecurity-code-scan\b/
+
+nodejs-scan:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bnodejs-scan\b/
+
+eslint:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\beslint\b/
+
+tslint:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\btslint\b/
+
+secrets:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/
+
+sobelow:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bsobelow\b/
+
+pmd-apex:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/
+
+kubesec:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bkubesec\b/
+#
+# Container Scanning jobs
+#
+
+klar:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bklar\b/
+
+clair-vulnerabilities-db:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bclair-vulnerabilities-db\b/
+ variables:
+ SECURE_BINARIES_IMAGE: arminc/clair-db
+ SECURE_BINARIES_ANALYZER_VERSION: latest
+
+#
+# Dependency Scanning jobs
+#
+
+bundler-audit:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bbundler-audit\b/
+
+retire.js:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bretire\.js\b/
+
+gemnasium:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium\b/
+
+gemnasium-maven:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-maven\b/
+
+gemnasium-python:
+ extends: .download_images
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-python\b/
+
+#
+# License Scanning
+#
+
+license-finder:
+ extends: .download_images
+ variables:
+ SECURE_BINARIES_ANALYZER_VERSION: "3"
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\blicense-finder\b/
+
+#
+# DAST
+#
+
+dast:
+ extends: .download_images
+ variables:
+ SECURE_BINARIES_ANALYZER_VERSION: "1"
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/
diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
index 83483108fde..a0832718214 100644
--- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
@@ -19,7 +19,7 @@ cache:
- .terraform
before_script:
- - alias convert_report="jq -r '([.resource_changes[].change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'"
+ - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'"
- terraform --version
- terraform init
diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml
index 5d9d3c74def..e8a99a6ea06 100644
--- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml
@@ -8,12 +8,14 @@ stages:
a11y:
stage: accessibility
- image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.2
+ image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:5.3.0-gitlab.3
script: /gitlab-accessibility.sh $a11y_urls
allow_failure: true
artifacts:
when: always
expose_as: 'Accessibility Reports'
paths: ['reports/']
+ reports:
+ accessibility: reports/gl-accessibility.json
rules:
- if: $a11y_urls
diff --git a/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml
new file mode 100644
index 00000000000..05635cf71be
--- /dev/null
+++ b/lib/gitlab/ci/templates/Workflows/Branch-Pipelines.gitlab-ci.yml
@@ -0,0 +1,7 @@
+# Read more on when to use this template at
+# https://docs.gitlab.com/ee/ci/yaml/#workflowrules
+
+workflow:
+ rules:
+ - if: $CI_COMMIT_TAG
+ - if: $CI_COMMIT_BRANCH
diff --git a/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml
new file mode 100644
index 00000000000..50ff4c1f60b
--- /dev/null
+++ b/lib/gitlab/ci/templates/Workflows/MergeRequest-Pipelines.gitlab-ci.yml
@@ -0,0 +1,8 @@
+# Read more on when to use this template at
+# https://docs.gitlab.com/ee/ci/yaml/#workflowrules
+
+workflow:
+ rules:
+ - if: $CI_MERGE_REQUEST_IID
+ - if: $CI_COMMIT_TAG
+ - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 933504ea82f..5816ac3bc54 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -157,7 +157,7 @@ module Gitlab
return unless job[:stage]
unless job[:stage].is_a?(String) && job[:stage].in?(@stages)
- raise ValidationError, "#{name} job: stage parameter should be #{@stages.join(", ")}"
+ raise ValidationError, "#{name} job: chosen stage does not exist; available stages are #{@stages.join(", ")}"
end
end
diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb
index a9961cb8968..3df243e319e 100644
--- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb
+++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb
@@ -35,6 +35,8 @@ module Gitlab
count += relation.delete_all
end
+ ProjectCacheWorker.perform_async(project.id, [], [:lfs_objects_size])
+
log_info("Removed invalid references: #{count}")
end
end
diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb
index 8dd2e9cb1bb..57aeb6c4fb2 100644
--- a/lib/gitlab/code_navigation_path.rb
+++ b/lib/gitlab/code_navigation_path.rb
@@ -5,7 +5,7 @@ module Gitlab
include Gitlab::Utils::StrongMemoize
include Gitlab::Routing
- CODE_NAVIGATION_JOB_NAME = 'code_navigation'
+ LATEST_COMMITS_LIMIT = 10
def initialize(project, commit_sha)
@project = project
@@ -16,7 +16,7 @@ module Gitlab
return if Feature.disabled?(:code_navigation, project)
return unless build
- raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json")
+ raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json", file_type: :lsif)
end
private
@@ -25,10 +25,14 @@ module Gitlab
def build
strong_memoize(:build) do
- artifact = ::Ci::JobArtifact
- .for_sha(commit_sha, project.id)
- .for_job_name(CODE_NAVIGATION_JOB_NAME)
- .last
+ latest_commits_shas =
+ project.repository.commits(commit_sha, limit: LATEST_COMMITS_LIMIT).map(&:sha)
+
+ artifact =
+ ::Ci::JobArtifact
+ .with_file_types(['lsif'])
+ .for_sha(latest_commits_shas, project.id)
+ .last
artifact&.job
end
diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb
new file mode 100644
index 00000000000..795082a10a0
--- /dev/null
+++ b/lib/gitlab/config_checker/external_database_checker.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ConfigChecker
+ module ExternalDatabaseChecker
+ extend self
+
+ # DB is considered deprecated if it is below version 11
+ def db_version_deprecated?
+ Gitlab::Database.version.to_f < 11
+ end
+
+ def check
+ return [] unless db_version_deprecated?
+
+ [
+ {
+ type: 'warning',
+ message: _('Note that PostgreSQL 11 will become the minimum required PostgreSQL version in GitLab 13.0 (May 2020). '\
+ 'PostgreSQL 9.6 and PostgreSQL 10 will no longer be supported in GitLab 13.0. '\
+ 'Please consider upgrading your PostgreSQL version (%{db_version}) soon.') % { db_version: Gitlab::Database.version.to_s }
+ }
+ ]
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb
deleted file mode 100644
index 09b33d01846..00000000000
--- a/lib/gitlab/cycle_analytics/group_stage_summary.rb
+++ /dev/null
@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module CycleAnalytics
- class GroupStageSummary
- attr_reader :group, :current_user, :options
-
- def initialize(group, options:)
- @group = group
- @current_user = options[:current_user]
- @options = options
- end
-
- def data
- [issue_stats,
- deploy_stats,
- deployment_frequency_stats]
- end
-
- private
-
- def issue_stats
- serialize(
- Summary::Group::Issue.new(
- group: group, current_user: current_user, options: options)
- )
- end
-
- def deployments_summary
- @deployments_summary ||=
- Summary::Group::Deploy.new(group: group, options: options)
- end
-
- def deploy_stats
- serialize deployments_summary
- end
-
- def deployment_frequency_stats
- serialize(
- Summary::Group::DeploymentFrequency.new(
- deployments: deployments_summary.value,
- group: group,
- options: options),
- with_unit: true
- )
- end
-
- def serialize(summary_object, with_unit: false)
- AnalyticsSummarySerializer.new.represent(
- summary_object, with_unit: with_unit)
- end
- end
- end
-end
diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb
index 564feb0319f..7559cd376bf 100644
--- a/lib/gitlab/cycle_analytics/stage_summary.rb
+++ b/lib/gitlab/cycle_analytics/stage_summary.rb
@@ -28,8 +28,7 @@ module Gitlab
end
def deployments_summary
- @deployments_summary ||=
- Summary::Deploy.new(project: @project, from: @from, to: @to)
+ @deployments_summary ||= Summary::Deploy.new(project: @project, from: @from, to: @to)
end
def deploy_stats
@@ -39,7 +38,7 @@ module Gitlab
def deployment_frequency_stats
serialize(
Summary::DeploymentFrequency.new(
- deployments: deployments_summary.value,
+ deployments: deployments_summary.value.raw_value,
from: @from,
to: @to),
with_unit: true
diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb
index 76049c6b742..1f426b81800 100644
--- a/lib/gitlab/cycle_analytics/summary/commit.rb
+++ b/lib/gitlab/cycle_analytics/summary/commit.rb
@@ -9,7 +9,7 @@ module Gitlab
end
def value
- @value ||= count_commits
+ @value ||= commits_count ? Value::PrettyNumeric.new(commits_count) : Value::None.new
end
private
@@ -18,10 +18,10 @@ module Gitlab
# a limit. Since we need a commit count, we _can't_ enforce a limit, so
# the easiest way forward is to replicate the relevant portions of the
# `log` function here.
- def count_commits
+ def commits_count
return unless ref
- gitaly_commit_client.commit_count(ref, after: @from, before: @to)
+ @commits_count ||= gitaly_commit_client.commit_count(ref, after: @from, before: @to)
end
def gitaly_commit_client
diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb
index 5ff8d881143..8544ea1a91e 100644
--- a/lib/gitlab/cycle_analytics/summary/deploy.rb
+++ b/lib/gitlab/cycle_analytics/summary/deploy.rb
@@ -4,18 +4,20 @@ module Gitlab
module CycleAnalytics
module Summary
class Deploy < Base
- include Gitlab::Utils::StrongMemoize
-
def title
n_('Deploy', 'Deploys', value)
end
def value
- strong_memoize(:value) do
- query = @project.deployments.success.where("created_at >= ?", @from)
- query = query.where("created_at <= ?", @to) if @to
- query.count
- end
+ @value ||= Value::PrettyNumeric.new(deployments_count)
+ end
+
+ private
+
+ def deployments_count
+ query = @project.deployments.success.where("created_at >= ?", @from)
+ query = query.where("created_at <= ?", @to) if @to
+ query.count
end
end
end
diff --git a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb
index 436dc91bd6b..00676a02a6f 100644
--- a/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb
+++ b/lib/gitlab/cycle_analytics/summary/deployment_frequency.rb
@@ -17,8 +17,7 @@ module Gitlab
end
def value
- @value ||=
- frequency(@deployments, @from, @to || Time.now)
+ @value ||= frequency(@deployments, @from, @to || Time.now)
end
def unit
diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb
deleted file mode 100644
index f1d20d5aefa..00000000000
--- a/lib/gitlab/cycle_analytics/summary/group/base.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module CycleAnalytics
- module Summary
- module Group
- class Base
- attr_reader :group, :options
-
- def initialize(group:, options:)
- @group = group
- @options = options
- end
-
- def title
- raise NotImplementedError.new("Expected #{self.name} to implement title")
- end
-
- def value
- raise NotImplementedError.new("Expected #{self.name} to implement value")
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb
deleted file mode 100644
index 11a9152cf0c..00000000000
--- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module CycleAnalytics
- module Summary
- module Group
- class Deploy < Group::Base
- include GroupProjectsProvider
-
- def title
- n_('Deploy', 'Deploys', value)
- end
-
- def value
- @value ||= find_deployments
- end
-
- private
-
- def find_deployments
- deployments = Deployment.joins(:project).merge(Project.inside_path(group.full_path))
- deployments = deployments.where(projects: { id: options[:projects] }) if options[:projects]
- deployments = deployments.where("deployments.created_at > ?", options[:from])
- deployments = deployments.where("deployments.created_at < ?", options[:to]) if options[:to]
- deployments.success.count
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb b/lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb
deleted file mode 100644
index 9fbbbb5a1ec..00000000000
--- a/lib/gitlab/cycle_analytics/summary/group/deployment_frequency.rb
+++ /dev/null
@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module CycleAnalytics
- module Summary
- module Group
- class DeploymentFrequency < Group::Base
- include GroupProjectsProvider
- include SummaryHelper
-
- def initialize(deployments:, group:, options:)
- @deployments = deployments
-
- super(group: group, options: options)
- end
-
- def title
- _('Deployment Frequency')
- end
-
- def value
- @value ||=
- frequency(@deployments, options[:from], options[:to] || Time.now)
- end
-
- def unit
- _('per day')
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb
deleted file mode 100644
index 4d5ee1d43ca..00000000000
--- a/lib/gitlab/cycle_analytics/summary/group/issue.rb
+++ /dev/null
@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module CycleAnalytics
- module Summary
- module Group
- class Issue < Group::Base
- attr_reader :group, :current_user, :options
-
- def initialize(group:, current_user:, options:)
- @group = group
- @current_user = current_user
- @options = options
- end
-
- def title
- n_('New Issue', 'New Issues', value)
- end
-
- def value
- @value ||= find_issues
- end
-
- private
-
- def find_issues
- issues = IssuesFinder.new(current_user, finder_params).execute
- issues = issues.where(projects: { id: options[:projects] }) if options[:projects]
- issues.count
- end
-
- def finder_params
- {
- group_id: group.id,
- include_subgroups: true,
- created_after: options[:from],
- created_before: options[:to]
- }.compact
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb
index 52892eb5a1a..ce7788590b9 100644
--- a/lib/gitlab/cycle_analytics/summary/issue.rb
+++ b/lib/gitlab/cycle_analytics/summary/issue.rb
@@ -16,7 +16,16 @@ module Gitlab
end
def value
- @value ||= IssuesFinder.new(@current_user, project_id: @project.id, created_after: @from, created_before: @to).execute.count
+ @value ||= Value::PrettyNumeric.new(issues_count)
+ end
+
+ private
+
+ def issues_count
+ IssuesFinder
+ .new(@current_user, project_id: @project.id, created_after: @from, created_before: @to)
+ .execute
+ .count
end
end
end
diff --git a/lib/gitlab/cycle_analytics/summary/value.rb b/lib/gitlab/cycle_analytics/summary/value.rb
new file mode 100644
index 00000000000..ce32132e048
--- /dev/null
+++ b/lib/gitlab/cycle_analytics/summary/value.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module CycleAnalytics
+ module Summary
+ class Value
+ attr_reader :value
+
+ def raw_value
+ value
+ end
+
+ def to_s
+ raise NotImplementedError
+ end
+
+ class None < self
+ def to_s
+ '-'
+ end
+ end
+
+ class Numeric < self
+ def initialize(value)
+ @value = value
+ end
+
+ def to_s
+ value.zero? ? '0' : value.to_s
+ end
+ end
+
+ class PrettyNumeric < Numeric
+ def to_s
+ # 0 is shown as -
+ (value || 0).nonzero? ? super : None.new.to_s
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/cycle_analytics/summary_helper.rb b/lib/gitlab/cycle_analytics/summary_helper.rb
index 06abcd151d4..3cf9f463024 100644
--- a/lib/gitlab/cycle_analytics/summary_helper.rb
+++ b/lib/gitlab/cycle_analytics/summary_helper.rb
@@ -4,10 +4,11 @@ module Gitlab
module CycleAnalytics
module SummaryHelper
def frequency(count, from, to)
- return count if count.zero?
+ return Summary::Value::None.new if count.zero?
freq = (count / days(from, to)).round(1)
- freq.zero? ? '0' : freq
+
+ Summary::Value::Numeric.new(freq)
end
def days(from, to)
diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb
index d64177f9565..85f386594be 100644
--- a/lib/gitlab/danger/changelog.rb
+++ b/lib/gitlab/danger/changelog.rb
@@ -14,10 +14,6 @@ module Gitlab
@found ||= git.added_files.find { |path| path =~ %r{\A(ee/)?(changelogs/unreleased)(-ee)?/} }
end
- def presented_no_changelog_labels
- NO_CHANGELOG_LABELS.map { |label| "~#{label}" }.join(', ')
- end
-
def sanitized_mr_title
gitlab.mr_json["title"].gsub(/^WIP: */, '').gsub(/`/, '\\\`')
end
diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb
index 616c05d0a02..58db2b58560 100644
--- a/lib/gitlab/danger/commit_linter.rb
+++ b/lib/gitlab/danger/commit_linter.rb
@@ -18,7 +18,7 @@ module Gitlab
PROBLEMS = {
subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words",
subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters",
- subject_above_warning: "The %s length is acceptable, but please try to [reduce it to #{WARN_SUBJECT_LENGTH} characters](#{URL_LIMIT_SUBJECT}).",
+ subject_above_warning: "The %s length is acceptable, but please try to [reduce it to #{WARN_SUBJECT_LENGTH} characters](#{URL_LIMIT_SUBJECT})",
subject_starts_with_lowercase: "The %s must start with a capital letter",
subject_ends_with_a_period: "The %s must not end with a period",
separator_missing: "The commit subject and body must be separated by a blank line",
diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb
index e31a6ae5011..a2867087428 100644
--- a/lib/gitlab/danger/emoji_checker.rb
+++ b/lib/gitlab/danger/emoji_checker.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-require 'json'
+require_relative '../json'
module Gitlab
module Danger
@@ -25,8 +25,8 @@ module Gitlab
)}x.freeze
def initialize
- names = JSON.parse(File.read(DIGESTS)).keys +
- JSON.parse(File.read(ALIASES)).keys
+ names = Gitlab::Json.parse(File.read(DIGESTS)).keys +
+ Gitlab::Json.parse(File.read(ALIASES)).keys
@emoji = names.map { |name| ":#{name}:" }
end
diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb
index aa2737262be..0f0af5f777b 100644
--- a/lib/gitlab/danger/helper.rb
+++ b/lib/gitlab/danger/helper.rb
@@ -191,6 +191,23 @@ module Gitlab
gitlab_helper.mr_json['web_url'].include?('/gitlab-org/security/')
end
+ def mr_has_labels?(*labels)
+ return false unless gitlab_helper
+
+ labels = labels.flatten.uniq
+ (labels & gitlab_helper.mr_labels) == labels
+ end
+
+ def labels_list(labels, sep: ', ')
+ labels.map { |label| %Q{~"#{label}"} }.join(sep)
+ end
+
+ def prepare_labels_for_mr(labels)
+ return '' unless labels.any?
+
+ "/label #{labels_list(labels, sep: ' ')}"
+ end
+
private
def has_database_scoped_labels?(current_mr_labels)
diff --git a/lib/gitlab/danger/request_helper.rb b/lib/gitlab/danger/request_helper.rb
index 06da4ed9ad3..ef51c3f2052 100644
--- a/lib/gitlab/danger/request_helper.rb
+++ b/lib/gitlab/danger/request_helper.rb
@@ -16,7 +16,7 @@ module Gitlab
raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}"
end
- JSON.parse(rsp.body)
+ Gitlab::Json.parse(rsp.body)
end
end
end
diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb
index 55476cd9789..651b002d2bf 100644
--- a/lib/gitlab/danger/teammate.rb
+++ b/lib/gitlab/danger/teammate.rb
@@ -1,12 +1,19 @@
# frozen_string_literal: true
require 'cgi'
+require 'set'
module Gitlab
module Danger
class Teammate
attr_reader :name, :username, :role, :projects
+ AT_CAPACITY_EMOJI = Set.new(%w[red_circle]).freeze
+ OOO_EMOJI = Set.new(%w[
+ palm_tree
+ beach beach_umbrella beach_with_umbrella
+ ]).freeze
+
def initialize(options = {})
@username = options['username']
@name = options['name'] || @username
@@ -37,10 +44,14 @@ module Gitlab
end
def status
- api_endpoint = "https://gitlab.com/api/v4/users/#{CGI.escape(username)}/status"
- @status ||= Gitlab::Danger::RequestHelper.http_get_json(api_endpoint)
- rescue Gitlab::Danger::RequestHelper::HTTPError, JSON::ParserError
- nil # better no status than a crashing Danger
+ return @status if defined?(@status)
+
+ @status ||=
+ begin
+ Gitlab::Danger::RequestHelper.http_get_json(status_api_endpoint)
+ rescue Gitlab::Danger::RequestHelper::HTTPError, JSON::ParserError
+ nil # better no status than a crashing Danger
+ end
end
# @return [Boolean]
@@ -50,14 +61,22 @@ module Gitlab
private
+ def status_api_endpoint
+ "https://gitlab.com/api/v4/users/#{CGI.escape(username)}/status"
+ end
+
+ def status_emoji
+ status&.dig("emoji")
+ end
+
# @return [Boolean]
def out_of_office?
- status&.dig("message")&.match?(/OOO/i) || false
+ status&.dig("message")&.match?(/OOO/i) || OOO_EMOJI.include?(status_emoji)
end
# @return [Boolean]
def has_capacity?
- status&.dig("emoji") != 'red_circle'
+ !AT_CAPACITY_EMOJI.include?(status_emoji)
end
def has_capability?(project, category, kind, labels)
diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb
index 9368446fa59..8aee25e9fe6 100644
--- a/lib/gitlab/data_builder/wiki_page.rb
+++ b/lib/gitlab/data_builder/wiki_page.rb
@@ -8,6 +8,9 @@ module Gitlab
def build(wiki_page, user, action)
wiki = wiki_page.wiki
+ # TODO: group hooks https://gitlab.com/gitlab-org/gitlab/-/issues/216904
+ return {} if wiki.container.is_a?(Group)
+
{
object_kind: wiki_page.class.name.underscore,
user: user.hook_attrs,
diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb
index 2359dceae48..ab069ce1da1 100644
--- a/lib/gitlab/database/batch_count.rb
+++ b/lib/gitlab/database/batch_count.rb
@@ -91,11 +91,17 @@ module Gitlab
def batch_fetch(start, finish, mode)
# rubocop:disable GitlabSecurity/PublicSend
- @relation.select(@column).public_send(mode).where(@column => start..(finish - 1)).count
+ @relation.select(@column).public_send(mode).where(between_condition(start, finish)).count
end
private
+ def between_condition(start, finish)
+ return @column.between(start..(finish - 1)) if @column.is_a?(Arel::Attributes::Attribute)
+
+ { @column => start..(finish - 1) }
+ end
+
def actual_start(start)
start || @relation.minimum(@column) || 0
end
diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb
index 6cd90c01ab2..e226ed7613a 100644
--- a/lib/gitlab/database/count/reltuples_count_strategy.rb
+++ b/lib/gitlab/database/count/reltuples_count_strategy.rb
@@ -72,7 +72,7 @@ module Gitlab
# @param [Array] table names
# @returns [Hash] Table name to count mapping (e.g. { 'projects' => 5, 'users' => 100 })
def get_statistics(table_names, check_statistics: true)
- time = 1.hour.ago
+ time = 6.hours.ago
query = PgClass.joins("LEFT JOIN pg_stat_user_tables USING (relname)")
.where(relname: table_names)
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index cf5ff8ddb7b..96be057f77e 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -265,12 +265,19 @@ module Gitlab
# or `RESET ALL` is executed
def disable_statement_timeout
if block_given?
- begin
- execute('SET statement_timeout TO 0')
-
+ if statement_timeout_disabled?
+ # Don't do anything if the statement_timeout is already disabled
+ # Allows for nested calls of disable_statement_timeout without
+ # resetting the timeout too early (before the outer call ends)
yield
- ensure
- execute('RESET ALL')
+ else
+ begin
+ execute('SET statement_timeout TO 0')
+
+ yield
+ ensure
+ execute('RESET ALL')
+ end
end
else
unless transaction_open?
@@ -378,7 +385,7 @@ module Gitlab
# make things _more_ complex).
#
# `batch_column_name` option is for tables without primary key, in this
- # case an other unique integer column can be used. Example: :user_id
+ # case another unique integer column can be used. Example: :user_id
#
# rubocop: disable Metrics/AbcSize
def update_column_in_batches(table, column, value, batch_size: nil, batch_column_name: :id)
@@ -444,66 +451,13 @@ module Gitlab
# Adds a column with a default value without locking an entire table.
#
- # This method runs the following steps:
- #
- # 1. Add the column with a default value of NULL.
- # 2. Change the default value of the column to the specified value.
- # 3. Update all existing rows in batches.
- # 4. Set a `NOT NULL` constraint on the column if desired (the default).
- #
- # These steps ensure a column can be added to a large and commonly used
- # table without locking the entire table for the duration of the table
- # modification.
- #
- # table - The name of the table to update.
- # column - The name of the column to add.
- # type - The column type (e.g. `:integer`).
- # default - The default value for the column.
- # limit - Sets a column limit. For example, for :integer, the default is
- # 4-bytes. Set `limit: 8` to allow 8-byte integers.
- # allow_null - When set to `true` the column will allow NULL values, the
- # default is to not allow NULL values.
- #
- # This method can also take a block which is passed directly to the
- # `update_column_in_batches` method.
- def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, update_column_in_batches_args: {}, &block)
- if transaction_open?
- raise 'add_column_with_default can not be run inside a transaction, ' \
- 'you can disable transactions by calling disable_ddl_transaction! ' \
- 'in the body of your migration class'
- end
-
- disable_statement_timeout do
- transaction do
- if limit
- add_column(table, column, type, default: nil, limit: limit)
- else
- add_column(table, column, type, default: nil)
- end
-
- # Changing the default before the update ensures any newly inserted
- # rows already use the proper default value.
- change_column_default(table, column, default)
- end
-
- begin
- default_after_type_cast = connection.type_cast(default, column_for(table, column))
+ # @deprecated With PostgreSQL 11, adding columns with a default does not lead to a table rewrite anymore.
+ # As such, this method is not needed anymore and the default `add_column` helper should be used.
+ # This helper is subject to be removed in a >13.0 release.
+ def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false)
+ raise 'Deprecated: add_column_with_default does not support being passed blocks anymore' if block_given?
- if update_column_in_batches_args.any?
- update_column_in_batches(table, column, default_after_type_cast, **update_column_in_batches_args, &block)
- else
- update_column_in_batches(table, column, default_after_type_cast, &block)
- end
-
- change_column_null(table, column, false) unless allow_null
- # We want to rescue _all_ exceptions here, even those that don't inherit
- # from StandardError.
- rescue Exception => error # rubocop: disable all
- remove_column(table, column)
-
- raise error
- end
- end
+ add_column(table, column, type, default: default, limit: limit, null: allow_null)
end
# Renames a column without requiring downtime.
@@ -519,14 +473,20 @@ module Gitlab
# new - The new column name.
# type - The type of the new column. If no type is given the old column's
# type is used.
- def rename_column_concurrently(table, old, new, type: nil)
+ # batch_column_name - option is for tables without primary key, in this
+ # case another unique integer column can be used. Example: :user_id
+ def rename_column_concurrently(table, old, new, type: nil, batch_column_name: :id)
+ unless column_exists?(table, batch_column_name)
+ raise "Column #{batch_column_name} does not exist on #{table}"
+ end
+
if transaction_open?
raise 'rename_column_concurrently can not be run inside a transaction'
end
check_trigger_permissions!(table)
- create_column_from(table, old, new, type: type)
+ create_column_from(table, old, new, type: type, batch_column_name: batch_column_name)
install_rename_triggers(table, old, new)
end
@@ -626,14 +586,20 @@ module Gitlab
# new - The new column name.
# type - The type of the old column. If no type is given the new column's
# type is used.
- def undo_cleanup_concurrent_column_rename(table, old, new, type: nil)
+ # batch_column_name - option is for tables without primary key, in this
+ # case another unique integer column can be used. Example: :user_id
+ def undo_cleanup_concurrent_column_rename(table, old, new, type: nil, batch_column_name: :id)
+ unless column_exists?(table, batch_column_name)
+ raise "Column #{batch_column_name} does not exist on #{table}"
+ end
+
if transaction_open?
raise 'undo_cleanup_concurrent_column_rename can not be run inside a transaction'
end
check_trigger_permissions!(table)
- create_column_from(table, new, old, type: type)
+ create_column_from(table, new, old, type: type, batch_column_name: batch_column_name)
install_rename_triggers(table, old, new)
end
@@ -1063,6 +1029,8 @@ into similar problems in the future (e.g. when new tables are created).
# batch_size - The maximum number of rows per job
# other_arguments - Other arguments to send to the job
#
+ # *Returns the final migration delay*
+ #
# Example:
#
# class Route < ActiveRecord::Base
@@ -1079,7 +1047,7 @@ into similar problems in the future (e.g. when new tables are created).
# # do something
# end
# end
- def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_arguments: [])
+ def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE, other_job_arguments: [], initial_delay: 0)
raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id')
# To not overload the worker too much we enforce a minimum interval both
@@ -1088,14 +1056,19 @@ into similar problems in the future (e.g. when new tables are created).
delay_interval = BackgroundMigrationWorker.minimum_interval
end
+ final_delay = 0
+
model_class.each_batch(of: batch_size) do |relation, index|
start_id, end_id = relation.pluck(Arel.sql('MIN(id), MAX(id)')).first
# `BackgroundMigrationWorker.bulk_perform_in` schedules all jobs for
# the same time, which is not helpful in most cases where we wish to
# spread the work over time.
- migrate_in(delay_interval * index, job_class_name, [start_id, end_id] + other_arguments)
+ final_delay = initial_delay + delay_interval * index
+ migrate_in(final_delay, job_class_name, [start_id, end_id] + other_job_arguments)
end
+
+ final_delay
end
# Fetches indexes on a column by name for postgres.
@@ -1315,12 +1288,73 @@ into similar problems in the future (e.g. when new tables are created).
check_constraint_exists?(table, text_limit_name(table, column, name: constraint_name))
end
+ # Migration Helpers for managing not null constraints
+ def add_not_null_constraint(table, column, constraint_name: nil, validate: true)
+ if column_is_nullable?(table, column)
+ add_check_constraint(
+ table,
+ "#{column} IS NOT NULL",
+ not_null_constraint_name(table, column, name: constraint_name),
+ validate: validate
+ )
+ else
+ warning_message = <<~MESSAGE
+ NOT NULL check constraint was not created:
+ column #{table}.#{column} is already defined as `NOT NULL`
+ MESSAGE
+
+ Rails.logger.warn warning_message
+ end
+ end
+
+ def validate_not_null_constraint(table, column, constraint_name: nil)
+ validate_check_constraint(
+ table,
+ not_null_constraint_name(table, column, name: constraint_name)
+ )
+ end
+
+ def remove_not_null_constraint(table, column, constraint_name: nil)
+ remove_check_constraint(
+ table,
+ not_null_constraint_name(table, column, name: constraint_name)
+ )
+ end
+
+ def check_not_null_constraint_exists?(table, column, constraint_name: nil)
+ check_constraint_exists?(
+ table,
+ not_null_constraint_name(table, column, name: constraint_name)
+ )
+ end
+
private
+ def statement_timeout_disabled?
+ # This is a string of the form "100ms" or "0" when disabled
+ connection.select_value('SHOW statement_timeout') == "0"
+ end
+
+ def column_is_nullable?(table, column)
+ # Check if table.column has not been defined with NOT NULL
+ check_sql = <<~SQL
+ SELECT c.is_nullable
+ FROM information_schema.columns c
+ WHERE c.table_name = '#{table}'
+ AND c.column_name = '#{column}'
+ SQL
+
+ connection.select_value(check_sql) == 'YES'
+ end
+
def text_limit_name(table, column, name: nil)
name.presence || check_constraint_name(table, column, 'max_length')
end
+ def not_null_constraint_name(table, column, name: nil)
+ name.presence || check_constraint_name(table, column, 'not_null')
+ end
+
def missing_schema_object_message(table, type, name)
<<~MESSAGE
Could not find #{type} "#{name}" on table "#{table}" which was referenced during the migration.
@@ -1348,7 +1382,7 @@ into similar problems in the future (e.g. when new tables are created).
"ON DELETE #{on_delete.upcase}"
end
- def create_column_from(table, old, new, type: nil)
+ def create_column_from(table, old, new, type: nil, batch_column_name: :id)
old_col = column_for(table, old)
new_type = type || old_col.type
@@ -1362,9 +1396,9 @@ into similar problems in the future (e.g. when new tables are created).
# necessary since we copy over old values further down.
change_column_default(table, new, old_col.default) unless old_col.default.nil?
- update_column_in_batches(table, new, Arel::Table.new(table)[old])
+ update_column_in_batches(table, new, Arel::Table.new(table)[old], batch_column_name: batch_column_name)
- change_column_null(table, new, false) unless old_col.null
+ add_not_null_constraint(table, new) unless old_col.null
copy_indexes(table, old, new)
copy_foreign_keys(table, old, new)
diff --git a/lib/gitlab/database/partitioning_migration_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers.rb
new file mode 100644
index 00000000000..55649ebbf8a
--- /dev/null
+++ b/lib/gitlab/database/partitioning_migration_helpers.rb
@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module PartitioningMigrationHelpers
+ include SchemaHelpers
+
+ def add_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id, on_delete: :cascade)
+ cascade_delete = extract_cascade_option(on_delete)
+
+ update_foreign_keys(from_table, to_table, column, primary_key, cascade_delete) do |current_keys, existing_key, specified_key|
+ if existing_key.nil?
+ unless specified_key.save
+ raise "failed to create foreign key: #{specified_key.errors.full_messages.to_sentence}"
+ end
+
+ current_keys << specified_key
+ else
+ Rails.logger.warn "foreign key not added because it already exists: #{specified_key}" # rubocop:disable Gitlab/RailsLogger
+ current_keys
+ end
+ end
+ end
+
+ def remove_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id)
+ update_foreign_keys(from_table, to_table, column, primary_key) do |current_keys, existing_key, specified_key|
+ if existing_key
+ existing_key.destroy!
+ current_keys.delete(existing_key)
+ else
+ Rails.logger.warn "foreign key not removed because it doesn't exist: #{specified_key}" # rubocop:disable Gitlab/RailsLogger
+ end
+
+ current_keys
+ end
+ end
+
+ def fk_function_name(table)
+ object_name(table, 'fk_cascade_function')
+ end
+
+ def fk_trigger_name(table)
+ object_name(table, 'fk_cascade_trigger')
+ end
+
+ private
+
+ def fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete)
+ PartitionedForeignKey.new(from_table: from_table.to_s, to_table: to_table.to_s, from_column: from_column.to_s,
+ to_column: to_column.to_s, cascade_delete: cascade_delete)
+ end
+
+ def update_foreign_keys(from_table, to_table, from_column, to_column, cascade_delete = nil)
+ if transaction_open?
+ raise 'partitioned foreign key operations can not be run inside a transaction block, ' \
+ 'you can disable transaction blocks by calling disable_ddl_transaction! ' \
+ 'in the body of your migration class'
+ end
+
+ from_column ||= "#{to_table.to_s.singularize}_id"
+ specified_key = fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete)
+
+ current_keys = PartitionedForeignKey.by_referenced_table(to_table).to_a
+ existing_key = find_existing_key(current_keys, specified_key)
+
+ final_keys = yield current_keys, existing_key, specified_key
+
+ fn_name = fk_function_name(to_table)
+ trigger_name = fk_trigger_name(to_table)
+
+ with_lock_retries do
+ drop_trigger(to_table, trigger_name, if_exists: true)
+
+ if final_keys.empty?
+ drop_function(fn_name, if_exists: true)
+ else
+ create_or_replace_fk_function(fn_name, final_keys)
+ create_function_trigger(trigger_name, fn_name, fires: "AFTER DELETE ON #{to_table}")
+ end
+ end
+ end
+
+ def extract_cascade_option(on_delete)
+ case on_delete
+ when :cascade then true
+ when :nullify then false
+ else raise ArgumentError, "invalid option #{on_delete} for :on_delete"
+ end
+ end
+
+ def with_lock_retries(&block)
+ Gitlab::Database::WithLockRetries.new({
+ klass: self.class,
+ logger: Gitlab::BackgroundMigration::Logger
+ }).run(&block)
+ end
+
+ def find_existing_key(keys, key)
+ keys.find { |k| k.from_table == key.from_table && k.from_column == key.from_column }
+ end
+
+ def create_or_replace_fk_function(fn_name, fk_specs)
+ create_trigger_function(fn_name, replace: true) do
+ cascade_statements = build_cascade_statements(fk_specs)
+ cascade_statements << 'RETURN OLD;'
+
+ cascade_statements.join("\n")
+ end
+ end
+
+ def build_cascade_statements(foreign_keys)
+ foreign_keys.map do |fks|
+ if fks.cascade_delete?
+ "DELETE FROM #{fks.from_table} WHERE #{fks.from_column} = OLD.#{fks.to_column};"
+ else
+ "UPDATE #{fks.from_table} SET #{fks.from_column} = NULL WHERE #{fks.from_column} = OLD.#{fks.to_column};"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb
new file mode 100644
index 00000000000..f9a90511f9b
--- /dev/null
+++ b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module PartitioningMigrationHelpers
+ class PartitionedForeignKey < ApplicationRecord
+ validates_with PartitionedForeignKeyValidator
+
+ scope :by_referenced_table, ->(table) { where(to_table: table) }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb
new file mode 100644
index 00000000000..089cf2b8931
--- /dev/null
+++ b/lib/gitlab/database/partitioning_migration_helpers/partitioned_foreign_key_validator.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module PartitioningMigrationHelpers
+ class PartitionedForeignKeyValidator < ActiveModel::Validator
+ def validate(record)
+ validate_key_part(record, :from_table, :from_column)
+ validate_key_part(record, :to_table, :to_column)
+ end
+
+ private
+
+ def validate_key_part(record, table_field, column_field)
+ if !connection.table_exists?(record[table_field])
+ record.errors.add(table_field, 'must be a valid table')
+ elsif !connection.column_exists?(record[table_field], record[column_field])
+ record.errors.add(column_field, 'must be a valid column')
+ end
+ end
+
+ def connection
+ ActiveRecord::Base.connection
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
index 565f34b78b7..2c9d0d6c0d1 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -157,7 +157,7 @@ module Gitlab
failed_reverts = []
while rename_info = redis.lpop(key)
- path_before_rename, path_after_rename = JSON.parse(rename_info)
+ path_before_rename, path_after_rename = Gitlab::Json.parse(rename_info)
say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}"
begin
yield(path_before_rename, path_after_rename)
diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb
new file mode 100644
index 00000000000..f8d01c78ae8
--- /dev/null
+++ b/lib/gitlab/database/schema_helpers.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module SchemaHelpers
+ def create_trigger_function(name, replace: true)
+ replace_clause = optional_clause(replace, "OR REPLACE")
+ execute(<<~SQL)
+ CREATE #{replace_clause} FUNCTION #{name}()
+ RETURNS TRIGGER AS
+ $$
+ BEGIN
+ #{yield}
+ END
+ $$ LANGUAGE PLPGSQL
+ SQL
+ end
+
+ def create_function_trigger(name, fn_name, fires: nil)
+ execute(<<~SQL)
+ CREATE TRIGGER #{name}
+ #{fires}
+ FOR EACH ROW
+ EXECUTE PROCEDURE #{fn_name}()
+ SQL
+ end
+
+ def drop_function(name, if_exists: true)
+ exists_clause = optional_clause(if_exists, "IF EXISTS")
+ execute("DROP FUNCTION #{exists_clause} #{name}()")
+ end
+
+ def drop_trigger(table_name, name, if_exists: true)
+ exists_clause = optional_clause(if_exists, "IF EXISTS")
+ execute("DROP TRIGGER #{exists_clause} #{name} ON #{table_name}")
+ end
+
+ def object_name(table, type)
+ identifier = "#{table}_#{type}"
+ hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10)
+
+ "#{type}_#{hashed_identifier}"
+ end
+
+ private
+
+ def optional_clause(flag, clause)
+ flag ? clause : ""
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/with_lock_retries.rb b/lib/gitlab/database/with_lock_retries.rb
index 2f36bfa1480..bebcba6f42e 100644
--- a/lib/gitlab/database/with_lock_retries.rb
+++ b/lib/gitlab/database/with_lock_retries.rb
@@ -78,12 +78,18 @@ module Gitlab
run_block_with_transaction
rescue ActiveRecord::LockWaitTimeout
if retry_with_lock_timeout?
+ disable_idle_in_transaction_timeout
wait_until_next_retry
+ reset_db_settings
retry
else
+ reset_db_settings
run_block_without_lock_timeout
end
+
+ ensure
+ reset_db_settings
end
end
@@ -153,6 +159,14 @@ module Gitlab
def current_sleep_time_in_seconds
timing_configuration[current_iteration - 1][1].to_f
end
+
+ def disable_idle_in_transaction_timeout
+ execute("SET LOCAL idle_in_transaction_session_timeout TO '0'")
+ end
+
+ def reset_db_settings
+ execute('RESET idle_in_transaction_session_timeout; RESET lock_timeout')
+ end
end
end
end
diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb
index 298d214df61..86dc7efb0d9 100644
--- a/lib/gitlab/dependency_linker/json_linker.rb
+++ b/lib/gitlab/dependency_linker/json_linker.rb
@@ -39,7 +39,7 @@ module Gitlab
end
def json
- @json ||= JSON.parse(plain_text) rescue nil
+ @json ||= Gitlab::Json.parse(plain_text) rescue nil
end
end
end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index 4fc5bfddf0c..d1398ddb642 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -314,6 +314,10 @@ module Gitlab
@rich_viewer = rich_viewer_class&.new(self)
end
+ def alternate_viewer
+ alternate_viewer_class&.new(self)
+ end
+
def rendered_as_text?(ignore_errors: true)
simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?)
end
@@ -353,7 +357,7 @@ module Gitlab
def fetch_blob(sha, path)
return unless sha
- Blob.lazy(repository.project, sha, path)
+ Blob.lazy(repository, sha, path)
end
def total_blob_lines(blob)
@@ -419,6 +423,17 @@ module Gitlab
return if collapsed?
return unless diffable?
return unless modified_file?
+
+ find_renderable_viewer_class(classes)
+ end
+
+ def alternate_viewer_class
+ return unless viewer.class == DiffViewer::Renamed
+
+ find_renderable_viewer_class(RICH_VIEWERS) || (DiffViewer::Text if text?)
+ end
+
+ def find_renderable_viewer_class(classes)
return if different_type? || external_storage_error?
verify_binary = !stored_externally?
diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb
index 728457b3139..9ea9bdfdf15 100644
--- a/lib/gitlab/diff/formatters/text_formatter.rb
+++ b/lib/gitlab/diff/formatters/text_formatter.rb
@@ -45,7 +45,8 @@ module Gitlab
def ==(other)
other.is_a?(self.class) &&
new_line == other.new_line &&
- old_line == other.old_line
+ old_line == other.old_line &&
+ line_range == other.line_range
end
end
end
diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb
index 01ec9798fe4..ccf09b37b9b 100644
--- a/lib/gitlab/diff/highlight_cache.rb
+++ b/lib/gitlab/diff/highlight_cache.rb
@@ -156,7 +156,7 @@ module Gitlab
end
results.map! do |result|
- JSON.parse(extract_data(result), symbolize_names: true) unless result.nil?
+ Gitlab::Json.parse(extract_data(result), symbolize_names: true) unless result.nil?
end
file_paths.zip(results).to_h
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index 8b99fd5cd42..10ad23b7774 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -68,7 +68,7 @@ module Gitlab
end
def to_json(opts = nil)
- JSON.generate(formatter.to_h, opts)
+ Gitlab::Json.generate(formatter.to_h, opts)
end
def as_json(opts = nil)
diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb
index 1f64883cb69..75d5a5df74b 100644
--- a/lib/gitlab/discussions_diff/highlight_cache.rb
+++ b/lib/gitlab/discussions_diff/highlight_cache.rb
@@ -42,7 +42,7 @@ module Gitlab
content.map! do |lines|
next unless lines
- JSON.parse(lines).map! do |line|
+ Gitlab::Json.parse(lines).map! do |line|
Gitlab::Diff::Line.safe_init_from_hash(line)
end
end
diff --git a/lib/gitlab/elasticsearch/logs/lines.rb b/lib/gitlab/elasticsearch/logs/lines.rb
index fb32a6c9fcd..ff9185dd331 100644
--- a/lib/gitlab/elasticsearch/logs/lines.rb
+++ b/lib/gitlab/elasticsearch/logs/lines.rb
@@ -13,7 +13,7 @@ module Gitlab
@client = client
end
- def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil)
+ def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil, chart_above_v2: true)
query = { bool: { must: [] } }.tap do |q|
filter_pod_name(q, pod_name)
filter_namespace(q, namespace)
@@ -22,7 +22,7 @@ module Gitlab
filter_times(q, start_time, end_time)
end
- body = build_body(query, cursor)
+ body = build_body(query, cursor, chart_above_v2)
response = @client.search body: body
format_response(response)
@@ -30,13 +30,14 @@ module Gitlab
private
- def build_body(query, cursor = nil)
+ def build_body(query, cursor = nil, chart_above_v2 = true)
+ offset_field = chart_above_v2 ? "log.offset" : "offset"
body = {
query: query,
# reverse order so we can query N-most recent records
sort: [
{ "@timestamp": { order: :desc } },
- { "offset": { order: :desc } }
+ { "#{offset_field}": { order: :desc } }
],
# only return these fields in the response
_source: ["@timestamp", "message", "kubernetes.pod.name"],
diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb
index e9a7c9bcf5c..7f8dd815103 100644
--- a/lib/gitlab/email/handler.rb
+++ b/lib/gitlab/email/handler.rb
@@ -3,18 +3,16 @@
module Gitlab
module Email
module Handler
- prepend_if_ee('::EE::Gitlab::Email::Handler') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
def self.handlers
@handlers ||= load_handlers
end
def self.load_handlers
[
- UnsubscribeHandler,
CreateNoteHandler,
- CreateMergeRequestHandler,
- CreateIssueHandler
+ CreateIssueHandler,
+ UnsubscribeHandler,
+ CreateMergeRequestHandler
]
end
@@ -27,3 +25,5 @@ module Gitlab
end
end
end
+
+Gitlab::Email::Handler.prepend_if_ee('::EE::Gitlab::Email::Handler')
diff --git a/lib/gitlab/email/hook/smime_signature_interceptor.rb b/lib/gitlab/email/hook/smime_signature_interceptor.rb
index 61c9c984f8e..fe39589d019 100644
--- a/lib/gitlab/email/hook/smime_signature_interceptor.rb
+++ b/lib/gitlab/email/hook/smime_signature_interceptor.rb
@@ -10,6 +10,7 @@ module Gitlab
signed_message = Gitlab::Email::Smime::Signer.sign(
cert: certificate.cert,
key: certificate.key,
+ ca_certs: certificate.ca_certs,
data: message.encoded)
signed_email = Mail.new(signed_message)
@@ -21,7 +22,7 @@ module Gitlab
private
def certificate
- @certificate ||= Gitlab::Email::Smime::Certificate.from_files(key_path, cert_path)
+ @certificate ||= Gitlab::Email::Smime::Certificate.from_files(key_path, cert_path, ca_certs_path)
end
def key_path
@@ -32,6 +33,10 @@ module Gitlab
Gitlab.config.gitlab.email_smime.cert_file
end
+ def ca_certs_path
+ Gitlab.config.gitlab.email_smime.ca_certs_file
+ end
+
def overwrite_body(message, signed_email)
# since this is a multipart email, assignment to nil is important,
# otherwise Message#body will add a new mail part
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index ec412e7a8b1..d55cf3202a6 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -52,7 +52,7 @@ module Gitlab
end
def compare
- @opts[:compare] if @opts[:compare]
+ @opts[:compare]
end
def diff_refs
diff --git a/lib/gitlab/email/smime/certificate.rb b/lib/gitlab/email/smime/certificate.rb
index 59d7b0c3c5b..3607b95b4bc 100644
--- a/lib/gitlab/email/smime/certificate.rb
+++ b/lib/gitlab/email/smime/certificate.rb
@@ -4,29 +4,53 @@ module Gitlab
module Email
module Smime
class Certificate
- attr_reader :key, :cert
+ CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze
+
+ attr_reader :key, :cert, :ca_certs
def key_string
- @key.to_s
+ key.to_s
end
def cert_string
- @cert.to_pem
+ cert.to_pem
+ end
+
+ def ca_certs_string
+ ca_certs.map(&:to_pem).join('\n') unless ca_certs.blank?
end
- def self.from_strings(key_string, cert_string)
+ def self.from_strings(key_string, cert_string, ca_certs_string = nil)
key = OpenSSL::PKey::RSA.new(key_string)
cert = OpenSSL::X509::Certificate.new(cert_string)
- new(key, cert)
+ ca_certs = load_ca_certs_bundle(ca_certs_string)
+
+ new(key, cert, ca_certs)
end
- def self.from_files(key_path, cert_path)
- from_strings(File.read(key_path), File.read(cert_path))
+ def self.from_files(key_path, cert_path, ca_certs_path = nil)
+ ca_certs_string = File.read(ca_certs_path) if ca_certs_path
+
+ from_strings(File.read(key_path), File.read(cert_path), ca_certs_string)
+ end
+
+ # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found
+ #
+ # Ruby OpenSSL::X509::Certificate.new will only load the first
+ # certificate if a bundle is presented, this allows to parse multiple certs
+ # in the same file
+ def self.load_ca_certs_bundle(ca_certs_string)
+ return [] unless ca_certs_string
+
+ ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string|
+ OpenSSL::X509::Certificate.new(ca_cert_string)
+ end
end
- def initialize(key, cert)
+ def initialize(key, cert, ca_certs = nil)
@key = key
@cert = cert
+ @ca_certs = ca_certs
end
end
end
diff --git a/lib/gitlab/email/smime/signer.rb b/lib/gitlab/email/smime/signer.rb
index db03e383ecf..6a445730463 100644
--- a/lib/gitlab/email/smime/signer.rb
+++ b/lib/gitlab/email/smime/signer.rb
@@ -7,19 +7,32 @@ module Gitlab
module Smime
# Tooling for signing and verifying data with SMIME
class Signer
- def self.sign(cert:, key:, data:)
- signed_data = OpenSSL::PKCS7.sign(cert, key, data, nil, OpenSSL::PKCS7::DETACHED)
+ # The `ca_certs` parameter, if provided, is an array of CA certificates
+ # that will be attached in the signature together with the main `cert`.
+ # This will be typically intermediate CAs
+ def self.sign(cert:, key:, ca_certs: nil, data:)
+ signed_data = OpenSSL::PKCS7.sign(cert, key, data, Array.wrap(ca_certs), OpenSSL::PKCS7::DETACHED)
OpenSSL::PKCS7.write_smime(signed_data)
end
- # return nil if data cannot be verified, otherwise the signed content data
- def self.verify_signature(cert:, ca_cert: nil, signed_data:)
+ # Return nil if data cannot be verified, otherwise the signed content data
+ #
+ # Be careful with the `ca_certs` parameter, it will implicitly trust all the CAs
+ # in the array by creating a trusted store, stopping validation at the first match
+ # This is relevant when using intermediate CAs, `ca_certs` should only
+ # include the trusted, root CA
+ def self.verify_signature(ca_certs: nil, signed_data:)
store = OpenSSL::X509::Store.new
store.set_default_paths
- store.add_cert(ca_cert) if ca_cert
+ Array.wrap(ca_certs).compact.each { |ca_cert| store.add_cert(ca_cert) }
signed_smime = OpenSSL::PKCS7.read_smime(signed_data)
- signed_smime if signed_smime.verify([cert], store)
+
+ # The S/MIME certificate(s) are included in the message and the trusted
+ # CAs are in the store parameter, so we pass no certs as parameters
+ # to `PKCS7.verify`
+ # See https://www.openssl.org/docs/manmaster/man3/PKCS7_verify.html
+ signed_smime if signed_smime.verify(nil, store)
end
end
end
diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb
index 305bcada2f5..bcf92b35720 100644
--- a/lib/gitlab/emoji.rb
+++ b/lib/gitlab/emoji.rb
@@ -21,7 +21,7 @@ module Gitlab
end
def emojis_aliases
- @emoji_aliases ||= JSON.parse(File.read(Rails.root.join('fixtures', 'emojis', 'aliases.json')))
+ @emoji_aliases ||= Gitlab::Json.parse(File.read(Rails.root.join('fixtures', 'emojis', 'aliases.json')))
end
def emoji_filename(name)
@@ -63,7 +63,7 @@ module Gitlab
def emoji_unicode_versions_by_name
@emoji_unicode_versions_by_name ||=
- JSON.parse(File.read(Rails.root.join('fixtures', 'emojis', 'emoji-unicode-version-map.json')))
+ Gitlab::Json.parse(File.read(Rails.root.join('fixtures', 'emojis', 'emoji-unicode-version-map.json')))
end
end
end
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index 425ef5d738a..1d2c1c69423 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -12,6 +12,9 @@ module Gitlab
# ExclusiveLease.
#
class ExclusiveLease
+ PREFIX = 'gitlab:exclusive_lease'
+ NoKey = Class.new(ArgumentError)
+
LUA_CANCEL_SCRIPT = <<~EOS.freeze
local key, uuid = KEYS[1], ARGV[1]
if redis.call("get", key) == uuid then
@@ -34,13 +37,21 @@ module Gitlab
end
def self.cancel(key, uuid)
+ return unless key.present?
+
Gitlab::Redis::SharedState.with do |redis|
- redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid])
+ redis.eval(LUA_CANCEL_SCRIPT, keys: [ensure_prefixed_key(key)], argv: [uuid])
end
end
def self.redis_shared_state_key(key)
- "gitlab:exclusive_lease:#{key}"
+ "#{PREFIX}:#{key}"
+ end
+
+ def self.ensure_prefixed_key(key)
+ raise NoKey unless key.present?
+
+ key.start_with?(PREFIX) ? key : redis_shared_state_key(key)
end
# Removes any existing exclusive_lease from redis
@@ -94,6 +105,11 @@ module Gitlab
ttl if ttl.positive?
end
end
+
+ # Gives up this lease, allowing it to be obtained by others.
+ def cancel
+ self.class.cancel(@redis_shared_state_key, @uuid)
+ end
end
end
diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb
index 61eb030563d..10762d83588 100644
--- a/lib/gitlab/exclusive_lease_helpers.rb
+++ b/lib/gitlab/exclusive_lease_helpers.rb
@@ -6,29 +6,38 @@ module Gitlab
FailedToObtainLockError = Class.new(StandardError)
##
- # This helper method blocks a process/thread until the other process cancel the obrainted lease key.
+ # This helper method blocks a process/thread until the lease can be acquired, either due to
+ # the lease TTL expiring, or due to the current holder explicitly releasing
+ # their hold.
#
- # Note: It's basically discouraged to use this method in the unicorn's thread,
- # because it holds the connection until all `retries` is consumed.
+ # If the lease cannot be obtained, raises `FailedToObtainLockError`.
+ #
+ # @param [String] key The lock the thread will try to acquire. Only one thread
+ # in one process across all Rails instances can hold this named lock at any
+ # one time.
+ # @param [Float] ttl: The length of time the lock will be valid for. The lock
+ # will be automatically be released after this time, so any work should be
+ # completed within this time.
+ # @param [Integer] retries: The maximum number of times we will re-attempt
+ # to acquire the lock. The maximum number of attempts will be `retries + 1`:
+ # one for the initial attempt, and then one for every re-try.
+ # @param [Float|Proc] sleep_sec: Either a number of seconds to sleep, or
+ # a proc that computes the sleep time given the number of preceding attempts
+ # (from 1 to retries - 1)
+ #
+ # Note: It's basically discouraged to use this method in a unicorn thread,
+ # because this ties up all thread related resources until all `retries` are consumed.
# This could potentially eat up all connection pools.
def in_lock(key, ttl: 1.minute, retries: 10, sleep_sec: 0.01.seconds)
raise ArgumentError, 'Key needs to be specified' unless key
- lease = Gitlab::ExclusiveLease.new(key, timeout: ttl)
- retried = false
-
- until uuid = lease.try_obtain
- # Keep trying until we obtain the lease. To prevent hammering Redis too
- # much we'll wait for a bit.
- sleep(sleep_sec)
- (retries -= 1) < 0 ? break : retried ||= true
- end
+ lease = SleepingLock.new(key, timeout: ttl, delay: sleep_sec)
- raise FailedToObtainLockError, 'Failed to obtain a lock' unless uuid
+ lease.obtain(1 + retries)
- yield(retried)
+ yield(lease.retried?)
ensure
- Gitlab::ExclusiveLease.cancel(key, uuid)
+ lease&.cancel
end
end
end
diff --git a/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb b/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb
new file mode 100644
index 00000000000..8213c9bc042
--- /dev/null
+++ b/lib/gitlab/exclusive_lease_helpers/sleeping_lock.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ExclusiveLeaseHelpers
+ # Wrapper around ExclusiveLease that adds retry logic
+ class SleepingLock
+ delegate :cancel, to: :@lease
+
+ def initialize(key, timeout:, delay:)
+ @lease = ::Gitlab::ExclusiveLease.new(key, timeout: timeout)
+ @delay = delay
+ @attempts = 0
+ end
+
+ def obtain(max_attempts)
+ until held?
+ raise FailedToObtainLockError, 'Failed to obtain a lock' if attempts >= max_attempts
+
+ sleep(sleep_sec) unless first_attempt?
+ try_obtain
+ end
+ end
+
+ def retried?
+ attempts > 1
+ end
+
+ private
+
+ attr_reader :delay, :attempts
+
+ def held?
+ @uuid.present?
+ end
+
+ def try_obtain
+ @uuid ||= @lease.try_obtain
+ @attempts += 1
+ end
+
+ def first_attempt?
+ attempts.zero?
+ end
+
+ def sleep_sec
+ delay.respond_to?(:call) ? delay.call(attempts) : delay
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
index 68f3c9e332e..3495b4a0b72 100644
--- a/lib/gitlab/experimentation.rb
+++ b/lib/gitlab/experimentation.rb
@@ -2,48 +2,53 @@
# == Experimentation
#
-# Utility module used for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant.
-# The feature_toggle and environment keys are optional. If the feature_toggle is not set, a feature with the name of
-# the experiment will be checked, with a default value of true. The enabled_ratio is required and should be
-# the ratio for the number of users for which this experiment is enabled. For example: a ratio of 0.1 will
-# enable the experiment for 10% of the users (determined by the `experimentation_subject_index`).
+# Utility module for A/B testing experimental features. Define your experiments in the `EXPERIMENTS` constant.
+# Experiment options:
+# - environment (optional, defaults to enabled for development and GitLab.com)
+# - tracking_category (optional, used to set the category when tracking an experiment event)
+#
+# The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html),
+# which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes.
+#
+# To enable the experiment for 10% of the users:
+#
+# chatops: `/chatops run feature set experiment_key_experiment_percentage 10`
+# console: `Feature.get(:experiment_key_experiment_percentage).enable_percentage_of_time(10)`
+#
+# To disable the experiment:
+#
+# chatops: `/chatops run feature delete experiment_key_experiment_percentage`
+# console: `Feature.get(:experiment_key_experiment_percentage).remove`
+#
+# To check the current rollout percentage:
+#
+# chatops: `/chatops run feature get experiment_key_experiment_percentage`
+# console: `Feature.get(:experiment_key_experiment_percentage).percentage_of_time_value`
#
module Gitlab
module Experimentation
EXPERIMENTS = {
signup_flow: {
- feature_toggle: :experimental_separate_sign_up_flow,
- environment: ::Gitlab.dev_env_or_com?,
- enabled_ratio: 1,
tracking_category: 'Growth::Acquisition::Experiment::SignUpFlow'
},
- paid_signup_flow: {
- feature_toggle: :paid_signup_flow,
- environment: ::Gitlab.dev_env_or_com?,
- enabled_ratio: 1,
- tracking_category: 'Growth::Acquisition::Experiment::PaidSignUpFlow'
+ onboarding_issues: {
+ tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues'
},
suggest_pipeline: {
- feature_toggle: :suggest_pipeline,
- environment: ::Gitlab.dev_env_or_com?,
- enabled_ratio: 0.1,
tracking_category: 'Growth::Expansion::Experiment::SuggestPipeline'
},
ci_notification_dot: {
- feature_toggle: :ci_notification_dot,
- environment: ::Gitlab.dev_env_or_com?,
- enabled_ratio: 0.1,
tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot'
},
buy_ci_minutes_version_a: {
- feature_toggle: :buy_ci_minutes_version_a,
- environment: ::Gitlab.dev_env_or_com?,
- enabled_ratio: 0.2,
tracking_category: 'Growth::Expansion::Experiment::BuyCiMinutesVersionA'
+ },
+ upgrade_link_in_user_menu_a: {
+ tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA'
}
}.freeze
- # Controller concern that checks if an experimentation_subject_id cookie is present and sets it if absent.
+ # Controller concern that checks if an `experimentation_subject_id cookie` is present and sets it if absent.
# Used for A/B testing of experimental features. Exposes the `experiment_enabled?(experiment_name)` method
# to controllers and views. It returns true when the experiment is enabled and the user is selected as part
# of the experimental group.
@@ -144,7 +149,7 @@ module Gitlab
return false unless EXPERIMENTS.key?(experiment_key)
experiment = experiment(experiment_key)
- experiment.feature_toggle_enabled? && experiment.enabled_for_environment?
+ experiment.enabled? && experiment.enabled_for_environment?
end
def enabled_for_user?(experiment_key, experimentation_subject_index)
@@ -153,23 +158,28 @@ module Gitlab
end
end
- Experiment = Struct.new(:key, :feature_toggle, :environment, :enabled_ratio, :tracking_category, keyword_init: true) do
- def feature_toggle_enabled?
- return Feature.enabled?(key, default_enabled: true) if feature_toggle.nil?
-
- Feature.enabled?(feature_toggle)
+ Experiment = Struct.new(:key, :environment, :tracking_category, keyword_init: true) do
+ def enabled?
+ experiment_percentage.positive?
end
def enabled_for_environment?
- return true if environment.nil?
+ return ::Gitlab.dev_env_or_com? if environment.nil?
environment
end
def enabled_for_experimentation_subject?(experimentation_subject_index)
- return false if enabled_ratio.nil? || experimentation_subject_index.blank?
+ return false if experimentation_subject_index.blank?
+
+ experimentation_subject_index <= experiment_percentage
+ end
+
+ private
- experimentation_subject_index <= enabled_ratio * 100
+ # When a feature does not exist, the `percentage_of_time_value` method will return 0
+ def experiment_percentage
+ @experiment_percentage ||= Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value
end
end
end
diff --git a/lib/gitlab/external_authorization/response.rb b/lib/gitlab/external_authorization/response.rb
index 4f3fe5882db..04f9688fad0 100644
--- a/lib/gitlab/external_authorization/response.rb
+++ b/lib/gitlab/external_authorization/response.rb
@@ -28,7 +28,7 @@ module Gitlab
end
def parse_response!
- JSON.parse(@excon_response.body)
+ Gitlab::Json.parse(@excon_response.body)
rescue JSON::JSONError
# The JSON response is optional, so don't fail when it's missing
nil
diff --git a/lib/gitlab/git/attributes_parser.rb b/lib/gitlab/git/attributes_parser.rb
index 8b9d74ae8e7..630b1aba2f5 100644
--- a/lib/gitlab/git/attributes_parser.rb
+++ b/lib/gitlab/git/attributes_parser.rb
@@ -85,6 +85,8 @@ module Gitlab
yield line.strip
end
+ # Catch invalid byte sequences
+ rescue ArgumentError
end
private
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 605084f1ec2..a554dc0b667 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -57,11 +57,8 @@ module Gitlab
# Already a commit?
return commit_id if commit_id.is_a?(Gitlab::Git::Commit)
- # Some weird thing?
- return unless commit_id.is_a?(String)
-
# This saves us an RPC round trip.
- return if commit_id.include?(':')
+ return unless valid?(commit_id)
commit = find_commit(repo, commit_id)
@@ -431,6 +428,15 @@ module Gitlab
def fetch_body_from_gitaly
self.class.get_message(@repository, id)
end
+
+ def self.valid?(commit_id)
+ commit_id.is_a?(String) && !(
+ commit_id.start_with?('-') ||
+ commit_id.include?(':') ||
+ commit_id.include?("\x00") ||
+ commit_id.match?(/\s/)
+ )
+ end
end
end
end
diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb
index 08dbd52e3fb..da86d6baf4a 100644
--- a/lib/gitlab/git/tag.rb
+++ b/lib/gitlab/git/tag.rb
@@ -66,6 +66,27 @@ module Gitlab
@raw_tag.tagger
end
+ def has_signature?
+ signature_type != :NONE
+ end
+
+ def signature_type
+ @raw_tag.signature_type || :NONE
+ end
+
+ def signature
+ return unless has_signature?
+
+ case signature_type
+ when :PGP
+ nil # not implemented, see https://gitlab.com/gitlab-org/gitlab/issues/19260
+ when :X509
+ X509::Tag.new(@raw_tag).signature
+ else
+ nil
+ end
+ end
+
private
def message_from_gitaly_tag
diff --git a/lib/gitlab/git_access_design.rb b/lib/gitlab/git_access_design.rb
new file mode 100644
index 00000000000..36604bd0b3b
--- /dev/null
+++ b/lib/gitlab/git_access_design.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ class GitAccessDesign < GitAccess
+ def check(_cmd, _changes)
+ check_protocol!
+ check_can_create_design!
+
+ success_result
+ end
+
+ private
+
+ def check_protocol!
+ if protocol != 'web'
+ raise ::Gitlab::GitAccess::ForbiddenError, "Designs are only accessible using the web interface"
+ end
+ end
+
+ def check_can_create_design!
+ unless user&.can?(:create_design, project)
+ raise ::Gitlab::GitAccess::ForbiddenError, "You are not allowed to manage designs of this project"
+ end
+ end
+ end
+end
+
+Gitlab::GitAccessDesign.prepend_if_ee('EE::Gitlab::GitAccessDesign')
diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb
index d05ffe9b508..70db4271469 100644
--- a/lib/gitlab/git_access_snippet.rb
+++ b/lib/gitlab/git_access_snippet.rb
@@ -61,6 +61,13 @@ module Gitlab
end
end
+ override :can_read_project?
+ def can_read_project?
+ return true if user&.migration_bot?
+
+ super
+ end
+
override :check_download_access!
def check_download_access!
passed = guest_can_download_code? || user_can_download_code?
@@ -99,7 +106,7 @@ module Gitlab
def check_single_change_access(change)
Checks::SnippetCheck.new(change, logger: logger).validate!
- Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet::MAX_FILE_COUNT, logger: logger).validate!
+ Checks::PushFileCountCheck.new(change, repository: repository, limit: Snippet.max_file_limit(user), logger: logger).validate!
rescue Checks::TimedLogger::TimeoutError
raise TimeoutError, logger.full_message
end
@@ -121,5 +128,12 @@ module Gitlab
def check_custom_action(cmd)
nil
end
+
+ override :check_size_limit?
+ def check_size_limit?
+ return false if user&.migration_bot?
+
+ super
+ end
end
end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 697c943b4ec..3aaed0edb87 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -130,7 +130,7 @@ module Gitlab
end
def self.address_metadata(storage)
- Base64.strict_encode64(JSON.dump(storage => connection_data(storage)))
+ Base64.strict_encode64(Gitlab::Json.dump(storage => connection_data(storage)))
end
def self.connection_data(storage)
@@ -209,7 +209,8 @@ module Gitlab
end
def self.query_time
- SafeRequestStore[:gitaly_query_time] ||= 0
+ query_time = SafeRequestStore[:gitaly_query_time] ||= 0
+ query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
end
def self.query_time=(duration)
@@ -457,7 +458,7 @@ module Gitlab
def self.filesystem_id_from_disk(storage)
metadata_file = File.read(storage_metadata_file_path(storage))
- metadata_hash = JSON.parse(metadata_file)
+ metadata_hash = Gitlab::Json.parse(metadata_file)
metadata_hash['gitaly_filesystem_id']
rescue Errno::ENOENT, Errno::EACCES, JSON::ParserError
nil
diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb
index 8fb99c996b3..1b4750da868 100644
--- a/lib/gitlab/github_import/parallel_importer.rb
+++ b/lib/gitlab/github_import/parallel_importer.rb
@@ -5,8 +5,6 @@ module Gitlab
# The ParallelImporter schedules the importing of a GitHub project using
# Sidekiq.
class ParallelImporter
- prepend_if_ee('::EE::Gitlab::GithubImport::ParallelImporter') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
attr_reader :project
def self.async?
@@ -41,3 +39,5 @@ module Gitlab
end
end
end
+
+Gitlab::GithubImport::ParallelImporter.prepend_if_ee('::EE::Gitlab::GithubImport::ParallelImporter')
diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb
index 26440e6f82d..8abefad1ef3 100644
--- a/lib/gitlab/gl_repository.rb
+++ b/lib/gitlab/gl_repository.rb
@@ -23,11 +23,18 @@ module Gitlab
project_resolver: -> (snippet) { snippet&.project },
guest_read_ability: :read_snippet
).freeze
+ DESIGN = ::Gitlab::GlRepository::RepoType.new(
+ name: :design,
+ access_checker_class: ::Gitlab::GitAccessDesign,
+ repository_resolver: -> (project) { ::DesignManagement::Repository.new(project) },
+ suffix: :design
+ ).freeze
TYPES = {
PROJECT.name.to_s => PROJECT,
WIKI.name.to_s => WIKI,
- SNIPPET.name.to_s => SNIPPET
+ SNIPPET.name.to_s => SNIPPET,
+ DESIGN.name.to_s => DESIGN
}.freeze
def self.types
@@ -58,5 +65,3 @@ module Gitlab
private_class_method :instance
end
end
-
-Gitlab::GlRepository.prepend_if_ee('::EE::Gitlab::GlRepository')
diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb
index 052ce578881..64c329b15ae 100644
--- a/lib/gitlab/gl_repository/repo_type.rb
+++ b/lib/gitlab/gl_repository/repo_type.rb
@@ -57,6 +57,10 @@ module Gitlab
self == SNIPPET
end
+ def design?
+ self == DESIGN
+ end
+
def path_suffix
suffix ? ".#{suffix}" : ''
end
@@ -87,5 +91,3 @@ module Gitlab
end
end
end
-
-Gitlab::GlRepository::RepoType.prepend_if_ee('EE::Gitlab::GlRepository::RepoType')
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index e4e69241bd9..fbbfed7279d 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -42,11 +42,11 @@ module Gitlab
# Initialize gon.features with any flags that should be
# made globally available to the frontend
- push_frontend_feature_flag(:snippets_vue, default_enabled: false)
- push_frontend_feature_flag(:monaco_snippets, default_enabled: false)
+ push_frontend_feature_flag(:snippets_vue, default_enabled: true)
push_frontend_feature_flag(:monaco_blobs, default_enabled: false)
push_frontend_feature_flag(:monaco_ci, default_enabled: false)
push_frontend_feature_flag(:snippets_edit_vue, default_enabled: false)
+ push_frontend_feature_flag(:webperf_experiment, default_enabled: false)
end
# Exposes the state of a feature flag to the frontend code.
diff --git a/lib/gitlab/grape_logging/loggers/cloudflare_logger.rb b/lib/gitlab/grape_logging/loggers/cloudflare_logger.rb
new file mode 100644
index 00000000000..3abb0100a86
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/cloudflare_logger.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module GrapeLogging
+ module Loggers
+ class CloudflareLogger < ::GrapeLogging::Loggers::Base
+ include ::Gitlab::Logging::CloudflareHelper
+
+ def parameters(request, _response)
+ data = {}
+ store_cloudflare_headers!(data, request)
+
+ data
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/grape_logging/loggers/context_logger.rb b/lib/gitlab/grape_logging/loggers/context_logger.rb
new file mode 100644
index 00000000000..0a8f0872fbe
--- /dev/null
+++ b/lib/gitlab/grape_logging/loggers/context_logger.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# This module adds additional correlation id the grape logger
+module Gitlab
+ module GrapeLogging
+ module Loggers
+ class ContextLogger < ::GrapeLogging::Loggers::Base
+ def parameters(_, _)
+ Labkit::Context.current.to_h
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/grape_logging/loggers/exception_logger.rb b/lib/gitlab/grape_logging/loggers/exception_logger.rb
index 606b7c0dbce..14147769422 100644
--- a/lib/gitlab/grape_logging/loggers/exception_logger.rb
+++ b/lib/gitlab/grape_logging/loggers/exception_logger.rb
@@ -4,14 +4,16 @@ module Gitlab
module GrapeLogging
module Loggers
class ExceptionLogger < ::GrapeLogging::Loggers::Base
- def parameters(request, _)
+ def parameters(request, response_body)
+ data = {}
+ data[:api_error] = format_body(response_body) if bad_request?(request)
+
# grape-logging attempts to pass the logger the exception
# (https://github.com/aserafin/grape_logging/blob/v1.7.0/lib/grape_logging/middleware/request_logger.rb#L63),
# but it appears that the rescue_all in api.rb takes
# precedence so the logger never sees it. We need to
# store and retrieve the exception from the environment.
exception = request.env[::API::Helpers::API_EXCEPTION_ENV]
- data = {}
return data unless exception.is_a?(Exception)
@@ -19,6 +21,28 @@ module Gitlab
data
end
+
+ private
+
+ def format_body(response_body)
+ # https://github.com/rack/rack/blob/master/SPEC.rdoc#label-The+Body:
+ # The response_body must respond to each, but just in case we
+ # guard against errors here.
+ response_body = Array(response_body) unless response_body.respond_to?(:each)
+
+ # To avoid conflicting types in Elasticsearch, convert every
+ # element into an Array of strings. A response body is usually
+ # an array of Strings so that the response can be sent in
+ # chunks.
+ body = []
+ # each_with_object doesn't work with Rack::BodyProxy
+ response_body.each { |chunk| body << chunk.to_s }
+ body
+ end
+
+ def bad_request?(request)
+ request.env[::API::Helpers::API_RESPONSE_STATUS_CODE] == 400
+ end
end
end
end
diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb
index c7f430490d6..61668b634fd 100644
--- a/lib/gitlab/graphql/authorize/authorize_field_service.rb
+++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb
@@ -70,7 +70,10 @@ module Gitlab
end
def filter_allowed(current_user, resolved_type, authorizing_object)
- if authorizing_object
+ if resolved_type.nil?
+ # We're not rendering anything, for example when a record was not found
+ # no need to do anything
+ elsif authorizing_object
# Authorizing fields representing scalars, or a simple field with an object
resolved_type if allowed_access?(current_user, authorizing_object)
elsif @field.connection?
@@ -83,9 +86,6 @@ module Gitlab
resolved_type.select do |single_object_type|
allowed_access?(current_user, single_object_type.object)
end
- elsif resolved_type.nil?
- # We're not rendering anything, for example when a record was not found
- # no need to do anything
else
raise "Can't authorize #{@field}"
end
diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb
index 5466924a794..1a32ab468b1 100644
--- a/lib/gitlab/graphql/pagination/keyset/connection.rb
+++ b/lib/gitlab/graphql/pagination/keyset/connection.rb
@@ -128,7 +128,7 @@ module Gitlab
end
def ordering_from_encoded_json(cursor)
- JSON.parse(decode(cursor))
+ Gitlab::Json.parse(decode(cursor))
rescue JSON::ParserError
raise Gitlab::Graphql::Errors::ArgumentError, "Please provide a valid cursor"
end
diff --git a/lib/gitlab/graphql/pagination/keyset/order_info.rb b/lib/gitlab/graphql/pagination/keyset/order_info.rb
index 876d6114f3c..12bcc4993b5 100644
--- a/lib/gitlab/graphql/pagination/keyset/order_info.rb
+++ b/lib/gitlab/graphql/pagination/keyset/order_info.rb
@@ -40,6 +40,8 @@ module Gitlab
end
if order_list.count > 2
+ # Keep in mind an order clause for primary key is added if one is not present
+ # lib/gitlab/graphql/pagination/keyset/connection.rb:97
raise ArgumentError.new('A maximum of 2 ordering fields are allowed')
end
diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
index 327a9c549d5..6f705239fa3 100644
--- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
+++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb
@@ -34,7 +34,7 @@ module Gitlab
memo[:depth] = depth
memo[:complexity] = complexity
- memo[:duration] = duration(memo[:time_started]).round(1)
+ memo[:duration_s] = duration(memo[:time_started]).round(1)
GraphqlLogger.info(memo.except!(:time_started, :query))
rescue => e
@@ -62,7 +62,7 @@ module Gitlab
query_string: nil,
query: query,
variables: nil,
- duration: nil
+ duration_s: nil
}
end
end
diff --git a/lib/gitlab/graphql/variables.rb b/lib/gitlab/graphql/variables.rb
index b13ea37c21f..1c6fb011012 100644
--- a/lib/gitlab/graphql/variables.rb
+++ b/lib/gitlab/graphql/variables.rb
@@ -20,7 +20,7 @@ module Gitlab
case ambiguous_param
when String
if ambiguous_param.present?
- ensure_hash(JSON.parse(ambiguous_param))
+ ensure_hash(Gitlab::Json.parse(ambiguous_param))
else
{}
end
diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb
index 8597903ad00..eb4361cdc53 100644
--- a/lib/gitlab/group_search_results.rb
+++ b/lib/gitlab/group_search_results.rb
@@ -4,8 +4,8 @@ module Gitlab
class GroupSearchResults < SearchResults
attr_reader :group
- def initialize(current_user, limit_projects, group, query, default_project_filter: false, per_page: 20)
- super(current_user, limit_projects, query, default_project_filter: default_project_filter, per_page: per_page)
+ def initialize(current_user, limit_projects, group, query, default_project_filter: false)
+ super(current_user, limit_projects, query, default_project_filter: default_project_filter)
@group = group
end
diff --git a/lib/gitlab/health_checks/puma_check.rb b/lib/gitlab/health_checks/puma_check.rb
index 9f09070a57d..2dc8a093572 100644
--- a/lib/gitlab/health_checks/puma_check.rb
+++ b/lib/gitlab/health_checks/puma_check.rb
@@ -21,7 +21,7 @@ module Gitlab
return unless Gitlab::Runtime.puma?
stats = Puma.stats
- stats = JSON.parse(stats)
+ stats = Gitlab::Json.parse(stats)
# If `workers` is missing this means that
# Puma server is running in single mode
diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb
index 7e0398f09af..18f4cb559c5 100644
--- a/lib/gitlab/i18n.rb
+++ b/lib/gitlab/i18n.rb
@@ -5,28 +5,28 @@ module Gitlab
extend self
AVAILABLE_LANGUAGES = {
+ 'bg' => 'Bulgarian - български',
+ 'cs_CZ' => 'Czech - čeština',
+ 'de' => 'German - Deutsch',
'en' => 'English',
- 'es' => 'Español',
- 'gl_ES' => 'Galego',
- 'de' => 'Deutsch',
- 'fr' => 'Français',
- 'pt_BR' => 'Português (Brasil)',
- 'zh_CN' => '简体中文',
- 'zh_HK' => '繁體中文 (香港)',
- 'zh_TW' => '繁體中文 (臺灣)',
- 'bg' => 'български',
- 'ru' => 'Русский',
- 'eo' => 'Esperanto',
- 'it' => 'Italiano',
- 'uk' => 'Українська',
- 'ja' => '日本語',
- 'ko' => '한국어',
- 'nl_NL' => 'Nederlands',
- 'tr_TR' => 'Türkçe',
- 'id_ID' => 'Bahasa Indonesia',
+ 'eo' => 'Esperanto - esperanto',
+ 'es' => 'Spanish - español',
'fil_PH' => 'Filipino',
- 'pl_PL' => 'Polski',
- 'cs_CZ' => 'Čeština'
+ 'fr' => 'French - français',
+ 'gl_ES' => 'Galician - galego',
+ 'id_ID' => 'Indonesian - Bahasa Indonesia',
+ 'it' => 'Italian - italiano',
+ 'ja' => 'Japanese - 日本語',
+ 'ko' => 'Korean - 한국어',
+ 'nl_NL' => 'Dutch - Nederlands',
+ 'pl_PL' => 'Polish - polski',
+ 'pt_BR' => 'Portuguese (Brazil) - português (Brasil)',
+ 'ru' => 'Russian - Русский',
+ 'tr_TR' => 'Turkish - Türkçe',
+ 'uk' => 'Ukrainian - українська',
+ 'zh_CN' => 'Chinese, Simplified - 简体中文',
+ 'zh_HK' => 'Chinese, Traditional (Hong Kong) - 繁體中文 (香港)',
+ 'zh_TW' => 'Chinese, Traditional (Taiwan) - 繁體中文 (台灣)'
}.freeze
def available_locales
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index 52102b6f508..921072a4970 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -42,6 +42,10 @@ module Gitlab
"project.wiki.bundle"
end
+ def design_repo_bundle_filename
+ 'project.design.bundle'
+ end
+
def snippet_repo_bundle_dir
'snippets'
end
@@ -88,6 +92,10 @@ module Gitlab
'group.json'
end
+ def legacy_group_config_file
+ Rails.root.join('lib/gitlab/import_export/group/legacy_import_export.yml')
+ end
+
def group_config_file
Rails.root.join('lib/gitlab/import_export/group/import_export.yml')
end
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index fd98bc2caad..e2dba831661 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -52,7 +52,10 @@ module Gitlab
end
def headers
- { 'Content-Length' => export_size.to_s }
+ {
+ 'Content-Type' => 'application/gzip',
+ 'Content-Length' => export_size.to_s
+ }
end
def export_size
diff --git a/lib/gitlab/import_export/design_repo_restorer.rb b/lib/gitlab/import_export/design_repo_restorer.rb
new file mode 100644
index 00000000000..a702c58a7c2
--- /dev/null
+++ b/lib/gitlab/import_export/design_repo_restorer.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ class DesignRepoRestorer < RepoRestorer
+ def initialize(project:, shared:, path_to_bundle:)
+ super(project: project, shared: shared, path_to_bundle: path_to_bundle)
+
+ @repository = project.design_repository
+ end
+
+ # `restore` method is handled in super class
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/design_repo_saver.rb b/lib/gitlab/import_export/design_repo_saver.rb
new file mode 100644
index 00000000000..db9ebee6a13
--- /dev/null
+++ b/lib/gitlab/import_export/design_repo_saver.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ class DesignRepoSaver < RepoSaver
+ def save
+ @repository = project.design_repository
+
+ super
+ end
+
+ private
+
+ def bundle_full_path
+ File.join(shared.export_path, ::Gitlab::ImportExport.design_repo_bundle_filename)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/group/group_restorer.rb b/lib/gitlab/import_export/group/group_restorer.rb
new file mode 100644
index 00000000000..b338950fb71
--- /dev/null
+++ b/lib/gitlab/import_export/group/group_restorer.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Group
+ class GroupRestorer
+ def initialize(
+ user:,
+ shared:,
+ group:,
+ attributes:,
+ importable_path:,
+ relation_reader:,
+ reader:
+ )
+ @user = user
+ @shared = shared
+ @group = group
+ @group_attributes = attributes
+ @importable_path = importable_path
+ @relation_reader = relation_reader
+ @reader = reader
+ end
+
+ def restore
+ # consume_relation returns a list of [relation, index]
+ @group_members = @relation_reader
+ .consume_relation(@importable_path, 'members')
+ .map(&:first)
+
+ return unless members_mapper.map
+
+ restorer.restore
+ end
+
+ private
+
+ def restorer
+ @relation_tree_restorer ||= RelationTreeRestorer.new(
+ user: @user,
+ shared: @shared,
+ relation_reader: @relation_reader,
+ members_mapper: members_mapper,
+ object_builder: object_builder,
+ relation_factory: relation_factory,
+ reader: @reader,
+ importable: @group,
+ importable_attributes: @group_attributes,
+ importable_path: @importable_path
+ )
+ end
+
+ def members_mapper
+ @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(
+ exported_members: @group_members,
+ user: @user,
+ importable: @group
+ )
+ end
+
+ def relation_factory
+ Gitlab::ImportExport::Group::RelationFactory
+ end
+
+ def object_builder
+ Gitlab::ImportExport::Group::ObjectBuilder
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml
index 5008639077c..e30206dc509 100644
--- a/lib/gitlab/import_export/group/import_export.yml
+++ b/lib/gitlab/import_export/group/import_export.yml
@@ -27,9 +27,7 @@ included_attributes:
excluded_attributes:
group:
- - :id
- :owner_id
- - :parent_id
- :created_at
- :updated_at
- :runners_token
diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml
new file mode 100644
index 00000000000..5008639077c
--- /dev/null
+++ b/lib/gitlab/import_export/group/legacy_import_export.yml
@@ -0,0 +1,86 @@
+# Model relationships to be included in the group import/export
+#
+# This list _must_ only contain relationships that are available to both FOSS and
+# Enterprise editions. EE specific relationships must be defined in the `ee` section further
+# down below.
+tree:
+ group:
+ - :milestones
+ - :badges
+ - labels:
+ - :priorities
+ - boards:
+ - lists:
+ - label:
+ - :priorities
+ - :board
+ - members:
+ - :user
+
+included_attributes:
+ user:
+ - :id
+ - :email
+ - :username
+ author:
+ - :name
+
+excluded_attributes:
+ group:
+ - :id
+ - :owner_id
+ - :parent_id
+ - :created_at
+ - :updated_at
+ - :runners_token
+ - :runners_token_encrypted
+ - :saml_discovery_token
+ - :visibility_level
+ - :trial_ends_on
+ - :shared_runners_minute_limit
+ - :extra_shared_runners_minutes_limit
+ epics:
+ - :state_id
+
+methods:
+ labels:
+ - :type
+ label:
+ - :type
+ badges:
+ - :type
+ notes:
+ - :type
+ events:
+ - :action
+ lists:
+ - :list_type
+ epics:
+ - :state
+
+preloads:
+
+# EE specific relationships and settings to include. All of this will be merged
+# into the previous structures if EE is used.
+ee:
+ tree:
+ group:
+ - epics:
+ - :parent
+ - :award_emoji
+ - events:
+ - :push_event_payload
+ - notes:
+ - :author
+ - :award_emoji
+ - events:
+ - :push_event_payload
+ - boards:
+ - :board_assignee
+ - :milestone
+ - labels:
+ - :priorities
+ - lists:
+ - milestone:
+ - events:
+ - :push_event_payload
diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb
index 5d96a0f3c0a..5499b79cee6 100644
--- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb
+++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb
@@ -122,7 +122,7 @@ module Gitlab
@reader ||= Gitlab::ImportExport::Reader.new(
shared: @shared,
config: Gitlab::ImportExport::Config.new(
- config: Gitlab::ImportExport.group_config_file
+ config: Gitlab::ImportExport.legacy_group_config_file
).to_h
)
end
diff --git a/lib/gitlab/import_export/group/legacy_tree_saver.rb b/lib/gitlab/import_export/group/legacy_tree_saver.rb
index 3776ef0d8f5..7ab81c09885 100644
--- a/lib/gitlab/import_export/group/legacy_tree_saver.rb
+++ b/lib/gitlab/import_export/group/legacy_tree_saver.rb
@@ -43,7 +43,7 @@ module Gitlab
@reader ||= Gitlab::ImportExport::Reader.new(
shared: @shared,
config: Gitlab::ImportExport::Config.new(
- config: Gitlab::ImportExport.group_config_file
+ config: Gitlab::ImportExport.legacy_group_config_file
).to_h
)
end
diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb
new file mode 100644
index 00000000000..d0c0999f291
--- /dev/null
+++ b/lib/gitlab/import_export/group/tree_restorer.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Group
+ class TreeRestorer
+ include Gitlab::Utils::StrongMemoize
+
+ attr_reader :user, :shared
+
+ def initialize(user:, shared:, group:)
+ @user = user
+ @shared = shared
+ @top_level_group = group
+ @groups_mapping = {}
+ end
+
+ def restore
+ group_ids = relation_reader.consume_relation('groups', '_all').map { |value, _idx| Integer(value) }
+ root_group_id = group_ids.delete_at(0)
+
+ process_root(root_group_id)
+
+ group_ids.each do |group_id|
+ process_child(group_id)
+ end
+
+ true
+ rescue => e
+ shared.error(e)
+ false
+ end
+
+ class GroupAttributes
+ attr_reader :attributes, :group_id, :id, :path
+
+ def initialize(group_id, relation_reader)
+ @group_id = group_id
+
+ @path = "groups/#{group_id}"
+ @attributes = relation_reader.consume_attributes(@path)
+ @id = @attributes.delete('id')
+
+ unless @id == @group_id
+ raise ArgumentError, "Invalid group_id for #{group_id}"
+ end
+ end
+
+ def delete_attribute(name)
+ attributes.delete(name)
+ end
+
+ def delete_attributes(*names)
+ names.map(&method(:delete_attribute))
+ end
+ end
+ private_constant :GroupAttributes
+
+ private
+
+ def process_root(group_id)
+ group_attributes = GroupAttributes.new(group_id, relation_reader)
+
+ # name and path are not imported on the root group to avoid conflict
+ # with existing groups name and/or path.
+ group_attributes.delete_attributes('name', 'path')
+
+ restore_group(@top_level_group, group_attributes)
+ end
+
+ def process_child(group_id)
+ group_attributes = GroupAttributes.new(group_id, relation_reader)
+
+ group = create_group(group_attributes)
+
+ restore_group(group, group_attributes)
+ end
+
+ def create_group(group_attributes)
+ parent_id = group_attributes.delete_attribute('parent_id')
+ name = group_attributes.delete_attribute('name')
+ path = group_attributes.delete_attribute('path')
+
+ parent_group = @groups_mapping.fetch(parent_id) { raise(ArgumentError, 'Parent group not found') }
+
+ ::Groups::CreateService.new(
+ user,
+ name: name,
+ path: path,
+ parent_id: parent_group.id,
+ visibility_level: sub_group_visibility_level(group_attributes.attributes, parent_group)
+ ).execute
+ end
+
+ def restore_group(group, group_attributes)
+ @groups_mapping[group_attributes.id] = group
+
+ Group::GroupRestorer.new(
+ user: user,
+ shared: shared,
+ group: group,
+ attributes: group_attributes.attributes,
+ importable_path: group_attributes.path,
+ relation_reader: relation_reader,
+ reader: reader
+ ).restore
+ end
+
+ def relation_reader
+ strong_memoize(:relation_reader) do
+ ImportExport::JSON::NdjsonReader.new(
+ File.join(shared.export_path, 'tree')
+ )
+ end
+ end
+
+ def sub_group_visibility_level(group_hash, parent_group)
+ original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE
+
+ if parent_group && parent_group.visibility_level < original_visibility_level
+ Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
+ else
+ original_visibility_level
+ end
+ end
+
+ def reader
+ strong_memoize(:reader) do
+ Gitlab::ImportExport::Reader.new(
+ shared: @shared,
+ config: Gitlab::ImportExport::Config.new(
+ config: Gitlab::ImportExport.group_config_file
+ ).to_h
+ )
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb
new file mode 100644
index 00000000000..d538de33c51
--- /dev/null
+++ b/lib/gitlab/import_export/group/tree_saver.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Group
+ class TreeSaver
+ attr_reader :full_path, :shared
+
+ def initialize(group:, current_user:, shared:, params: {})
+ @params = params
+ @current_user = current_user
+ @shared = shared
+ @group = group
+ @full_path = File.join(@shared.export_path, 'tree')
+ end
+
+ def save
+ all_groups = Enumerator.new do |group_ids|
+ groups.each do |group|
+ serialize(group)
+ group_ids << group.id
+ end
+ end
+
+ json_writer.write_relation_array('groups', '_all', all_groups)
+
+ true
+ rescue => e
+ @shared.error(e)
+ false
+ ensure
+ json_writer&.close
+ end
+
+ private
+
+ def groups
+ @groups ||= Gitlab::ObjectHierarchy
+ .new(::Group.where(id: @group.id))
+ .base_and_descendants(with_depth: true)
+ .order_by(:depth)
+ end
+
+ def serialize(group)
+ ImportExport::JSON::StreamingSerializer.new(
+ group,
+ group_tree,
+ json_writer,
+ exportable_path: "groups/#{group.id}"
+ ).execute
+ end
+
+ def group_tree
+ @group_tree ||= Gitlab::ImportExport::Reader.new(
+ shared: @shared,
+ config: group_config
+ ).group_tree
+ end
+
+ def group_config
+ Gitlab::ImportExport::Config.new(
+ config: Gitlab::ImportExport.group_config_file
+ ).to_h
+ end
+
+ def json_writer
+ @json_writer ||= ImportExport::JSON::NdjsonWriter.new(@full_path)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 4b761eb86ae..b1219384732 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -34,7 +34,7 @@ module Gitlab
attr_accessor :archive_file, :current_user, :project, :shared
def restorers
- [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
+ [repo_restorer, wiki_restorer, project_tree, avatar_restorer, design_repo_restorer,
uploads_restorer, lfs_restorer, statistics_restorer, snippets_repo_restorer]
end
@@ -71,6 +71,12 @@ module Gitlab
wiki_enabled: project.wiki_enabled?)
end
+ def design_repo_restorer
+ Gitlab::ImportExport::DesignRepoRestorer.new(path_to_bundle: design_repo_path,
+ shared: shared,
+ project: project)
+ end
+
def uploads_restorer
Gitlab::ImportExport::UploadsRestorer.new(project: project, shared: shared)
end
@@ -101,6 +107,10 @@ module Gitlab
File.join(shared.export_path, Gitlab::ImportExport.wiki_repo_bundle_filename)
end
+ def design_repo_path
+ File.join(shared.export_path, Gitlab::ImportExport.design_repo_bundle_filename)
+ end
+
def remove_import_file
upload = project.import_export_upload
@@ -111,13 +121,17 @@ module Gitlab
end
def overwrite_project
- return unless can?(current_user, :admin_namespace, project.namespace)
+ return true unless overwrite_project?
- if overwrite_project?
- ::Projects::OverwriteProjectService.new(project, current_user)
- .execute(project_to_overwrite)
+ unless can?(current_user, :admin_namespace, project.namespace)
+ message = "User #{current_user&.username} (#{current_user&.id}) cannot overwrite a project in #{project.namespace.path}"
+ @shared.error(::Projects::ImportService::PermissionError.new(message))
+ return false
end
+ ::Projects::OverwriteProjectService.new(project, current_user)
+ .execute(project_to_overwrite)
+
true
end
@@ -137,5 +151,3 @@ module Gitlab
end
end
end
-
-Gitlab::ImportExport::Importer.prepend_if_ee('EE::Gitlab::ImportExport::Importer')
diff --git a/lib/gitlab/import_export/project/base_task.rb b/lib/gitlab/import_export/project/base_task.rb
index 6a7b24421c9..356e261e251 100644
--- a/lib/gitlab/import_export/project/base_task.rb
+++ b/lib/gitlab/import_export/project/base_task.rb
@@ -11,17 +11,27 @@ module Gitlab
@file_path = opts.fetch(:file_path)
@namespace = Namespace.find_by_full_path(opts.fetch(:namespace_path))
@current_user = User.find_by_username(opts.fetch(:username))
- @measurement_enabled = opts.fetch(:measurement_enabled)
- @measurement = Gitlab::Utils::Measuring.new(logger: logger) if @measurement_enabled
@logger = logger
end
private
- attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path, :logger
+ attr_reader :project, :namespace, :current_user, :file_path, :project_path, :logger
- def measurement_enabled?
- @measurement_enabled
+ def disable_upload_object_storage
+ overwrite_uploads_setting('enabled', false) do
+ yield
+ end
+ end
+
+ def overwrite_uploads_setting(key, value)
+ old_value = Settings.uploads.object_store[key]
+ Settings.uploads.object_store[key] = value
+
+ yield
+
+ ensure
+ Settings.uploads.object_store[key] = old_value
end
def success(message)
diff --git a/lib/gitlab/import_export/project/export_task.rb b/lib/gitlab/import_export/project/export_task.rb
index ec287380c48..5e105b4653d 100644
--- a/lib/gitlab/import_export/project/export_task.rb
+++ b/lib/gitlab/import_export/project/export_task.rb
@@ -19,7 +19,11 @@ module Gitlab
.execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path))
end
+ return error(project.import_export_shared.errors.join(', ')) if project.import_export_shared.errors.any?
+
success('Done!')
+ rescue Gitlab::ImportExport::Error => e
+ error(e.message)
end
private
@@ -32,8 +36,13 @@ module Gitlab
def with_export
with_request_store do
- ::Gitlab::GitalyClient.allow_n_plus_1_calls do
- measurement_enabled? ? measurement.with_measuring { yield } : yield
+ # We are disabling ObjectStorage for `export`
+ # since when direct upload is enabled, remote storage will be used
+ # and Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy will fail to copy exported archive
+ disable_upload_object_storage do
+ ::Gitlab::GitalyClient.allow_n_plus_1_calls do
+ yield
+ end
end
end
end
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 3cbd0d144e6..8851b106ad5 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -29,6 +29,14 @@ tree:
- resource_label_events:
- label:
- :priorities
+ - designs:
+ - notes:
+ - :author
+ - events:
+ - :push_event_payload
+ - design_versions:
+ - actions:
+ - :design # Duplicate export of issues.designs in order to link the record to both Issue and Action
- :issue_assignees
- :zoom_meetings
- :sentry_issue
@@ -160,6 +168,7 @@ excluded_attributes:
- :marked_for_deletion_at
- :marked_for_deletion_by_user_id
- :compliance_framework_setting
+ - :show_default_award_emojis
namespaces:
- :runners_token
- :runners_token_encrypted
@@ -190,6 +199,7 @@ excluded_attributes:
- :merge_request_diff_id
issues:
- :milestone_id
+ - :sprint_id
- :moved_to_id
- :sent_notifications
- :state_id
@@ -197,6 +207,7 @@ excluded_attributes:
- :promoted_to_epic_id
merge_request:
- :milestone_id
+ - :sprint_id
- :ref_fetched
- :merge_jid
- :rebase_jid
@@ -205,6 +216,7 @@ excluded_attributes:
- :state_id
merge_requests:
- :milestone_id
+ - :sprint_id
- :ref_fetched
- :merge_jid
- :rebase_jid
@@ -250,8 +262,9 @@ excluded_attributes:
- :token
- :token_encrypted
services:
- - :template
+ - :inherit_from_id
- :instance
+ - :template
error_tracking_setting:
- :encrypted_token
- :encrypted_token_iv
@@ -284,6 +297,7 @@ excluded_attributes:
actions:
- :design_id
- :version_id
+ - image_v432x230
links:
- :release_id
project_members:
@@ -376,14 +390,6 @@ ee:
tree:
project:
- issues:
- - designs:
- - notes:
- - :author
- - events:
- - :push_event_payload
- - design_versions:
- - actions:
- - :design # Duplicate export of issues.designs in order to link the record to both Issue and Action
- epic_issue:
- :epic
- protected_branches:
@@ -391,6 +397,3 @@ ee:
- protected_environments:
- :deploy_access_levels
- :service_desk_setting
- excluded_attributes:
- actions:
- - image_v432x230
diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb
index ae654ddbeaf..59bb8af750e 100644
--- a/lib/gitlab/import_export/project/import_task.rb
+++ b/lib/gitlab/import_export/project/import_task.rb
@@ -32,7 +32,7 @@ module Gitlab
# https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635
# For development setups, this code-path will be excluded from n+1 detection.
::Gitlab::GitalyClient.allow_n_plus_1_calls do
- measurement_enabled? ? measurement.with_measuring { yield } : yield
+ yield
end
end
@@ -56,11 +56,7 @@ module Gitlab
disable_upload_object_storage do
service = Projects::GitlabProjectsImportService.new(
current_user,
- {
- namespace_id: namespace.id,
- path: project_path,
- file: File.open(file_path)
- }
+ import_params
)
service.execute
@@ -71,24 +67,6 @@ module Gitlab
Sidekiq::Worker.drain_all
end
- def disable_upload_object_storage
- overwrite_uploads_setting('background_upload', false) do
- overwrite_uploads_setting('direct_upload', false) do
- yield
- end
- end
- end
-
- def overwrite_uploads_setting(key, value)
- old_value = Settings.uploads.object_store[key]
- Settings.uploads.object_store[key] = value
-
- yield
-
- ensure
- Settings.uploads.object_store[key] = old_value
- end
-
def full_path
"#{namespace.full_path}/#{project_path}"
end
@@ -99,6 +77,14 @@ module Gitlab
" as #{current_user.name}"
end
+ def import_params
+ {
+ namespace_id: namespace.id,
+ path: project_path,
+ file: File.open(file_path)
+ }
+ end
+
def show_import_failures_count
return unless project.import_failures.exists?
diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb
index c3637b1c115..831e38f3034 100644
--- a/lib/gitlab/import_export/project/object_builder.rb
+++ b/lib/gitlab/import_export/project/object_builder.rb
@@ -57,6 +57,8 @@ module Gitlab
# Returns Arel clause for a particular model or `nil`.
def where_clause_for_klass
+ return attrs_to_arel(attributes.slice('filename')).and(table[:issue_id].eq(nil)) if design?
+
attrs_to_arel(attributes.slice('iid')) if merge_request?
end
@@ -95,6 +97,10 @@ module Gitlab
klass == Epic
end
+ def design?
+ klass == DesignManagement::Design
+ end
+
# If an existing group milestone used the IID
# claim the IID back and set the group milestone to use one available
# This is necessary to fix situations like the following:
@@ -115,5 +121,3 @@ module Gitlab
end
end
end
-
-Gitlab::ImportExport::Project::ObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::Project::ObjectBuilder')
diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb
index f7f1195f2f1..3ab9f2c4bfa 100644
--- a/lib/gitlab/import_export/project/relation_factory.rb
+++ b/lib/gitlab/import_export/project/relation_factory.rb
@@ -4,8 +4,6 @@ module Gitlab
module ImportExport
module Project
class RelationFactory < Base::RelationFactory
- prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
OVERRIDES = { snippets: :project_snippets,
ci_pipelines: 'Ci::Pipeline',
pipelines: 'Ci::Pipeline',
@@ -19,6 +17,10 @@ module Gitlab
merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
push_access_levels: 'ProtectedBranch::PushAccessLevel',
create_access_levels: 'ProtectedTag::CreateAccessLevel',
+ design: 'DesignManagement::Design',
+ designs: 'DesignManagement::Design',
+ design_versions: 'DesignManagement::Version',
+ actions: 'DesignManagement::Action',
labels: :project_labels,
priorities: :label_priorities,
auto_devops: :project_auto_devops,
@@ -53,6 +55,7 @@ module Gitlab
container_expiration_policy
external_pull_request
external_pull_requests
+ DesignManagement::Design
].freeze
def create
@@ -161,3 +164,5 @@ module Gitlab
end
end
end
+
+Gitlab::ImportExport::Project::RelationFactory.prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory')
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
index e9c89b803ba..a16ffe36054 100644
--- a/lib/gitlab/import_export/project/tree_restorer.rb
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -54,7 +54,7 @@ module Gitlab
end
def ndjson_relation_reader
- return unless Feature.enabled?(:project_import_ndjson, project.namespace)
+ return unless Feature.enabled?(:project_import_ndjson, project.namespace, default_enabled: true)
ImportExport::JSON::NdjsonReader.new(
File.join(shared.export_path, 'tree')
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 0017aa523c1..7cca3596da6 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -54,7 +54,7 @@ module Gitlab
def json_writer
@json_writer ||= begin
- if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace)
+ if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace, default_enabled: true)
full_path = File.join(@shared.export_path, 'tree')
Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path)
else
diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb
index 056945d0294..ea16d978127 100644
--- a/lib/gitlab/import_export/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/relation_tree_restorer.rb
@@ -37,9 +37,7 @@ module Gitlab
ActiveRecord::Base.no_touching do
update_params!
- bulk_inserts_enabled = @importable.class == ::Project &&
- Feature.enabled?(:import_bulk_inserts, @importable.group, default_enabled: true)
- BulkInsertableAssociations.with_bulk_insert(enabled: bulk_inserts_enabled) do
+ BulkInsertableAssociations.with_bulk_insert(enabled: @importable.class == ::Project) do
fix_ci_pipelines_not_sorted_on_legacy_project_json!
create_relations!
end
diff --git a/lib/gitlab/import_export/snippets_repo_restorer.rb b/lib/gitlab/import_export/snippets_repo_restorer.rb
index 8fe83225812..9ff3e74a6b1 100644
--- a/lib/gitlab/import_export/snippets_repo_restorer.rb
+++ b/lib/gitlab/import_export/snippets_repo_restorer.rb
@@ -10,7 +10,6 @@ module Gitlab
end
def restore
- return true unless Feature.enabled?(:version_snippets, @user)
return true unless Dir.exist?(snippets_repo_bundle_path)
@project.snippets.find_each.all? do |snippet|
diff --git a/lib/gitlab/import_export/snippets_repo_saver.rb b/lib/gitlab/import_export/snippets_repo_saver.rb
index 85e094c0d15..d3b0fe1c18c 100644
--- a/lib/gitlab/import_export/snippets_repo_saver.rb
+++ b/lib/gitlab/import_export/snippets_repo_saver.rb
@@ -12,8 +12,6 @@ module Gitlab
end
def save
- return true unless Feature.enabled?(:version_snippets, @current_user)
-
create_snippets_repo_directory
@project.snippets.find_each.all? do |snippet|
diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb
index 6b066b800a5..cc99e828251 100644
--- a/lib/gitlab/instrumentation/redis.rb
+++ b/lib/gitlab/instrumentation/redis.rb
@@ -38,7 +38,8 @@ module Gitlab
end
def self.query_time
- ::RequestStore[REDIS_CALL_DURATION] || 0
+ query_time = ::RequestStore[REDIS_CALL_DURATION] || 0
+ query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION)
end
def self.add_duration(duration)
diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index 308c3007720..7c5a601cd5b 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -5,27 +5,28 @@ module Gitlab
extend self
KEYS = %i(gitaly_calls gitaly_duration_s rugged_calls rugged_duration_s redis_calls redis_duration_s).freeze
+ DURATION_PRECISION = 6 # microseconds
def add_instrumentation_data(payload)
gitaly_calls = Gitlab::GitalyClient.get_request_count
if gitaly_calls > 0
payload[:gitaly_calls] = gitaly_calls
- payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time.round(2)
+ payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time
end
rugged_calls = Gitlab::RuggedInstrumentation.query_count
if rugged_calls > 0
payload[:rugged_calls] = rugged_calls
- payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time.round(2)
+ payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time
end
redis_calls = Gitlab::Instrumentation::Redis.get_request_count
if redis_calls > 0
payload[:redis_calls] = redis_calls
- payload[:redis_duration_s] = Gitlab::Instrumentation::Redis.query_time.round(2)
+ payload[:redis_duration_s] = Gitlab::Instrumentation::Redis.query_time
end
end
@@ -47,7 +48,7 @@ module Gitlab
# Its possible that if theres clock-skew between two nodes
# this value may be less than zero. In that event, we record the value
# as zero.
- [elapsed_by_absolute_time(enqueued_at_time), 0].max.round(2)
+ [elapsed_by_absolute_time(enqueued_at_time), 0].max.round(DURATION_PRECISION)
end
# Calculates the time in seconds, as a float, from
diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb
index 5381812186d..306736df30f 100644
--- a/lib/gitlab/jira_import/base_importer.rb
+++ b/lib/gitlab/jira_import/base_importer.rb
@@ -3,12 +3,13 @@
module Gitlab
module JiraImport
class BaseImporter
- attr_reader :project, :client, :formatter, :jira_project_key
+ attr_reader :project, :client, :formatter, :jira_project_key, :running_import
def initialize(project)
project.validate_jira_import_settings!
- @jira_project_key = project.latest_jira_import&.jira_project_key
+ @running_import = project.latest_jira_import
+ @jira_project_key = running_import&.jira_project_key
raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key
diff --git a/lib/gitlab/jira_import/handle_labels_service.rb b/lib/gitlab/jira_import/handle_labels_service.rb
new file mode 100644
index 00000000000..1b00515cced
--- /dev/null
+++ b/lib/gitlab/jira_import/handle_labels_service.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module JiraImport
+ class HandleLabelsService
+ def initialize(project, jira_labels)
+ @project = project
+ @jira_labels = jira_labels
+ end
+
+ def execute
+ return if jira_labels.blank?
+
+ existing_labels = LabelsFinder.new(nil, project: project, title: jira_labels)
+ .execute(skip_authorization: true).select(:id, :name)
+ new_labels = create_missing_labels(existing_labels)
+
+ label_ids = existing_labels.map(&:id)
+ label_ids += new_labels if new_labels.present?
+ label_ids
+ end
+
+ private
+
+ attr_reader :project, :jira_labels
+
+ def create_missing_labels(existing_labels)
+ labels_to_create = jira_labels - existing_labels.map(&:name)
+ return if labels_to_create.empty?
+
+ new_labels_hash = labels_to_create.map do |title|
+ { project_id: project.id, title: title, type: 'ProjectLabel' }
+ end
+
+ Label.insert_all(new_labels_hash).rows.flatten
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb
index cdcb62ac6e9..df57680073e 100644
--- a/lib/gitlab/jira_import/issue_serializer.rb
+++ b/lib/gitlab/jira_import/issue_serializer.rb
@@ -3,15 +3,14 @@
module Gitlab
module JiraImport
class IssueSerializer
- attr_reader :jira_issue, :project, :params, :formatter
- attr_accessor :metadata
+ attr_reader :jira_issue, :project, :import_owner_id, :params, :formatter
- def initialize(project, jira_issue, params = {})
+ def initialize(project, jira_issue, import_owner_id, params = {})
@jira_issue = jira_issue
@project = project
+ @import_owner_id = import_owner_id
@params = params
@formatter = Gitlab::ImportFormatter.new
- @metadata = []
end
def execute
@@ -23,7 +22,9 @@ module Gitlab
state_id: map_status(jira_issue.status.statusCategory),
updated_at: jira_issue.updated,
created_at: jira_issue.created,
- author_id: project.creator_id # TODO: map actual author: https://gitlab.com/gitlab-org/gitlab/-/issues/210580
+ author_id: reporter,
+ assignee_ids: assignees,
+ label_ids: label_ids
}
end
@@ -35,10 +36,8 @@ module Gitlab
def description
body = []
- body << formatter.author_line(jira_issue.reporter.displayName)
- body << formatter.assignee_line(jira_issue.assignee.displayName) if jira_issue.assignee
body << jira_issue.description
- body << add_metadata
+ body << MetadataCollector.new(jira_issue).execute
body.join
end
@@ -52,48 +51,33 @@ module Gitlab
end
end
- def add_metadata
- add_field(%w(issuetype name), 'Issue type')
- add_field(%w(priority name), 'Priority')
- add_labels
- add_field('environment', 'Environment')
- add_field('duedate', 'Due date')
- add_parent
- add_versions
-
- return if metadata.empty?
-
- metadata.join("\n").prepend("\n\n---\n\n**Issue metadata**\n\n")
+ def map_user_id(jira_user)
+ Gitlab::JiraImport::UserMapper.new(project, jira_user).execute&.id
end
- def add_field(keys, field_label)
- value = fields.dig(*keys)
- return if value.blank?
-
- metadata << "- #{field_label}: #{value}"
+ def reporter
+ map_user_id(jira_issue.reporter&.attrs) || import_owner_id
end
- def add_labels
- return if fields['labels'].blank?
-
- metadata << "- Labels: #{fields['labels'].join(', ')}"
- end
+ def assignees
+ found_user_id = map_user_id(jira_issue.assignee&.attrs)
- def add_parent
- parent_issue_key = fields.dig('parent', 'key')
- return if parent_issue_key.blank?
+ return unless found_user_id
- metadata << "- Parent issue: [#{parent_issue_key}] #{fields['parent']['fields']['summary']}"
+ [found_user_id]
end
- def add_versions
- return if fields['fixVersions'].blank?
+ # We already create labels in Gitlab::JiraImport::LabelsImporter stage but
+ # there is a possibility it may fail or
+ # new labels were created on the Jira in the meantime
+ def label_ids
+ return if jira_issue.fields['labels'].blank?
- metadata << "- Fix versions: #{fields['fixVersions'].map { |version| version['name'] }.join(', ')}"
+ Gitlab::JiraImport::HandleLabelsService.new(project, jira_issue.fields['labels']).execute
end
- def fields
- jira_issue.fields
+ def logger
+ @logger ||= Gitlab::Import::Logger.build
end
end
end
diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb
index 6543b633ddf..8c18e58d9df 100644
--- a/lib/gitlab/jira_import/issues_importer.rb
+++ b/lib/gitlab/jira_import/issues_importer.rb
@@ -57,7 +57,7 @@ module Gitlab
# For such cases we exit early if issue was already imported.
next if already_imported?(jira_issue.id)
- issue_attrs = IssueSerializer.new(project, jira_issue, { iid: next_iid }).execute
+ issue_attrs = IssueSerializer.new(project, jira_issue, running_import.user_id, { iid: next_iid }).execute
Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key)
job_waiter.jobs_remaining += 1
diff --git a/lib/gitlab/jira_import/labels_importer.rb b/lib/gitlab/jira_import/labels_importer.rb
index 35c434e48a4..6e6842e06bf 100644
--- a/lib/gitlab/jira_import/labels_importer.rb
+++ b/lib/gitlab/jira_import/labels_importer.rb
@@ -5,6 +5,8 @@ module Gitlab
class LabelsImporter < BaseImporter
attr_reader :job_waiter
+ MAX_LABELS = 500
+
def initialize(project)
super
@job_waiter = JobWaiter.new
@@ -19,15 +21,35 @@ module Gitlab
def cache_import_label(project)
label = project.jira_imports.by_jira_project_key(jira_project_key).last.label
- raise Projects::ImportService::Error, _('Failed to find import label for jira import.') unless label
+ raise Projects::ImportService::Error, _('Failed to find import label for Jira import.') unless label
JiraImport.cache_import_label_id(project.id, label.id)
end
def import_jira_labels
- # todo: import jira labels, see https://gitlab.com/gitlab-org/gitlab/-/issues/212651
+ start_at = 0
+ loop do
+ break if process_jira_page(start_at)
+
+ start_at += MAX_LABELS
+ end
+
job_waiter
end
+
+ def process_jira_page(start_at)
+ request = "/rest/api/2/label?maxResults=#{MAX_LABELS}&startAt=#{start_at}"
+ response = client.get(request)
+
+ return true if response['values'].blank?
+ return true unless response.key?('isLast')
+
+ Gitlab::JiraImport::HandleLabelsService.new(project, response['values']).execute
+
+ response['isLast']
+ rescue => e
+ Gitlab::ErrorTracking.track_exception(e, project_id: project.id, request: request)
+ end
end
end
end
diff --git a/lib/gitlab/jira_import/metadata_collector.rb b/lib/gitlab/jira_import/metadata_collector.rb
new file mode 100644
index 00000000000..4551f38ba98
--- /dev/null
+++ b/lib/gitlab/jira_import/metadata_collector.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module JiraImport
+ class MetadataCollector
+ attr_accessor :jira_issue, :metadata
+
+ def initialize(jira_issue)
+ @jira_issue = jira_issue
+ @metadata = []
+ end
+
+ def execute
+ add_field(%w(issuetype name), 'Issue type')
+ add_field(%w(priority name), 'Priority')
+ add_field('environment', 'Environment')
+ add_field('duedate', 'Due date')
+ add_parent
+ add_versions
+
+ return if metadata.empty?
+
+ metadata.join("\n").prepend("\n\n---\n\n**Issue metadata**\n\n")
+ end
+
+ private
+
+ def add_field(keys, field_label)
+ value = fields.dig(*keys)
+ return if value.blank?
+
+ metadata << "- #{field_label}: #{value}"
+ end
+
+ def add_parent
+ parent_issue_key = fields.dig('parent', 'key')
+
+ return if parent_issue_key.blank?
+
+ parent_summary_key = fields.dig('parent', 'fields', 'summary')
+
+ metadata << "- Parent issue: [#{parent_issue_key}] #{parent_summary_key}".strip
+ end
+
+ def add_versions
+ return if fields['fixVersions'].blank? || !fields['fixVersions'].is_a?(Array)
+
+ versions = fields['fixVersions'].map { |version| version['name'] }.compact.join(', ')
+ metadata << "- Fix versions: #{versions}"
+ end
+
+ def fields
+ jira_issue.fields
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/jira_import/user_mapper.rb b/lib/gitlab/jira_import/user_mapper.rb
new file mode 100644
index 00000000000..208ee49b724
--- /dev/null
+++ b/lib/gitlab/jira_import/user_mapper.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module JiraImport
+ class UserMapper
+ include ::Gitlab::Utils::StrongMemoize
+
+ def initialize(project, jira_user)
+ @project = project
+ @jira_user = jira_user
+ end
+
+ def execute
+ return unless jira_user
+
+ email = jira_user['emailAddress']
+
+ # We also include emails that are not yet confirmed
+ users = User.by_any_email(email).to_a
+
+ user = users.first
+
+ # this event should never happen but we should log it in case we have invalid data
+ log_user_mapping_message('Multiple users found for an email address', email) if users.count > 1
+
+ unless project.project_member(user) || project.group&.group_member(user)
+ log_user_mapping_message('Jira user not found', email)
+
+ return
+ end
+
+ user
+ end
+
+ private
+
+ attr_reader :project, :jira_user, :params
+
+ def log_user_mapping_message(message, email)
+ logger.info(
+ project_id: project.id,
+ project_path: project.full_path,
+ user_email: email,
+ message: message
+ )
+ end
+
+ def logger
+ @logger ||= Gitlab::Import::Logger.build
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb
index 5ebda67e2ae..5b6689dbefe 100644
--- a/lib/gitlab/json.rb
+++ b/lib/gitlab/json.rb
@@ -2,13 +2,25 @@
module Gitlab
module Json
+ INVALID_LEGACY_TYPES = [String, TrueClass, FalseClass].freeze
+
class << self
- def parse(*args)
- adapter.parse(*args)
+ def parse(string, *args, **named_args)
+ legacy_mode = legacy_mode_enabled?(named_args.delete(:legacy_mode))
+ data = adapter.parse(string, *args, **named_args)
+
+ handle_legacy_mode!(data) if legacy_mode
+
+ data
end
- def parse!(*args)
- adapter.parse!(*args)
+ def parse!(string, *args, **named_args)
+ legacy_mode = legacy_mode_enabled?(named_args.delete(:legacy_mode))
+ data = adapter.parse!(string, *args, **named_args)
+
+ handle_legacy_mode!(data) if legacy_mode
+
+ data
end
def dump(*args)
@@ -28,6 +40,20 @@ module Gitlab
def adapter
::JSON
end
+
+ def parser_error
+ ::JSON::ParserError
+ end
+
+ def legacy_mode_enabled?(arg_value)
+ arg_value.nil? ? false : arg_value
+ end
+
+ def handle_legacy_mode!(data)
+ return data unless Feature.enabled?(:json_wrapper_legacy_mode, default_enabled: true)
+
+ raise parser_error if INVALID_LEGACY_TYPES.any? { |type| data.is_a?(type) }
+ end
end
end
end
diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb
index 3e201d68297..00ab7109267 100644
--- a/lib/gitlab/kubernetes/helm.rb
+++ b/lib/gitlab/kubernetes/helm.rb
@@ -3,7 +3,7 @@
module Gitlab
module Kubernetes
module Helm
- HELM_VERSION = '2.16.3'
+ HELM_VERSION = '2.16.6'
KUBECTL_VERSION = '1.13.12'
NAMESPACE = 'gitlab-managed-apps'
NAMESPACE_LABELS = { 'app.gitlab.com/managed_by' => :gitlab }.freeze
diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb
index 3b843799d66..ceda18442d6 100644
--- a/lib/gitlab/kubernetes/helm/api.rb
+++ b/lib/gitlab/kubernetes/helm/api.rb
@@ -99,11 +99,7 @@ module Gitlab
command.cluster_role_binding_resource.tap do |cluster_role_binding_resource|
break unless cluster_role_binding_resource
- if cluster_role_binding_exists?(cluster_role_binding_resource)
- kubeclient.update_cluster_role_binding(cluster_role_binding_resource)
- else
- kubeclient.create_cluster_role_binding(cluster_role_binding_resource)
- end
+ kubeclient.update_cluster_role_binding(cluster_role_binding_resource)
end
end
diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb
index 9d0fd30ba8f..771444ee9ee 100644
--- a/lib/gitlab/kubernetes/helm/delete_command.rb
+++ b/lib/gitlab/kubernetes/helm/delete_command.rb
@@ -36,8 +36,6 @@ module Gitlab
@rbac
end
- private
-
def delete_command
command = ['helm', 'delete', '--purge', name] + tls_flags_if_remote_tiller
diff --git a/lib/gitlab/kubernetes/helm/parsers/list_v2.rb b/lib/gitlab/kubernetes/helm/parsers/list_v2.rb
new file mode 100644
index 00000000000..c5c5d198a6c
--- /dev/null
+++ b/lib/gitlab/kubernetes/helm/parsers/list_v2.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ module Helm
+ module Parsers
+ # Parses Helm v2 list (JSON) output
+ class ListV2
+ ParserError = Class.new(StandardError)
+
+ attr_reader :contents, :json
+
+ def initialize(contents)
+ @contents = contents
+ @json = Gitlab::Json.parse(contents)
+ rescue JSON::ParserError => e
+ raise ParserError, e.message
+ end
+
+ def releases
+ @releases = helm_releases
+ end
+
+ private
+
+ def helm_releases
+ helm_releases = json['Releases'] || []
+
+ raise ParserError, 'Invalid format for Releases' unless helm_releases.all? { |item| item.is_a?(Hash) }
+
+ helm_releases
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb
index 7c5525b982c..2110d586d30 100644
--- a/lib/gitlab/kubernetes/kube_client.rb
+++ b/lib/gitlab/kubernetes/kube_client.rb
@@ -19,7 +19,9 @@ module Gitlab
apps: { group: 'apis/apps', version: 'v1' },
extensions: { group: 'apis/extensions', version: 'v1beta1' },
istio: { group: 'apis/networking.istio.io', version: 'v1alpha3' },
- knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' }
+ knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' },
+ metrics: { group: 'apis/metrics.k8s.io', version: 'v1beta1' },
+ networking: { group: 'apis/networking.k8s.io', version: 'v1' }
}.freeze
SUPPORTED_API_GROUPS.each do |name, params|
@@ -33,7 +35,8 @@ module Gitlab
end
# Core API methods delegates to the core api group client
- delegate :get_pods,
+ delegate :get_nodes,
+ :get_pods,
:get_secrets,
:get_config_map,
:get_namespace,
@@ -56,9 +59,7 @@ module Gitlab
# RBAC methods delegates to the apis/rbac.authorization.k8s.io api
# group client
- delegate :create_cluster_role_binding,
- :get_cluster_role_binding,
- :update_cluster_role_binding,
+ delegate :update_cluster_role_binding,
to: :rbac_client
# RBAC methods delegates to the apis/rbac.authorization.k8s.io api
@@ -70,9 +71,7 @@ module Gitlab
# RBAC methods delegates to the apis/rbac.authorization.k8s.io api
# group client
- delegate :create_role_binding,
- :get_role_binding,
- :update_role_binding,
+ delegate :update_role_binding,
to: :rbac_client
# non-entity methods that can only work with the core client
@@ -88,6 +87,14 @@ module Gitlab
:update_gateway,
to: :istio_client
+ # NetworkPolicy methods delegate to the apis/networking.k8s.io api
+ # group client
+ delegate :create_network_policy,
+ :get_network_policies,
+ :update_network_policy,
+ :delete_network_policy,
+ to: :networking_client
+
attr_reader :api_prefix, :kubeclient_options
DEFAULT_KUBECLIENT_OPTIONS = {
@@ -97,6 +104,31 @@ module Gitlab
}
}.freeze
+ def self.graceful_request(cluster_id)
+ { status: :connected, response: yield }
+ rescue *Gitlab::Kubernetes::Errors::CONNECTION
+ { status: :unreachable }
+ rescue *Gitlab::Kubernetes::Errors::AUTHENTICATION
+ { status: :authentication_failure }
+ rescue Kubeclient::HttpError => e
+ { status: kubeclient_error_status(e.message) }
+ rescue => e
+ Gitlab::ErrorTracking.track_exception(e, cluster_id: cluster_id)
+
+ { status: :unknown_failure }
+ end
+
+ # KubeClient uses the same error class
+ # For connection errors (eg. timeout) and
+ # for Kubernetes errors.
+ def self.kubeclient_error_status(message)
+ if message&.match?(/timed out|timeout/i)
+ :unreachable
+ else
+ :authentication_failure
+ end
+ end
+
# We disable redirects through 'http_max_redirects: 0',
# so that KubeClient does not follow redirects and
# expose internal services.
@@ -125,19 +157,11 @@ module Gitlab
end
def create_or_update_cluster_role_binding(resource)
- if cluster_role_binding_exists?(resource)
- update_cluster_role_binding(resource)
- else
- create_cluster_role_binding(resource)
- end
+ update_cluster_role_binding(resource)
end
def create_or_update_role_binding(resource)
- if role_binding_exists?(resource)
- update_role_binding(resource)
- else
- create_role_binding(resource)
- end
+ update_role_binding(resource)
end
def create_or_update_service_account(resource)
@@ -164,18 +188,6 @@ module Gitlab
Gitlab::UrlBlocker.validate!(api_prefix, allow_local_network: false)
end
- def cluster_role_binding_exists?(resource)
- get_cluster_role_binding(resource.metadata.name)
- rescue ::Kubeclient::ResourceNotFoundError
- false
- end
-
- def role_binding_exists?(resource)
- get_role_binding(resource.metadata.name, resource.metadata.namespace)
- rescue ::Kubeclient::ResourceNotFoundError
- false
- end
-
def service_account_exists?(resource)
get_service_account(resource.metadata.name, resource.metadata.namespace)
rescue ::Kubeclient::ResourceNotFoundError
diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb
new file mode 100644
index 00000000000..ea25d81cbd2
--- /dev/null
+++ b/lib/gitlab/kubernetes/network_policy.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Kubernetes
+ class NetworkPolicy
+ def initialize(name:, namespace:, pod_selector:, ingress:, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil)
+ @name = name
+ @namespace = namespace
+ @creation_timestamp = creation_timestamp
+ @pod_selector = pod_selector
+ @policy_types = policy_types
+ @ingress = ingress
+ @egress = egress
+ end
+
+ def self.from_yaml(manifest)
+ return unless manifest
+
+ policy = YAML.safe_load(manifest, symbolize_names: true)
+ return if !policy[:metadata] || !policy[:spec]
+
+ metadata = policy[:metadata]
+ spec = policy[:spec]
+ self.new(
+ name: metadata[:name],
+ namespace: metadata[:namespace],
+ pod_selector: spec[:podSelector],
+ policy_types: spec[:policyTypes],
+ ingress: spec[:ingress],
+ egress: spec[:egress]
+ )
+ rescue Psych::SyntaxError, Psych::DisallowedClass
+ nil
+ end
+
+ def self.from_resource(resource)
+ return unless resource
+ return if !resource[:metadata] || !resource[:spec]
+
+ metadata = resource[:metadata]
+ spec = resource[:spec].to_h
+ self.new(
+ name: metadata[:name],
+ namespace: metadata[:namespace],
+ creation_timestamp: metadata[:creationTimestamp],
+ pod_selector: spec[:podSelector],
+ policy_types: spec[:policyTypes],
+ ingress: spec[:ingress],
+ egress: spec[:egress]
+ )
+ end
+
+ def generate
+ ::Kubeclient::Resource.new.tap do |resource|
+ resource.metadata = metadata
+ resource.spec = spec
+ end
+ end
+
+ def as_json(opts = nil)
+ {
+ name: name,
+ namespace: namespace,
+ creation_timestamp: creation_timestamp,
+ manifest: manifest
+ }
+ end
+
+ private
+
+ attr_reader :name, :namespace, :creation_timestamp, :pod_selector, :policy_types, :ingress, :egress
+
+ def metadata
+ { name: name, namespace: namespace }
+ end
+
+ def spec
+ {
+ podSelector: pod_selector,
+ policyTypes: policy_types,
+ ingress: ingress,
+ egress: egress
+ }
+ end
+
+ def manifest
+ YAML.dump({ metadata: metadata, spec: spec }.deep_stringify_keys)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/logging/cloudflare_helper.rb b/lib/gitlab/logging/cloudflare_helper.rb
new file mode 100644
index 00000000000..5cffe335bb5
--- /dev/null
+++ b/lib/gitlab/logging/cloudflare_helper.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Logging
+ module CloudflareHelper
+ CLOUDFLARE_CUSTOM_HEADERS = { 'Cf-Ray' => :cf_ray, 'Cf-Request-Id' => :cf_request_id }.freeze
+
+ def store_cloudflare_headers!(payload, request)
+ CLOUDFLARE_CUSTOM_HEADERS.each do |header, value|
+ payload[value] = request.headers[header] if valid_cloudflare_header?(request.headers[header])
+ end
+ end
+
+ def valid_cloudflare_header?(value)
+ return false unless value.present?
+ return false if value.length > 64
+ return false if value.index(/[^[A-Za-z0-9-]]/)
+
+ true
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb
index 145d67d7101..55c46c365f6 100644
--- a/lib/gitlab/lograge/custom_options.rb
+++ b/lib/gitlab/lograge/custom_options.rb
@@ -3,6 +3,8 @@
module Gitlab
module Lograge
module CustomOptions
+ include ::Gitlab::Logging::CloudflareHelper
+
LIMITED_ARRAY_SENTINEL = { key: 'truncated', value: '...' }.freeze
IGNORE_PARAMS = Set.new(%w(controller action format)).freeze
@@ -21,6 +23,8 @@ module Gitlab
queue_duration_s: event.payload[:queue_duration_s]
}
+ payload.merge!(event.payload[:metadata]) if event.payload[:metadata]
+
::Gitlab::InstrumentationHelper.add_instrumentation_data(payload)
payload[:response] = event.payload[:response] if event.payload[:response]
@@ -31,6 +35,10 @@ module Gitlab
payload[:cpu_s] = cpu_s.round(2)
end
+ CLOUDFLARE_CUSTOM_HEADERS.each do |_, value|
+ payload[value] = event.payload[value] if event.payload[value]
+ end
+
# https://github.com/roidrage/lograge#logging-errors--exceptions
exception = event.payload[:exception_object]
diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb
index bd69843adf1..0633efc6b0c 100644
--- a/lib/gitlab/mail_room.rb
+++ b/lib/gitlab/mail_room.rb
@@ -19,7 +19,8 @@ module Gitlab
start_tls: false,
mailbox: 'inbox',
idle_timeout: 60,
- log_path: RAILS_ROOT_DIR.join('log', 'mail_room_json.log')
+ log_path: RAILS_ROOT_DIR.join('log', 'mail_room_json.log'),
+ expunge_deleted: false
}.freeze
# Email specific configuration which is merged with configuration
diff --git a/lib/gitlab/markdown_cache.rb b/lib/gitlab/markdown_cache.rb
index d7a0a9b6518..489fc6fddac 100644
--- a/lib/gitlab/markdown_cache.rb
+++ b/lib/gitlab/markdown_cache.rb
@@ -3,7 +3,7 @@
module Gitlab
module MarkdownCache
# Increment this number every time the renderer changes its output
- CACHE_COMMONMARK_VERSION = 20
+ CACHE_COMMONMARK_VERSION = 21
CACHE_COMMONMARK_VERSION_START = 10
BaseError = Class.new(StandardError)
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index d759ae24051..5fed3d38d7c 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -2,17 +2,106 @@
module Gitlab
module Metrics
- include Gitlab::Metrics::InfluxDb
include Gitlab::Metrics::Prometheus
+ include Gitlab::Metrics::Methods
+
+ EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze
@error = false
def self.enabled?
- influx_metrics_enabled? || prometheus_metrics_enabled?
+ prometheus_metrics_enabled?
end
def self.error?
@error
end
+
+ # Tracks an event.
+ #
+ # See `Gitlab::Metrics::Transaction#add_event` for more details.
+ def self.add_event(*args)
+ current_transaction&.add_event(*args)
+ end
+
+ # Allow access from other metrics related middlewares
+ def self.current_transaction
+ Transaction.current
+ end
+
+ # Returns the prefix to use for the name of a series.
+ def self.series_prefix
+ @series_prefix ||= Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_'
+ end
+
+ def self.settings
+ @settings ||= begin
+ current_settings = Gitlab::CurrentSettings.current_application_settings
+
+ {
+
+ method_call_threshold: current_settings[:metrics_method_call_threshold]
+
+ }
+ end
+ end
+
+ def self.method_call_threshold
+ # This is memoized since this method is called for every instrumented
+ # method. Loading data from an external cache on every method call slows
+ # things down too much.
+ # in milliseconds
+ @method_call_threshold ||= settings[:method_call_threshold]
+ end
+
+ # Measures the execution time of a block.
+ #
+ # Example:
+ #
+ # Gitlab::Metrics.measure(:find_by_username_duration) do
+ # UserFinder.new(some_username).find_by_username
+ # end
+ #
+ # name - The name of the field to store the execution time in.
+ #
+ # Returns the value yielded by the supplied block.
+ def self.measure(name)
+ trans = current_transaction
+
+ return yield unless trans
+
+ real_start = System.monotonic_time
+ cpu_start = System.cpu_time
+
+ retval = yield
+
+ cpu_stop = System.cpu_time
+ real_stop = System.monotonic_time
+
+ real_time = (real_stop - real_start)
+ cpu_time = cpu_stop - cpu_start
+
+ real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do
+ docstring "Measure #{name}"
+ base_labels Transaction::BASE_LABELS
+ buckets EXECUTION_MEASUREMENT_BUCKETS
+ end
+
+ real_duration_seconds.observe(trans.labels, real_time)
+
+ cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do
+ docstring "Measure #{name}"
+ base_labels Transaction::BASE_LABELS
+ buckets EXECUTION_MEASUREMENT_BUCKETS
+ with_feature "prometheus_metrics_measure_#{name}_cpu_duration"
+ end
+ cpu_duration_seconds.observe(trans.labels, cpu_time)
+
+ trans.increment("#{name}_real_time", real_time.in_milliseconds, false)
+ trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false)
+ trans.increment("#{name}_call_count", 1, false)
+
+ retval
+ end
end
end
diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb
index 1d948883151..31670a3f533 100644
--- a/lib/gitlab/metrics/dashboard/url.rb
+++ b/lib/gitlab/metrics/dashboard/url.rb
@@ -23,7 +23,7 @@ module Gitlab
%r{
/environments
/(?<environment>\d+)
- /metrics
+ /(metrics_dashboard|metrics)
}x
)
end
diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
index 5ba7b29734b..054b4949dd6 100644
--- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
+++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb
@@ -32,7 +32,7 @@ module Gitlab
Sidekiq.logger.error(
class: self.class.to_s,
message: 'Cannot start sidekiq_exporter',
- exception: e.message
+ 'exception.message' => e.message
)
false
diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb
deleted file mode 100644
index 1f252572461..00000000000
--- a/lib/gitlab/metrics/influx_db.rb
+++ /dev/null
@@ -1,183 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Metrics
- module InfluxDb
- extend ActiveSupport::Concern
- include Gitlab::Metrics::Methods
-
- EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze
-
- MUTEX = Mutex.new
- private_constant :MUTEX
-
- class_methods do
- def influx_metrics_enabled?
- settings[:enabled] || false
- end
-
- # Prometheus histogram buckets used for arbitrary code measurements
-
- def settings
- @settings ||= begin
- current_settings = Gitlab::CurrentSettings.current_application_settings
-
- {
- enabled: current_settings[:metrics_enabled],
- pool_size: current_settings[:metrics_pool_size],
- timeout: current_settings[:metrics_timeout],
- method_call_threshold: current_settings[:metrics_method_call_threshold],
- host: current_settings[:metrics_host],
- port: current_settings[:metrics_port],
- sample_interval: current_settings[:metrics_sample_interval] || 15,
- packet_size: current_settings[:metrics_packet_size] || 1
- }
- end
- end
-
- def mri?
- RUBY_ENGINE == 'ruby'
- end
-
- def method_call_threshold
- # This is memoized since this method is called for every instrumented
- # method. Loading data from an external cache on every method call slows
- # things down too much.
- # in milliseconds
- @method_call_threshold ||= settings[:method_call_threshold]
- end
-
- def submit_metrics(metrics)
- prepared = prepare_metrics(metrics)
-
- pool&.with do |connection|
- prepared.each_slice(settings[:packet_size]) do |slice|
- connection.write_points(slice)
- rescue StandardError
- end
- end
- rescue Errno::EADDRNOTAVAIL, SocketError => ex
- Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
- Gitlab::EnvironmentLogger.error(ex)
- end
-
- def prepare_metrics(metrics)
- metrics.map do |hash|
- new_hash = hash.symbolize_keys
-
- new_hash[:tags].each do |key, value|
- if value.blank?
- new_hash[:tags].delete(key)
- else
- new_hash[:tags][key] = escape_value(value)
- end
- end
-
- new_hash
- end
- end
-
- def escape_value(value)
- value.to_s.gsub('=', '\\=')
- end
-
- # Measures the execution time of a block.
- #
- # Example:
- #
- # Gitlab::Metrics.measure(:find_by_username_duration) do
- # UserFinder.new(some_username).find_by_username
- # end
- #
- # name - The name of the field to store the execution time in.
- #
- # Returns the value yielded by the supplied block.
- def measure(name)
- trans = current_transaction
-
- return yield unless trans
-
- real_start = Time.now.to_f
- cpu_start = System.cpu_time
-
- retval = yield
-
- cpu_stop = System.cpu_time
- real_stop = Time.now.to_f
-
- real_time = (real_stop - real_start)
- cpu_time = cpu_stop - cpu_start
-
- real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do
- docstring "Measure #{name}"
- base_labels Transaction::BASE_LABELS
- buckets EXECUTION_MEASUREMENT_BUCKETS
- end
-
- real_duration_seconds.observe(trans.labels, real_time)
-
- cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do
- docstring "Measure #{name}"
- base_labels Transaction::BASE_LABELS
- buckets EXECUTION_MEASUREMENT_BUCKETS
- with_feature "prometheus_metrics_measure_#{name}_cpu_duration"
- end
- cpu_duration_seconds.observe(trans.labels, cpu_time)
-
- # InfluxDB stores the _real_time and _cpu_time time values as milliseconds
- trans.increment("#{name}_real_time", real_time.in_milliseconds, false)
- trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false)
- trans.increment("#{name}_call_count", 1, false)
-
- retval
- end
-
- # Sets the action of the current transaction (if any)
- #
- # action - The name of the action.
- def action=(action)
- trans = current_transaction
-
- trans&.action = action
- end
-
- # Tracks an event.
- #
- # See `Gitlab::Metrics::Transaction#add_event` for more details.
- def add_event(*args)
- current_transaction&.add_event(*args)
- end
-
- # Returns the prefix to use for the name of a series.
- def series_prefix
- @series_prefix ||= Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_'
- end
-
- # Allow access from other metrics related middlewares
- def current_transaction
- Transaction.current
- end
-
- # When enabled this should be set before being used as the usual pattern
- # "@foo ||= bar" is _not_ thread-safe.
- def pool
- if influx_metrics_enabled?
- if @pool.nil?
- MUTEX.synchronize do
- @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
- host = settings[:host]
- port = settings[:port]
-
- InfluxDB::Client
- .new(udp: { host: host, port: port })
- end
- end
- end
-
- @pool
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb
index d0c63a862c2..fbeda3b75e0 100644
--- a/lib/gitlab/metrics/method_call.rb
+++ b/lib/gitlab/metrics/method_call.rb
@@ -49,19 +49,6 @@ module Gitlab
retval
end
- # Returns a Metric instance of the current method call.
- def to_metric
- Metric.new(
- Instrumentation.series,
- {
- duration: real_time.in_milliseconds.to_i,
- cpu_duration: cpu_time.in_milliseconds.to_i,
- call_count: call_count
- },
- method: @name
- )
- end
-
# Returns true if the total runtime of this method exceeds the method call
# threshold.
def above_threshold?
diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb
deleted file mode 100644
index 30f181542be..00000000000
--- a/lib/gitlab/metrics/metric.rb
+++ /dev/null
@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Metrics
- # Class for storing details of a single metric (label, value, etc).
- class Metric
- JITTER_RANGE = (0.000001..0.001).freeze
-
- attr_reader :series, :values, :tags, :type
-
- # series - The name of the series (as a String) to store the metric in.
- # values - A Hash containing the values to store.
- # tags - A Hash containing extra tags to add to the metrics.
- def initialize(series, values, tags = {}, type = :metric)
- @values = values
- @series = series
- @tags = tags
- @type = type
- end
-
- def event?
- type == :event
- end
-
- # Returns a Hash in a format that can be directly written to InfluxDB.
- def to_hash
- # InfluxDB overwrites an existing point if a new point has the same
- # series, tag set, and timestamp. In a highly concurrent environment
- # this means that using the number of seconds since the Unix epoch is
- # inevitably going to collide with another timestamp. For example, two
- # Rails requests processed by different processes may end up generating
- # metrics using the _exact_ same timestamp (in seconds).
- #
- # Due to the way InfluxDB is set up there's no solution to this problem,
- # all we can do is lower the amount of collisions. We do this by using
- # System.real_time which returns the nanoseconds as a Float providing
- # greater accuracy. We then add a small random value that is large
- # enough to distinguish most timestamps but small enough to not alter
- # the timestamp significantly.
- #
- # See https://gitlab.com/gitlab-com/operations/issues/175 for more
- # information.
- time = System.real_time(:nanosecond) + rand(JITTER_RANGE)
-
- {
- series: @series,
- tags: @tags,
- values: @values,
- timestamp: time.to_i
- }
- end
- end
- end
-end
diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb
index 9aa97515961..c6a0457ffe5 100644
--- a/lib/gitlab/metrics/rack_middleware.rb
+++ b/lib/gitlab/metrics/rack_middleware.rb
@@ -20,10 +20,6 @@ module Gitlab
trans.add_event(:rails_exception)
raise error
- # Even in the event of an error we want to submit any metrics we
- # might've gathered up to this point.
- ensure
- trans.finish
end
retval
diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb
new file mode 100644
index 00000000000..9ee4b0960c5
--- /dev/null
+++ b/lib/gitlab/metrics/samplers/database_sampler.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Samplers
+ class DatabaseSampler < BaseSampler
+ SAMPLING_INTERVAL_SECONDS = 5
+
+ METRIC_PREFIX = 'gitlab_database_connection_pool_'
+
+ METRIC_DESCRIPTIONS = {
+ size: 'Total connection pool capacity',
+ connections: 'Current connections in the pool',
+ busy: 'Connections in use where the owner is still alive',
+ dead: 'Connections in use where the owner is not alive',
+ idle: 'Connections not in use',
+ waiting: 'Threads currently waiting on this queue'
+ }.freeze
+
+ def metrics
+ @metrics ||= init_metrics
+ end
+
+ def sample
+ host_stats.each do |host_stat|
+ METRIC_DESCRIPTIONS.each_key do |metric|
+ metrics[metric].set(host_stat[:labels], host_stat[:stats][metric])
+ end
+ end
+ end
+
+ private
+
+ def init_metrics
+ METRIC_DESCRIPTIONS.map do |name, description|
+ [name, ::Gitlab::Metrics.gauge(:"#{METRIC_PREFIX}#{name}", description)]
+ end.to_h
+ end
+
+ def host_stats
+ return [] unless ActiveRecord::Base.connected?
+
+ [{ labels: labels_for_class(ActiveRecord::Base), stats: ActiveRecord::Base.connection_pool.stat }]
+ end
+
+ def labels_for_class(klass)
+ {
+ host: klass.connection_config[:host],
+ port: klass.connection_config[:port],
+ class: klass.to_s
+ }
+ end
+ end
+ end
+ end
+end
+
+Gitlab::Metrics::Samplers::DatabaseSampler.prepend_if_ee('EE::Gitlab::Metrics::Samplers::DatabaseSampler')
diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb
deleted file mode 100644
index 4e16e335bee..00000000000
--- a/lib/gitlab/metrics/samplers/influx_sampler.rb
+++ /dev/null
@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Metrics
- module Samplers
- # Class that sends certain metrics to InfluxDB at a specific interval.
- #
- # This class is used to gather statistics that can't be directly associated
- # with a transaction such as system memory usage, garbage collection
- # statistics, etc.
- class InfluxSampler < BaseSampler
- # interval - The sampling interval in seconds.
- def initialize(interval = ::Gitlab::Metrics.settings[:sample_interval])
- super(interval)
- @last_step = nil
-
- @metrics = []
- end
-
- def sample
- sample_memory_usage
- sample_file_descriptors
-
- flush
- ensure
- @metrics.clear
- end
-
- def flush
- ::Gitlab::Metrics.submit_metrics(@metrics.map(&:to_hash))
- end
-
- def sample_memory_usage
- add_metric('memory_usage', value: System.memory_usage)
- end
-
- def sample_file_descriptors
- add_metric('file_descriptors', value: System.file_descriptor_count)
- end
-
- def add_metric(series, values, tags = {})
- prefix = Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_'
-
- @metrics << Metric.new("#{prefix}#{series}", values, tags)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb
index f788f51b1ce..98dd517ee3b 100644
--- a/lib/gitlab/metrics/samplers/puma_sampler.rb
+++ b/lib/gitlab/metrics/samplers/puma_sampler.rb
@@ -26,7 +26,7 @@ module Gitlab
json_stats = puma_stats
return unless json_stats
- stats = JSON.parse(json_stats)
+ stats = Gitlab::Json.parse(json_stats)
if cluster?(stats)
sample_cluster(stats)
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index c38769f39a9..df59c06911b 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -34,14 +34,15 @@ module Gitlab
def init_metrics
metrics = {
- file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels),
- memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels),
- process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'),
- process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'),
- process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used', labels),
- process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'),
- sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
- gc_duration_seconds: ::Gitlab::Metrics.histogram(with_prefix(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS)
+ file_descriptors: ::Gitlab::Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels),
+ process_cpu_seconds_total: ::Gitlab::Metrics.gauge(with_prefix(:process, :cpu_seconds_total), 'Process CPU seconds total'),
+ process_max_fds: ::Gitlab::Metrics.gauge(with_prefix(:process, :max_fds), 'Process max fds'),
+ process_resident_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :resident_memory_bytes), 'Memory used (RSS)', labels),
+ process_unique_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :unique_memory_bytes), 'Memory used (USS)', labels),
+ process_proportional_memory_bytes: ::Gitlab::Metrics.gauge(with_prefix(:process, :proportional_memory_bytes), 'Memory used (PSS)', labels),
+ process_start_time_seconds: ::Gitlab::Metrics.gauge(with_prefix(:process, :start_time_seconds), 'Process start time seconds'),
+ sampler_duration: ::Gitlab::Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels),
+ gc_duration_seconds: ::Gitlab::Metrics.histogram(with_prefix(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS)
}
GC.stat.keys.each do |key|
@@ -85,10 +86,13 @@ module Gitlab
end
def set_memory_usage_metrics
- memory_usage = System.memory_usage
+ metrics[:process_resident_memory_bytes].set(labels, System.memory_usage_rss)
- metrics[:memory_bytes].set(labels, memory_usage)
- metrics[:process_resident_memory_bytes].set(labels, memory_usage)
+ if Gitlab::Utils.to_boolean(ENV['enable_memory_uss_pss'] || '1')
+ memory_uss_pss = System.memory_usage_uss_pss
+ metrics[:process_unique_memory_bytes].set(labels, memory_uss_pss[:uss])
+ metrics[:process_proportional_memory_bytes].set(labels, memory_uss_pss[:pss])
+ end
end
end
end
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index 0b4485feea9..8dfb61046c4 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -17,8 +17,6 @@ module Gitlab
trans.add_event(:sidekiq_exception)
raise error
- ensure
- trans.finish
end
end
end
diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb
index 5bd21b8e5d1..24107e42aa9 100644
--- a/lib/gitlab/metrics/subscribers/action_view.rb
+++ b/lib/gitlab/metrics/subscribers/action_view.rb
@@ -26,23 +26,17 @@ module Gitlab
private
def track(event)
- values = values_for(event)
- tags = tags_for(event)
+ tags = tags_for(event)
self.class.gitlab_view_rendering_duration_seconds.observe(current_transaction.labels.merge(tags), event.duration)
current_transaction.increment(:view_duration, event.duration)
- current_transaction.add_metric(SERIES, values, tags)
end
def relative_path(path)
path.gsub(%r{^#{Rails.root}/?}, '')
end
- def values_for(event)
- { duration: event.duration }
- end
-
def tags_for(event)
path = relative_path(event.payload[:identifier])
diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb
index 2a61b3de405..43005303dec 100644
--- a/lib/gitlab/metrics/system.rb
+++ b/lib/gitlab/metrics/system.rb
@@ -7,47 +7,37 @@ module Gitlab
# This module relies on the /proc filesystem being available. If /proc is
# not available the methods of this module will be stubbed.
module System
- if File.exist?('/proc')
- # Returns the current process' memory usage in bytes.
- def self.memory_usage
- mem = 0
- match = File.read('/proc/self/status').match(/VmRSS:\s+(\d+)/)
-
- if match && match[1]
- mem = match[1].to_f * 1024
- end
-
- mem
- end
-
- def self.file_descriptor_count
- Dir.glob('/proc/self/fd/*').length
- end
-
- def self.max_open_file_descriptors
- match = File.read('/proc/self/limits').match(/Max open files\s*(\d+)/)
-
- return unless match && match[1]
+ PROC_STATUS_PATH = '/proc/self/status'
+ PROC_SMAPS_ROLLUP_PATH = '/proc/self/smaps_rollup'
+ PROC_LIMITS_PATH = '/proc/self/limits'
+ PROC_FD_GLOB = '/proc/self/fd/*'
+
+ PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/.freeze
+ PSS_PATTERN = /^Pss:\s+(?<value>\d+)/.freeze
+ RSS_PATTERN = /VmRSS:\s+(?<value>\d+)/.freeze
+ MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze
+
+ # Returns the current process' RSS (resident set size) in bytes.
+ def self.memory_usage_rss
+ sum_matches(PROC_STATUS_PATH, rss: RSS_PATTERN)[:rss].kilobytes
+ end
- match[1].to_i
- end
- else
- def self.memory_usage
- 0.0
- end
+ # Returns the current process' USS/PSS (unique/proportional set size) in bytes.
+ def self.memory_usage_uss_pss
+ sum_matches(PROC_SMAPS_ROLLUP_PATH, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN)
+ .transform_values(&:kilobytes)
+ end
- def self.file_descriptor_count
- 0
- end
+ def self.file_descriptor_count
+ Dir.glob(PROC_FD_GLOB).length
+ end
- def self.max_open_file_descriptors
- 0
- end
+ def self.max_open_file_descriptors
+ sum_matches(PROC_LIMITS_PATH, max_fds: MAX_OPEN_FILES_PATTERN)[:max_fds]
end
def self.cpu_time
- Process
- .clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second)
+ Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second)
end
# Returns the current real time in a given precision.
@@ -78,6 +68,27 @@ module Gitlab
end_time - start_time
end
+
+ # Given a path to a file in /proc and a hash of (metric, pattern) pairs,
+ # sums up all values found for those patterns under the respective metric.
+ def self.sum_matches(proc_file, **patterns)
+ results = patterns.transform_values { 0 }
+
+ begin
+ File.foreach(proc_file) do |line|
+ patterns.each do |metric, pattern|
+ match = line.match(pattern)
+ value = match&.named_captures&.fetch('value', 0)
+ results[metric] += value.to_i
+ end
+ end
+ rescue Errno::ENOENT
+ # This means the procfile we're reading from did not exist;
+ # this is safe to ignore, since we initialize each metric to 0
+ end
+
+ results
+ end
end
end
end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index 552eae639e6..b126efd2dd5 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -16,20 +16,18 @@ module Gitlab
# The series to store events (e.g. Git pushes) in.
EVENT_SERIES = 'events'
- attr_reader :tags, :values, :method, :metrics
+ attr_reader :tags, :method
def self.current
Thread.current[THREAD_KEY]
end
def initialize
- @metrics = []
@methods = {}
@started_at = nil
@finished_at = nil
- @values = Hash.new(0)
@tags = {}
@memory_before = 0
@@ -40,10 +38,6 @@ module Gitlab
@finished_at ? (@finished_at - @started_at) : 0.0
end
- def duration_milliseconds
- duration.in_milliseconds.to_i
- end
-
def thread_cpu_duration
System.thread_cpu_duration(@thread_cputime_start)
end
@@ -55,13 +49,13 @@ module Gitlab
def run
Thread.current[THREAD_KEY] = self
- @memory_before = System.memory_usage
+ @memory_before = System.memory_usage_rss
@started_at = System.monotonic_time
@thread_cputime_start = System.thread_cpu_time
yield
ensure
- @memory_after = System.memory_usage
+ @memory_after = System.memory_usage_rss
@finished_at = System.monotonic_time
self.class.gitlab_transaction_cputime_seconds.observe(labels, thread_cpu_duration)
@@ -71,10 +65,6 @@ module Gitlab
Thread.current[THREAD_KEY] = nil
end
- def add_metric(series, values, tags = {})
- @metrics << Metric.new("#{::Gitlab::Metrics.series_prefix}#{series}", values, filter_tags(tags))
- end
-
# Tracks a business level event
#
# Business level events including events such as Git pushes, Emails being
@@ -85,7 +75,6 @@ module Gitlab
def add_event(event_name, tags = {})
filtered_tags = filter_tags(tags)
self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: filtered_tags).increment(filtered_tags.merge(labels))
- @metrics << Metric.new(EVENT_SERIES, { count: 1 }, filtered_tags.merge(event: event_name), :event)
end
# Returns a MethodCall object for the given name.
@@ -99,55 +88,16 @@ module Gitlab
def increment(name, value, use_prometheus = true)
self.class.transaction_metric(name, :counter).increment(labels, value) if use_prometheus
- @values[name] += value
end
def set(name, value, use_prometheus = true)
self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus
- @values[name] = value
- end
-
- def finish
- track_self
- submit
- end
-
- def track_self
- values = { duration: duration_milliseconds, allocated_memory: allocated_memory }
-
- @values.each do |name, value|
- values[name] = value
- end
-
- add_metric('transactions', values, @tags)
- end
-
- def submit
- submit = @metrics.dup
-
- @methods.each do |name, method|
- submit << method.to_metric if method.above_threshold?
- end
-
- submit_hashes = submit.map do |metric|
- hash = metric.to_hash
- hash[:tags][:action] ||= action if action && !metric.event?
-
- hash
- end
-
- ::Gitlab::Metrics.submit_metrics(submit_hashes)
end
def labels
BASE_LABELS
end
- # returns string describing the action performed, usually the class plus method name.
- def action
- "#{labels[:controller]}##{labels[:action]}" if labels && !labels.empty?
- end
-
define_histogram :gitlab_transaction_cputime_seconds do
docstring 'Transaction thread cputime'
base_labels BASE_LABELS
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index 7d0de3aee1c..3c45f841653 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -110,6 +110,7 @@ module Gitlab
::FileUploader.root,
Gitlab.config.uploads.storage_path,
JobArtifactUploader.workhorse_upload_path,
+ LfsObjectUploader.workhorse_upload_path,
File.join(Rails.root, 'public/uploads/tmp')
]
end
diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb
index cdab86540f8..1c49379e8d2 100644
--- a/lib/gitlab/middleware/read_only/controller.rb
+++ b/lib/gitlab/middleware/read_only/controller.rb
@@ -4,8 +4,6 @@ module Gitlab
module Middleware
class ReadOnly
class Controller
- prepend_if_ee('EE::Gitlab::Middleware::ReadOnly::Controller') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze
APPLICATION_JSON = 'application/json'
APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze
@@ -144,3 +142,5 @@ module Gitlab
end
end
end
+
+Gitlab::Middleware::ReadOnly::Controller.prepend_if_ee('EE::Gitlab::Middleware::ReadOnly::Controller')
diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb
index c051a581837..b60ecb6631b 100644
--- a/lib/gitlab/omniauth_initializer.rb
+++ b/lib/gitlab/omniauth_initializer.rb
@@ -2,7 +2,7 @@
module Gitlab
class OmniauthInitializer
- prepend_if_ee('::EE::Gitlab::OmniauthInitializer') # rubocop: disable Cop/InjectEnterpriseEditionModule
+ OAUTH2_TIMEOUT_SECONDS = 10
def initialize(devise_config)
@devise_config = devise_config
@@ -17,6 +17,47 @@ module Gitlab
end
end
+ class << self
+ def default_arguments_for(provider_name)
+ case provider_name
+ when 'cas3'
+ { on_single_sign_out: cas3_signout_handler }
+ when 'authentiq'
+ { remote_sign_out_handler: authentiq_signout_handler }
+ when 'shibboleth'
+ { fail_with_empty_uid: true }
+ when 'google_oauth2'
+ { client_options: { connection_opts: { request: { timeout: OAUTH2_TIMEOUT_SECONDS } } } }
+ else
+ {}
+ end
+ end
+
+ private
+
+ def cas3_signout_handler
+ lambda do |request|
+ ticket = request.params[:session_index]
+ raise "Service Ticket not found." unless Gitlab::Auth::OAuth::Session.valid?(:cas3, ticket)
+
+ Gitlab::Auth::OAuth::Session.destroy(:cas3, ticket)
+ true
+ end
+ end
+
+ def authentiq_signout_handler
+ lambda do |request|
+ authentiq_session = request.params['sid']
+ if Gitlab::Auth::OAuth::Session.valid?(:authentiq, authentiq_session)
+ Gitlab::Auth::OAuth::Session.destroy(:authentiq, authentiq_session)
+ true
+ else
+ false
+ end
+ end
+ end
+ end
+
private
def add_provider_to_devise(*args)
@@ -35,7 +76,8 @@ module Gitlab
# An Array from the configuration will be expanded.
provider_arguments.concat provider['args']
when Hash
- hash_arguments = provider['args'].merge(provider_defaults(provider))
+ defaults = provider_defaults(provider)
+ hash_arguments = provider['args'].deep_symbolize_keys.deep_merge(defaults)
# A Hash from the configuration will be passed as is.
provider_arguments << normalize_hash_arguments(hash_arguments)
@@ -45,7 +87,7 @@ module Gitlab
end
def normalize_hash_arguments(args)
- args.symbolize_keys!
+ args.deep_symbolize_keys!
# Rails 5.1 deprecated the use of string names in the middleware
# (https://github.com/rails/rails/commit/83b767ce), so we need to
@@ -68,38 +110,7 @@ module Gitlab
end
def provider_defaults(provider)
- case provider['name']
- when 'cas3'
- { on_single_sign_out: cas3_signout_handler }
- when 'authentiq'
- { remote_sign_out_handler: authentiq_signout_handler }
- when 'shibboleth'
- { fail_with_empty_uid: true }
- else
- {}
- end
- end
-
- def cas3_signout_handler
- lambda do |request|
- ticket = request.params[:session_index]
- raise "Service Ticket not found." unless Gitlab::Auth::OAuth::Session.valid?(:cas3, ticket)
-
- Gitlab::Auth::OAuth::Session.destroy(:cas3, ticket)
- true
- end
- end
-
- def authentiq_signout_handler
- lambda do |request|
- authentiq_session = request.params['sid']
- if Gitlab::Auth::OAuth::Session.valid?(:authentiq, authentiq_session)
- Gitlab::Auth::OAuth::Session.destroy(:authentiq, authentiq_session)
- true
- else
- false
- end
- end
+ self.class.default_arguments_for(provider['name'])
end
def omniauth_customized_providers
@@ -121,3 +132,5 @@ module Gitlab
end
end
end
+
+Gitlab::OmniauthInitializer.prepend_if_ee('::EE::Gitlab::OmniauthInitializer')
diff --git a/lib/gitlab/pagination/keyset.rb b/lib/gitlab/pagination/keyset.rb
index 8692f30e165..67a5530d46c 100644
--- a/lib/gitlab/pagination/keyset.rb
+++ b/lib/gitlab/pagination/keyset.rb
@@ -3,11 +3,18 @@
module Gitlab
module Pagination
module Keyset
+ SUPPORTED_TYPES = [
+ Project
+ ].freeze
+
+ def self.available_for_type?(relation)
+ SUPPORTED_TYPES.include?(relation.klass)
+ end
+
def self.available?(request_context, relation)
order_by = request_context.page.order_by
- # This is only available for Project and order-by id (asc/desc)
- return false unless relation.klass == Project
+ return false unless available_for_type?(relation)
return false unless order_by.size == 1 && order_by[:id]
true
diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb
index 4d1b57fbbbb..f5fcd5c6093 100644
--- a/lib/gitlab/patch/draw_route.rb
+++ b/lib/gitlab/patch/draw_route.rb
@@ -5,8 +5,6 @@
module Gitlab
module Patch
module DrawRoute
- prepend_if_ee('EE::Gitlab::Patch::DrawRoute') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
RoutesNotFound = Class.new(StandardError)
def draw(routes_name)
@@ -38,3 +36,5 @@ module Gitlab
end
end
end
+
+Gitlab::Patch::DrawRoute.prepend_if_ee('EE::Gitlab::Patch::DrawRoute')
diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb
index 4445c876e7a..e26309b5dfd 100644
--- a/lib/gitlab/performance_bar.rb
+++ b/lib/gitlab/performance_bar.rb
@@ -44,7 +44,7 @@ module Gitlab
end
def self.l1_cache_backend
- Gitlab::ThreadMemoryCache.cache_backend
+ Gitlab::ProcessMemoryCache.cache_backend
end
def self.l2_cache_backend
diff --git a/lib/gitlab/phabricator_import/conduit/response.rb b/lib/gitlab/phabricator_import/conduit/response.rb
index 6053ecfbd5e..1b03cfa05e6 100644
--- a/lib/gitlab/phabricator_import/conduit/response.rb
+++ b/lib/gitlab/phabricator_import/conduit/response.rb
@@ -9,7 +9,7 @@ module Gitlab
"Phabricator responded with #{http_response.status}"
end
- response = new(JSON.parse(http_response.body))
+ response = new(Gitlab::Json.parse(http_response.body))
unless response.success?
raise ResponseError,
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index eb7ca80dd60..fbdfe166645 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -2,30 +2,29 @@
module Gitlab
class ProjectSearchResults < SearchResults
- attr_reader :project, :repository_ref, :per_page
+ attr_reader :project, :repository_ref
- def initialize(current_user, project, query, repository_ref = nil, per_page: 20)
+ def initialize(current_user, project, query, repository_ref = nil)
@current_user = current_user
@project = project
@repository_ref = repository_ref.presence
@query = query
- @per_page = per_page
end
- def objects(scope, page = nil)
+ def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE)
case scope
when 'notes'
notes.page(page).per(per_page)
when 'blobs'
- paginated_blobs(blobs(page), page)
+ paginated_blobs(blobs(limit: limit_up_to_page(page, per_page)), page, per_page)
when 'wiki_blobs'
- paginated_blobs(wiki_blobs, page)
+ paginated_wiki_blobs(wiki_blobs(limit: limit_up_to_page(page, per_page)), page, per_page)
when 'commits'
Kaminari.paginate_array(commits).page(page).per(per_page)
when 'users'
users.page(page).per(per_page)
else
- super(scope, page, false)
+ super(scope, page: page, per_page: per_page, without_count: false)
end
end
@@ -49,7 +48,7 @@ module Gitlab
end
def limited_blobs_count
- @limited_blobs_count ||= blobs.count
+ @limited_blobs_count ||= blobs(limit: count_limit).count
end
# rubocop: disable CodeReuse/ActiveRecord
@@ -69,7 +68,7 @@ module Gitlab
# rubocop: enable CodeReuse/ActiveRecord
def wiki_blobs_count
- @wiki_blobs_count ||= wiki_blobs.count
+ @wiki_blobs_count ||= wiki_blobs(limit: count_limit).count
end
def commits_count
@@ -87,7 +86,7 @@ module Gitlab
private
- def paginated_blobs(blobs, page)
+ def paginated_blobs(blobs, page, per_page)
results = Kaminari.paginate_array(blobs).page(page).per(per_page)
Gitlab::Search::FoundBlob.preload_blobs(results)
@@ -95,19 +94,26 @@ module Gitlab
results
end
- def limit_up_to_page(page)
+ def paginated_wiki_blobs(blobs, page, per_page)
+ blob_array = paginated_blobs(blobs, page, per_page)
+ blob_array.map! do |blob|
+ Gitlab::Search::FoundWikiPage.new(blob)
+ end
+ end
+
+ def limit_up_to_page(page, per_page)
current_page = page&.to_i || 1
offset = per_page * (current_page - 1)
count_limit + offset
end
- def blobs(page = 1)
+ def blobs(limit: count_limit)
return [] unless Ability.allowed?(@current_user, :download_code, @project)
- @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref).find(query, content_match_cutoff: limit_up_to_page(page))
+ @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref).find(query, content_match_cutoff: limit)
end
- def wiki_blobs
+ def wiki_blobs(limit: count_limit)
return [] unless Ability.allowed?(@current_user, :read_wiki, @project)
@wiki_blobs ||= begin
@@ -115,7 +121,7 @@ module Gitlab
if project.wiki.empty?
[]
else
- Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query)
+ Gitlab::WikiFileFinder.new(project, repository_wiki_ref).find(query, content_match_cutoff: limit)
end
else
[]
diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb
index 1b6f7282eb3..4a39260a340 100644
--- a/lib/gitlab/prometheus/metric_group.rb
+++ b/lib/gitlab/prometheus/metric_group.rb
@@ -3,7 +3,6 @@
module Gitlab
module Prometheus
class MetricGroup
- prepend_if_ee('EE::Gitlab::Prometheus::MetricGroup') # rubocop: disable Cop/InjectEnterpriseEditionModule
include ActiveModel::Model
attr_accessor :name, :priority, :metrics
@@ -31,3 +30,5 @@ module Gitlab
end
end
end
+
+Gitlab::Prometheus::MetricGroup.prepend_if_ee('EE::Gitlab::Prometheus::MetricGroup')
diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
index a5e7d0ac9d5..d24b98e790b 100644
--- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb
+++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
@@ -4,8 +4,6 @@ module Gitlab
module Prometheus
module Queries
module QueryAdditionalMetrics
- prepend_if_ee('EE::Gitlab::Prometheus::Queries::QueryAdditionalMetrics') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
def query_metrics(project, environment, query_context)
matched_metrics(project).map(&query_group(query_context))
.select(&method(:group_with_any_metrics))
@@ -99,3 +97,5 @@ module Gitlab
end
end
end
+
+Gitlab::Prometheus::Queries::QueryAdditionalMetrics.prepend_if_ee('EE::Gitlab::Prometheus::Queries::QueryAdditionalMetrics')
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index 71a0d528bd7..b03d8a4d254 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -163,7 +163,7 @@ module Gitlab
end
def parse_json(response_body)
- JSON.parse(response_body)
+ Gitlab::Json.parse(response_body, legacy_mode: true)
rescue JSON::ParserError
raise PrometheusClient::Error, 'Parsing response failed'
end
diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb
index 942f90e8040..6aa3f515ef0 100644
--- a/lib/gitlab/quick_actions/issuable_actions.rb
+++ b/lib/gitlab/quick_actions/issuable_actions.rb
@@ -109,7 +109,7 @@ module Gitlab
quick_action_target.labels.any? &&
current_user.can?(:"admin_#{quick_action_target.to_ability_name}", parent)
end
- command :unlabel do |labels_param = nil|
+ command :unlabel, :remove_label do |labels_param = nil|
if labels_param.present?
labels = find_labels(labels_param)
label_ids = labels.map(&:id)
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index db531f06f11..c8b04ce2a5c 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -79,6 +79,12 @@ module Gitlab
"Must start with a letter, and cannot end with '-'"
end
+ # Pod name adheres to DNS Subdomain Names(RFC 1123) naming convention
+ # https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
+ def kubernetes_dns_subdomain_regex
+ /\A[a-z0-9]([a-z0-9\-\.]*[a-z0-9])?\z/
+ end
+
def environment_slug_regex
@environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze
end
@@ -121,7 +127,7 @@ module Gitlab
# Based on Jira's project key format
# https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html
def jira_issue_key_regex
- @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+\b/
+ @jira_issue_key_regex ||= /[A-Z][A-Z_0-9]+-\d+/
end
def jira_transition_id_regex
diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb
index 9da6732796a..952ae55d90a 100644
--- a/lib/gitlab/request_context.rb
+++ b/lib/gitlab/request_context.rb
@@ -24,11 +24,12 @@ module Gitlab
end
def ensure_deadline_not_exceeded!
+ return unless enabled?
return unless request_deadline
return if Gitlab::Metrics::System.real_time < request_deadline
raise RequestDeadlineExceeded,
- "Request takes longer than #{max_request_duration_seconds}"
+ "Request takes longer than #{max_request_duration_seconds} seconds"
end
private
@@ -36,5 +37,9 @@ module Gitlab
def max_request_duration_seconds
Settings.gitlab.max_request_duration_seconds
end
+
+ def enabled?
+ !Rails.env.test?
+ end
end
end
diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb
index c2b55431547..9a5917ffba9 100644
--- a/lib/gitlab/rugged_instrumentation.rb
+++ b/lib/gitlab/rugged_instrumentation.rb
@@ -3,7 +3,8 @@
module Gitlab
module RuggedInstrumentation
def self.query_time
- SafeRequestStore[:rugged_query_time] ||= 0
+ query_time = SafeRequestStore[:rugged_query_time] ||= 0
+ query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
end
def self.query_time=(duration)
diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb
index bf579dd3b77..abf6ee07d53 100644
--- a/lib/gitlab/runtime.rb
+++ b/lib/gitlab/runtime.rb
@@ -37,7 +37,7 @@ module Gitlab
end
def puma?
- !!defined?(::Puma)
+ !!defined?(::Puma) && !defined?(ACTION_CABLE_SERVER)
end
# For unicorn, we need to check for actual server instances to avoid false positives.
@@ -70,25 +70,31 @@ module Gitlab
end
def web_server?
- puma? || unicorn?
+ puma? || unicorn? || action_cable?
+ end
+
+ def action_cable?
+ !!defined?(ACTION_CABLE_SERVER)
end
def multi_threaded?
- puma? || sidekiq?
+ puma? || sidekiq? || action_cable?
end
def max_threads
main_thread = 1
- if puma?
- Puma.cli_config.options[:max_threads] + main_thread
+ if action_cable?
+ Gitlab::Application.config.action_cable.worker_pool_size
+ elsif puma?
+ Puma.cli_config.options[:max_threads]
elsif sidekiq?
# An extra thread for the poller in Sidekiq Cron:
# https://github.com/ondrejbartas/sidekiq-cron#under-the-hood
- Sidekiq.options[:concurrency] + main_thread + 1
+ Sidekiq.options[:concurrency] + 1
else
- main_thread
- end
+ 0
+ end + main_thread
end
end
end
diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb
index 5eeb8b00ff3..7e22bf4d7df 100644
--- a/lib/gitlab/sanitizers/exif.rb
+++ b/lib/gitlab/sanitizers/exif.rb
@@ -152,7 +152,7 @@ module Gitlab
raise "failed to get exif tags: #{output}" if status != 0
- JSON.parse(output).first
+ Gitlab::Json.parse(output).first
end
end
end
diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb
index f3136fff294..1f6e0519b4c 100644
--- a/lib/gitlab/search/parsed_query.rb
+++ b/lib/gitlab/search/parsed_query.rb
@@ -3,8 +3,6 @@
module Gitlab
module Search
class ParsedQuery
- prepend_if_ee('EE::Gitlab::Search::ParsedQuery') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
attr_reader :term, :filters
def initialize(term, filters)
@@ -25,3 +23,5 @@ module Gitlab
end
end
end
+
+Gitlab::Search::ParsedQuery.prepend_if_ee('EE::Gitlab::Search::ParsedQuery')
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 0473fa89a0d..c35ee62163a 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -4,8 +4,10 @@ module Gitlab
class SearchResults
COUNT_LIMIT = 100
COUNT_LIMIT_MESSAGE = "#{COUNT_LIMIT - 1}+"
+ DEFAULT_PAGE = 1
+ DEFAULT_PER_PAGE = 20
- attr_reader :current_user, :query, :per_page
+ attr_reader :current_user, :query
# Limit search results by passed projects
# It allows us to search only for projects user has access to
@@ -17,15 +19,14 @@ module Gitlab
# query
attr_reader :default_project_filter
- def initialize(current_user, limit_projects, query, default_project_filter: false, per_page: 20)
+ def initialize(current_user, limit_projects, query, default_project_filter: false)
@current_user = current_user
@limit_projects = limit_projects || Project.all
@query = query
@default_project_filter = default_project_filter
- @per_page = per_page
end
- def objects(scope, page = nil, without_count = true)
+ def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true)
collection = case scope
when 'projects'
projects
@@ -39,7 +40,9 @@ module Gitlab
users
else
Kaminari.paginate_array([])
- end.page(page).per(per_page)
+ end
+
+ collection = collection.page(page).per(per_page)
without_count ? collection.without_count : collection
end
diff --git a/lib/gitlab/services/logger.rb b/lib/gitlab/services/logger.rb
new file mode 100644
index 00000000000..4e7ef73922c
--- /dev/null
+++ b/lib/gitlab/services/logger.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Services
+ class Logger < ::Gitlab::JsonLogger
+ def self.file_name_noext
+ 'service_measurement'
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 196dc0e3447..3f36725cb66 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -96,7 +96,8 @@ module Gitlab
class << self
def configuration_toml(gitaly_dir, storage_paths)
nodes = [{ storage: 'default', address: "unix:#{gitaly_dir}/gitaly.socket", primary: true, token: 'secret' }]
- config = { socket_path: "#{gitaly_dir}/praefect.socket", virtual_storage_name: 'default', token: 'secret', node: nodes }
+ storages = [{ name: 'default', node: nodes }]
+ config = { socket_path: "#{gitaly_dir}/praefect.socket", virtual_storage: storages }
config[:token] = 'secret' if Rails.env.test?
TomlRB.dump(config)
diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb
index c49432f0fc6..0d0efe8ffbd 100644
--- a/lib/gitlab/sidekiq_config/cli_methods.rb
+++ b/lib/gitlab/sidekiq_config/cli_methods.rb
@@ -48,7 +48,6 @@ module Gitlab
# rubocop:enable Gitlab/ModuleWithInstanceVariables
def worker_queues(rails_path = Rails.root.to_s)
- # https://gitlab.com/gitlab-org/gitlab/issues/199230
worker_names(all_queues(rails_path))
end
@@ -75,7 +74,7 @@ module Gitlab
private
def worker_names(workers)
- workers.map { |queue| queue.is_a?(Hash) ? queue[:name] : queue }
+ workers.map { |queue| queue[:name] }
end
def query_string_to_lambda(query_string)
diff --git a/lib/gitlab/sidekiq_daemon/monitor.rb b/lib/gitlab/sidekiq_daemon/monitor.rb
index 0723b514c90..1f1d63877b5 100644
--- a/lib/gitlab/sidekiq_daemon/monitor.rb
+++ b/lib/gitlab/sidekiq_daemon/monitor.rb
@@ -134,7 +134,7 @@ module Gitlab
end
def safe_parse(message)
- JSON.parse(message)
+ Gitlab::Json.parse(message)
rescue JSON::ParserError
end
diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb
index 45c6842c59b..64782e1e1d1 100644
--- a/lib/gitlab/sidekiq_logging/json_formatter.rb
+++ b/lib/gitlab/sidekiq_logging/json_formatter.rb
@@ -19,6 +19,7 @@ module Gitlab
output[:message] = data
when Hash
convert_to_iso8601!(data)
+ convert_retry_to_integer!(data)
stringify_args!(data)
output.merge!(data)
end
@@ -41,6 +42,20 @@ module Gitlab
Time.at(timestamp).utc.iso8601(3)
end
+ def convert_retry_to_integer!(payload)
+ payload['retry'] =
+ case payload['retry']
+ when Integer
+ payload['retry']
+ when false, nil
+ 0
+ when true
+ Sidekiq::JobRetry::DEFAULT_MAX_RETRY_ATTEMPTS
+ else
+ -1
+ end
+ end
+
def stringify_args!(payload)
payload['args'] = Gitlab::Utils::LogLimitedArray.log_limited_array(payload['args'].map(&:to_s)) if payload['args']
end
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index ea60190353e..4e39120f8a7 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -30,6 +30,12 @@ module Gitlab
output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS))
end
+ def add_logging_extras!(job, output_payload)
+ output_payload.merge!(
+ job.select { |key, _| key.to_s.start_with?("#{ApplicationWorker::LOGGING_EXTRA_KEY}.") }
+ )
+ end
+
def log_job_start(payload)
payload['message'] = "#{base_message(payload)}: start"
payload['job_status'] = 'start'
@@ -43,6 +49,7 @@ module Gitlab
def log_job_done(job, started_time, payload, job_exception = nil)
payload = payload.dup
add_instrumentation_keys!(job, payload)
+ add_logging_extras!(job, payload)
elapsed_time = elapsed(started_time)
add_time_keys!(elapsed_time, payload)
@@ -66,11 +73,11 @@ module Gitlab
end
def add_time_keys!(time, payload)
- payload['duration_s'] = time[:duration].round(2)
+ payload['duration_s'] = time[:duration].round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
# ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0)
# supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime
- payload['cpu_s'] = time[:cputime].round(2) if time[:cputime] > 0
+ payload['cpu_s'] = time[:cputime].round(Gitlab::InstrumentationHelper::DURATION_PRECISION) if time[:cputime] > 0
payload['completed_at'] = Time.now.utc.to_f
end
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index 1b155570f18..4eef3fbd12e 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -7,13 +7,14 @@ module Gitlab
# The result of this method should be passed to
# Sidekiq's `config.server_middleware` method
# eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)`
- def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true, request_store: true)
+ def self.server_configurator(metrics: true, arguments_logger: true, memory_killer: true)
lambda do |chain|
chain.add ::Gitlab::SidekiqMiddleware::Monitor
chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics if metrics
chain.add ::Gitlab::SidekiqMiddleware::ArgumentsLogger if arguments_logger
chain.add ::Gitlab::SidekiqMiddleware::MemoryKiller if memory_killer
- chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware if request_store
+ chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware
+ chain.add ::Gitlab::SidekiqMiddleware::ExtraDoneLogMetadata
chain.add ::Gitlab::SidekiqMiddleware::BatchLoader
chain.add ::Labkit::Middleware::Sidekiq::Server
chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger
diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb
index e227ee654ee..36204e1bee0 100644
--- a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb
+++ b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb
@@ -8,7 +8,7 @@ module Gitlab
# If enabled then it injects a job field that persists through the job execution
class Client
def call(_worker_class, job, _queue, _redis_pool)
- return yield unless Feature.enabled?(:user_mode_in_session)
+ return yield unless ::Feature.enabled?(:user_mode_in_session)
# Admin mode enabled in the original request or in a nested sidekiq job
admin_mode_user_id = find_admin_user_id
@@ -16,7 +16,7 @@ module Gitlab
if admin_mode_user_id
job['admin_mode_user_id'] ||= admin_mode_user_id
- Gitlab::AppLogger.debug("AdminMode::Client injected admin mode for job: #{job.inspect}")
+ ::Gitlab::AppLogger.debug("AdminMode::Client injected admin mode for job: #{job.inspect}")
end
yield
@@ -25,8 +25,8 @@ module Gitlab
private
def find_admin_user_id
- Gitlab::Auth::CurrentUserMode.current_admin&.id ||
- Gitlab::Auth::CurrentUserMode.bypass_session_admin_id
+ ::Gitlab::Auth::CurrentUserMode.current_admin&.id ||
+ ::Gitlab::Auth::CurrentUserMode.bypass_session_admin_id
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb
index 2859aa5f4a6..fe5213fc5d7 100644
--- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb
+++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb
@@ -4,7 +4,7 @@ module Gitlab
module SidekiqMiddleware
class ArgumentsLogger
def call(worker, job, queue)
- Sidekiq.logger.info "arguments: #{JSON.dump(job['args'])}"
+ Sidekiq.logger.info "arguments: #{Gitlab::Json.dump(job['args'])}"
yield
end
end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index 79bbb99752e..fa742d07af2 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -67,7 +67,7 @@ module Gitlab
end
def droppable?
- idempotent? && duplicate?
+ idempotent? && duplicate? && ::Feature.disabled?("disable_#{queue_name}_deduplication")
end
private
diff --git a/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb b/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb
new file mode 100644
index 00000000000..93c3131d50e
--- /dev/null
+++ b/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SidekiqMiddleware
+ class ExtraDoneLogMetadata
+ def call(worker, job, queue)
+ yield
+
+ # We needed a way to pass state from a worker in to the
+ # Gitlab::SidekiqLogging::StructuredLogger . Unfortunately the
+ # StructuredLogger itself is not a middleware so cannot access the
+ # worker object. We also tried to use SafeRequestStore but to pass the
+ # data up but that doesn't work either because this is reset in
+ # Gitlab::SidekiqMiddleware::RequestStoreMiddleware inside yield for
+ # the StructuredLogger so it's cleared before we get to logging the
+ # done statement. As such the only way to do this is to pass the data
+ # up in the `job` object. Since `job` is just a Hash we can add this
+ # extra metadata there.
+ if worker.respond_to?(:logging_extras)
+ job.merge!(worker.logging_extras)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/slash_commands/presenters/issue_base.rb b/lib/gitlab/slash_commands/presenters/issue_base.rb
index 4bc05d1f318..017fb8a62c4 100644
--- a/lib/gitlab/slash_commands/presenters/issue_base.rb
+++ b/lib/gitlab/slash_commands/presenters/issue_base.rb
@@ -4,8 +4,6 @@ module Gitlab
module SlashCommands
module Presenters
module IssueBase
- prepend_if_ee('EE::Gitlab::SlashCommands::Presenters::IssueBase') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
def color(issuable)
issuable.open? ? '#38ae67' : '#d22852'
end
@@ -51,3 +49,5 @@ module Gitlab
end
end
end
+
+Gitlab::SlashCommands::Presenters::IssueBase.prepend_if_ee('EE::Gitlab::SlashCommands::Presenters::IssueBase')
diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb
index e955ccd35da..9911f9e62a6 100644
--- a/lib/gitlab/snippet_search_results.rb
+++ b/lib/gitlab/snippet_search_results.rb
@@ -11,36 +11,18 @@ module Gitlab
@query = query
end
- def objects(scope, page = nil)
- case scope
- when 'snippet_titles'
- paginated_objects(snippet_titles, page)
- when 'snippet_blobs'
- paginated_objects(snippet_blobs, page)
- else
- super(scope, nil, false)
- end
+ def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE)
+ paginated_objects(snippet_titles, page, per_page)
end
def formatted_count(scope)
- case scope
- when 'snippet_titles'
- formatted_limited_count(limited_snippet_titles_count)
- when 'snippet_blobs'
- formatted_limited_count(limited_snippet_blobs_count)
- else
- super
- end
+ formatted_limited_count(limited_snippet_titles_count)
end
def limited_snippet_titles_count
@limited_snippet_titles_count ||= limited_count(snippet_titles)
end
- def limited_snippet_blobs_count
- @limited_snippet_blobs_count ||= limited_count(snippet_blobs)
- end
-
private
# rubocop: disable CodeReuse/ActiveRecord
@@ -56,15 +38,7 @@ module Gitlab
snippets.search(query)
end
- def snippet_blobs
- snippets.search_code(query)
- end
-
- def default_scope
- 'snippet_blobs'
- end
-
- def paginated_objects(relation, page)
+ def paginated_objects(relation, page, per_page)
relation.page(page).per(per_page)
end
diff --git a/lib/gitlab/static_site_editor/config.rb b/lib/gitlab/static_site_editor/config.rb
index 41d54ee0a92..c931cdecbeb 100644
--- a/lib/gitlab/static_site_editor/config.rb
+++ b/lib/gitlab/static_site_editor/config.rb
@@ -22,7 +22,8 @@ module Gitlab
project: project.path,
namespace: project.namespace.path,
return_url: return_url,
- is_supported_content: supported_content?
+ is_supported_content: supported_content?.to_s,
+ base_url: Gitlab::Routing.url_helpers.project_show_sse_path(project, full_path)
}
end
@@ -47,6 +48,10 @@ module Gitlab
def file_exists?
commit_id.present? && repository.blob_at(commit_id, file_path).present?
end
+
+ def full_path
+ "#{ref}/#{file_path}"
+ end
end
end
end
diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb
index ac02ec635e4..6ccb442b1e0 100644
--- a/lib/gitlab/task_helpers.rb
+++ b/lib/gitlab/task_helpers.rb
@@ -157,8 +157,8 @@ module Gitlab
Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home
end
- def checkout_or_clone_version(version:, repo:, target_dir:)
- clone_repo(repo, target_dir) unless Dir.exist?(target_dir)
+ def checkout_or_clone_version(version:, repo:, target_dir:, clone_opts: [])
+ clone_repo(repo, target_dir, clone_opts: clone_opts) unless Dir.exist?(target_dir)
checkout_version(get_version(version), target_dir)
end
@@ -171,8 +171,8 @@ module Gitlab
"v#{component_version}"
end
- def clone_repo(repo, target_dir)
- run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}])
+ def clone_repo(repo, target_dir, clone_opts: [])
+ run_command!(%W[#{Gitlab.config.git.bin_path} clone] + clone_opts + %W[-- #{repo} #{target_dir}])
end
def checkout_version(version, target_dir)
diff --git a/lib/gitlab/testing/clear_thread_memory_cache_middleware.rb b/lib/gitlab/testing/clear_process_memory_cache_middleware.rb
index 6f54038ae22..1e69e5e142d 100644
--- a/lib/gitlab/testing/clear_thread_memory_cache_middleware.rb
+++ b/lib/gitlab/testing/clear_process_memory_cache_middleware.rb
@@ -2,13 +2,13 @@
module Gitlab
module Testing
- class ClearThreadMemoryCacheMiddleware
+ class ClearProcessMemoryCacheMiddleware
def initialize(app)
@app = app
end
def call(env)
- Gitlab::ThreadMemoryCache.cache_backend.clear
+ Gitlab::ProcessMemoryCache.cache_backend.clear
@app.call(env)
end
diff --git a/lib/gitlab/thread_memory_cache.rb b/lib/gitlab/thread_memory_cache.rb
deleted file mode 100644
index 7f363dc7feb..00000000000
--- a/lib/gitlab/thread_memory_cache.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- class ThreadMemoryCache
- THREAD_KEY = :thread_memory_cache
-
- def self.cache_backend
- # Note ActiveSupport::Cache::MemoryStore is thread-safe. Since
- # each backend is local per thread we probably don't need to worry
- # about synchronizing access, but this is a drop-in replacement
- # for ActiveSupport::Cache::RedisStore.
- Thread.current[THREAD_KEY] ||= ActiveSupport::Cache::MemoryStore.new
- end
- end
-end
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index 9a7b4cc65f6..37688d6e0e7 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -9,13 +9,6 @@ module Gitlab
module ControllerConcern
extend ActiveSupport::Concern
- included do
- # Tracking events from the template is not ideal and we are moving this to the client in https://gitlab.com/gitlab-org/gitlab/-/issues/213712
- # In the meantime, using this method from the view is frowned upon and this line will likely be removed
- # in the near future
- helper_method :track_event
- end
-
protected
def track_event(action = action_name, **args)
diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb
index 5df53b5adde..4ec43e62c19 100644
--- a/lib/gitlab/tree_summary.rb
+++ b/lib/gitlab/tree_summary.rb
@@ -2,21 +2,24 @@
module Gitlab
class TreeSummary
- prepend_if_ee('::EE::Gitlab::TreeSummary') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
include ::Gitlab::Utils::StrongMemoize
+ include ::MarkupHelper
+
+ CACHE_EXPIRE_IN = 1.hour
+ MAX_OFFSET = 2**31
- attr_reader :commit, :project, :path, :offset, :limit
+ attr_reader :commit, :project, :path, :offset, :limit, :user
attr_reader :resolved_commits
private :resolved_commits
- def initialize(commit, project, params = {})
+ def initialize(commit, project, user, params = {})
@commit = commit
@project = project
+ @user = user
@path = params.fetch(:path, nil).presence
- @offset = params.fetch(:offset, 0).to_i
+ @offset = [params.fetch(:offset, 0).to_i, MAX_OFFSET].min
@limit = (params.fetch(:limit, 25) || 25).to_i
# Ensure that if multiple tree entries share the same last commit, they share
@@ -43,6 +46,17 @@ module Gitlab
[summary, commits]
end
+ def fetch_logs
+ cache_key = ['projects', project.id, 'logs', commit.id, path, offset]
+ Rails.cache.fetch(cache_key, expires_in: CACHE_EXPIRE_IN) do
+ logs, _ = summarize
+
+ new_offset = next_offset if more?
+
+ [logs.as_json, new_offset]
+ end
+ end
+
# Does the tree contain more entries after the given offset + limit?
def more?
all_contents[next_offset].present?
@@ -84,6 +98,7 @@ module Gitlab
end
commits_hsh = repository.list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit)
+ prerender_commit_full_titles!(commits_hsh.values)
entries.each do |entry|
path_key = entry_path(entry)
@@ -92,6 +107,7 @@ module Gitlab
if commit
entry[:commit] = commit
entry[:commit_path] = commit_path(commit)
+ entry[:commit_title_html] = markdown_field(commit, :full_title)
end
end
end
@@ -119,5 +135,15 @@ module Gitlab
def tree
strong_memoize(:tree) { repository.tree(commit.id, path) }
end
+
+ def prerender_commit_full_titles!(commits)
+ # Preload commit authors as they are used in rendering
+ commits.each(&:lazy_author)
+
+ renderer = Banzai::ObjectRenderer.new(user: user, default_project: project)
+ renderer.render(commits, :full_title)
+ end
end
end
+
+Gitlab::TreeSummary.prepend_if_ee('::EE::Gitlab::TreeSummary')
diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb
index 96ee6f0e8e6..9377ccfec1e 100644
--- a/lib/gitlab/uploads/migration_helper.rb
+++ b/lib/gitlab/uploads/migration_helper.rb
@@ -15,6 +15,7 @@ module Gitlab
%w(FileUploader Project),
%w(PersonalFileUploader Snippet),
%w(NamespaceFileUploader Snippet),
+ %w(DesignManagement::DesignV432x230Uploader DesignManagement::Action :image_v432x230),
%w(FileUploader MergeRequest)].freeze
def initialize(args, logger)
@@ -74,5 +75,3 @@ module Gitlab
end
end
end
-
-Gitlab::Uploads::MigrationHelper.prepend_if_ee('EE::Gitlab::Uploads::MigrationHelper')
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 4c56b9bb3c9..329f87d8be8 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -11,6 +11,10 @@ module Gitlab
class << self
include ActionView::RecordIdentifier
+ # Using a case statement here is preferable for readability and maintainability.
+ # See discussion in https://gitlab.com/gitlab-org/gitlab/-/issues/217397
+ #
+ # rubocop:disable Metrics/CyclomaticComplexity
def build(object, **options)
# Objects are sometimes wrapped in a BatchLoader instance
case object.itself
@@ -34,14 +38,17 @@ module Gitlab
snippet_url(object, **options)
when User
instance.user_url(object, **options)
- when ProjectWiki
- instance.project_wiki_url(object.project, :home, **options)
+ when Wiki
+ wiki_url(object, **options)
when WikiPage
instance.project_wiki_url(object.wiki.project, object.slug, **options)
+ when ::DesignManagement::Design
+ design_url(object, **options)
else
raise NotImplementedError.new("No URL builder defined for #{object.inspect}")
end
end
+ # rubocop:enable Metrics/CyclomaticComplexity
def commit_url(commit, **options)
return '' unless commit.project
@@ -70,6 +77,25 @@ module Gitlab
instance.gitlab_snippet_url(snippet, **options)
end
end
+
+ def wiki_url(object, **options)
+ if object.container.is_a?(Project)
+ instance.project_wiki_url(object.container, Wiki::HOMEPAGE, **options)
+ else
+ raise NotImplementedError.new("No URL builder defined for #{object.inspect}")
+ end
+ end
+
+ def design_url(design, **options)
+ size, ref = options.values_at(:size, :ref)
+ options.except!(:size, :ref)
+
+ if size
+ instance.project_design_management_designs_resized_image_url(design.project, design, ref, size, **options)
+ else
+ instance.project_design_management_designs_raw_image_url(design.project, design, ref, **options)
+ end
+ end
end
end
end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index f8ee0ca6877..e60c786b52c 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -1,15 +1,25 @@
# frozen_string_literal: true
-# For hardening usage ping and make it easier to add measures there is in place alt_usage_data method
-# which handles StandardError and fallbacks into -1
-# this way not all measures fail if we encounter one exception
+# For hardening usage ping and make it easier to add measures there is in place
+# * alt_usage_data method
+# handles StandardError and fallbacks into -1 this way not all measures fail if we encounter one exception
#
-# Examples:
-# alt_usage_data { Gitlab::VERSION }
-# alt_usage_data { Gitlab::CurrentSettings.uuid }
+# Examples:
+# alt_usage_data { Gitlab::VERSION }
+# alt_usage_data { Gitlab::CurrentSettings.uuid }
+#
+# * redis_usage_data method
+# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
+# returns -1 when a block is sent or hash with all values -1 when a counter is sent
+# different behaviour due to 2 different implementations of redis counter
+#
+# Examples:
+# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
+# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
module Gitlab
class UsageData
BATCH_SIZE = 100
+ FALLBACK = -1
class << self
def data(force_refresh: false)
@@ -24,6 +34,8 @@ module Gitlab
.merge(features_usage_data)
.merge(components_usage_data)
.merge(cycle_analytics_usage_data)
+ .merge(object_store_usage_data)
+ .merge(recording_ce_finish_data)
end
def to_json(force_refresh: false)
@@ -32,19 +44,27 @@ module Gitlab
def license_usage_data
{
+ recorded_at: Time.now, # should be calculated very first
uuid: alt_usage_data { Gitlab::CurrentSettings.uuid },
hostname: alt_usage_data { Gitlab.config.gitlab.host },
version: alt_usage_data { Gitlab::VERSION },
installation_type: alt_usage_data { installation_type },
active_user_count: count(User.active),
- recorded_at: Time.now,
edition: 'CE'
}
end
+ def recording_ce_finish_data
+ {
+ recording_ce_finished_at: Time.now
+ }
+ end
+
# rubocop: disable Metrics/AbcSize
# rubocop: disable CodeReuse/ActiveRecord
def system_usage_data
+ alert_bot_incident_count = count(::Issue.authored(::User.alert_bot))
+
{
counts: {
assignee_lists: count(List.assignee),
@@ -94,7 +114,10 @@ module Gitlab
issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data,
- incident_issues: count(::Issue.authored(::User.alert_bot)),
+ issues_created_gitlab_alerts: count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot)),
+ incident_issues: alert_bot_incident_count,
+ alert_bot_incident_issues: alert_bot_incident_count,
+ incident_labeled_issues: count(::Issue.with_label_attributes(IncidentManagement::CreateIssueService::INCIDENT_LABEL)),
keys: count(Key),
label_lists: count(List.label),
lfs_objects: count(LfsObject),
@@ -123,7 +146,8 @@ module Gitlab
services_usage,
usage_counters,
user_preferences_usage,
- ingress_modsecurity_usage
+ ingress_modsecurity_usage,
+ container_expiration_policies_usage
)
}
end
@@ -154,7 +178,6 @@ module Gitlab
dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled,
gitlab_shared_runners_enabled: alt_usage_data { Gitlab.config.gitlab_ci.shared_runners_enabled },
gravatar_enabled: alt_usage_data { Gitlab::CurrentSettings.gravatar_enabled? },
- influxdb_metrics_enabled: alt_usage_data { Gitlab::Metrics.influx_metrics_enabled? },
ldap_enabled: alt_usage_data { Gitlab.config.ldap.enabled },
mattermost_enabled: alt_usage_data { Gitlab.config.mattermost.enabled },
omniauth_enabled: alt_usage_data { Gitlab::Auth.omniauth_enabled? },
@@ -162,36 +185,14 @@ module Gitlab
reply_by_email_enabled: alt_usage_data { Gitlab::IncomingEmail.enabled? },
signup_enabled: alt_usage_data { Gitlab::CurrentSettings.allow_signup? },
web_ide_clientside_preview_enabled: alt_usage_data { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? },
- ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity)
- }.merge(features_usage_data_container_expiration_policies)
- end
-
- # rubocop: disable CodeReuse/ActiveRecord
- def features_usage_data_container_expiration_policies
- results = {}
- start = ::Project.minimum(:id)
- finish = ::Project.maximum(:id)
-
- results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish)
- base = ::ContainerExpirationPolicy.active
- results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish)
-
- %i[keep_n cadence older_than].each do |option|
- ::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend
- results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish)
- end
- end
-
- results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish)
- results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish)
-
- results
+ ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity),
+ grafana_link_enabled: alt_usage_data { Gitlab::CurrentSettings.grafana_enabled? }
+ }
end
- # rubocop: enable CodeReuse/ActiveRecord
# @return [Hash<Symbol, Integer>]
def usage_counters
- usage_data_counters.map(&:totals).reduce({}) { |a, b| a.merge(b) }
+ usage_data_counters.map { |counter| redis_usage_data(counter) }.reduce({}, :merge)
end
# @return [Array<#totals>] An array of objects that respond to `#totals`
@@ -205,7 +206,8 @@ module Gitlab
Gitlab::UsageDataCounters::CycleAnalyticsCounter,
Gitlab::UsageDataCounters::ProductivityAnalyticsCounter,
Gitlab::UsageDataCounters::SourceCodeCounter,
- Gitlab::UsageDataCounters::MergeRequestCounter
+ Gitlab::UsageDataCounters::MergeRequestCounter,
+ Gitlab::UsageDataCounters::DesignsCounter
]
end
@@ -237,9 +239,83 @@ module Gitlab
'unknown_app_server_type'
end
+ def object_store_config(component)
+ config = alt_usage_data(fallback: nil) do
+ Settings[component]['object_store']
+ end
+
+ if config
+ {
+ enabled: alt_usage_data { Settings[component]['enabled'] },
+ object_store: {
+ enabled: alt_usage_data { config['enabled'] },
+ direct_upload: alt_usage_data { config['direct_upload'] },
+ background_upload: alt_usage_data { config['background_upload'] },
+ provider: alt_usage_data { config['connection']['provider'] }
+ }
+ }
+ else
+ {
+ enabled: alt_usage_data { Settings[component]['enabled'] }
+ }
+ end
+ end
+
+ def object_store_usage_data
+ {
+ object_store: {
+ artifacts: object_store_config('artifacts'),
+ external_diffs: object_store_config('external_diffs'),
+ lfs: object_store_config('lfs'),
+ uploads: object_store_config('uploads'),
+ packages: object_store_config('packages')
+ }
+ }
+ end
+
def ingress_modsecurity_usage
- ::Clusters::Applications::IngressModsecurityUsageService.new.execute
+ ##
+ # This method measures usage of the Modsecurity Web Application Firewall across the entire
+ # instance's deployed environments.
+ #
+ # NOTE: this service is an approximation as it does not yet take into account if environment
+ # is enabled and only measures applications installed using GitLab Managed Apps (disregards
+ # CI-based managed apps).
+ #
+ # More details: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28331#note_318621786
+ ##
+
+ column = ::Deployment.arel_table[:environment_id]
+ {
+ ingress_modsecurity_logging: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.logging), column),
+ ingress_modsecurity_blocking: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_enabled.blocking), column),
+ ingress_modsecurity_disabled: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_disabled), column),
+ ingress_modsecurity_not_installed: distinct_count(successful_deployments_with_cluster(::Clusters::Applications::Ingress.modsecurity_not_installed), column)
+ }
+ end
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def container_expiration_policies_usage
+ results = {}
+ start = ::Project.minimum(:id)
+ finish = ::Project.maximum(:id)
+
+ results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish)
+ base = ::ContainerExpirationPolicy.active
+ results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish)
+
+ %i[keep_n cadence older_than].each do |option|
+ ::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend
+ results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish)
+ end
+ end
+
+ results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish)
+ results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish)
+
+ results
end
+ # rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def services_usage
@@ -251,7 +327,7 @@ module Gitlab
results[:projects_slack_notifications_active] = results[:projects_slack_active]
results[:projects_slack_slash_active] = results[:projects_slack_slash_commands_active]
- results.merge(jira_usage)
+ results.merge(jira_usage).merge(jira_import_usage)
end
def jira_usage
@@ -281,35 +357,52 @@ module Gitlab
results
rescue ActiveRecord::StatementInvalid
- { projects_jira_server_active: -1, projects_jira_cloud_active: -1, projects_jira_active: -1 }
+ { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK, projects_jira_active: FALLBACK }
+ end
+
+ def successful_deployments_with_cluster(scope)
+ scope
+ .joins(cluster: :deployments)
+ .merge(Clusters::Cluster.enabled)
+ .merge(Deployment.success)
end
# rubocop: enable CodeReuse/ActiveRecord
+ def jira_import_usage
+ finished_jira_imports = JiraImportState.finished
+
+ {
+ jira_imports_total_imported_count: count(finished_jira_imports),
+ jira_imports_projects_count: distinct_count(finished_jira_imports, :project_id),
+ jira_imports_total_imported_issues_count: alt_usage_data { JiraImportState.finished_imports_count }
+ }
+ end
+
def user_preferences_usage
{} # augmented in EE
end
- def count(relation, column = nil, fallback: -1, batch: true, start: nil, finish: nil)
+ def count(relation, column = nil, batch: true, start: nil, finish: nil)
if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish)
else
relation.count
end
rescue ActiveRecord::StatementInvalid
- fallback
+ FALLBACK
end
- def distinct_count(relation, column = nil, fallback: -1, batch: true, start: nil, finish: nil)
+ def distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)
if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
Gitlab::Database::BatchCount.batch_distinct_count(relation, column, start: start, finish: finish)
else
relation.distinct_count_by(column)
end
rescue ActiveRecord::StatementInvalid
- fallback
+ FALLBACK
end
- def alt_usage_data(value = nil, fallback: -1, &block)
+ def alt_usage_data(value = nil, fallback: FALLBACK, &block)
if block_given?
yield
else
@@ -319,8 +412,28 @@ module Gitlab
fallback
end
+ def redis_usage_data(counter = nil, &block)
+ if block_given?
+ redis_usage_counter(&block)
+ elsif counter.present?
+ redis_usage_data_totals(counter)
+ end
+ end
+
private
+ def redis_usage_counter
+ yield
+ rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
+ FALLBACK
+ end
+
+ def redis_usage_data_totals(counter)
+ counter.totals
+ rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
+ counter.fallback_totals
+ end
+
def installation_type
if Rails.env.production?
Gitlab::INSTALLATION_TYPE
diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb
index 33111b46381..96898e5189c 100644
--- a/lib/gitlab/usage_data_counters/base_counter.rb
+++ b/lib/gitlab/usage_data_counters/base_counter.rb
@@ -22,11 +22,19 @@ module Gitlab::UsageDataCounters
end
def totals
- known_events.map { |e| ["#{prefix}_#{e}".to_sym, read(e)] }.to_h
+ known_events.map { |event| [counter_key(event), read(event)] }.to_h
+ end
+
+ def fallback_totals
+ known_events.map { |event| [counter_key(event), -1] }.to_h
end
private
+ def counter_key(event)
+ "#{prefix}_#{event}".to_sym
+ end
+
def known_events
self::KNOWN_EVENTS
end
diff --git a/lib/gitlab/usage_data_counters/designs_counter.rb b/lib/gitlab/usage_data_counters/designs_counter.rb
new file mode 100644
index 00000000000..801fb8f3b3d
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/designs_counter.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab::UsageDataCounters
+ class DesignsCounter
+ extend Gitlab::UsageDataCounters::RedisCounter
+
+ KNOWN_EVENTS = %w[create update delete].map(&:freeze).freeze
+
+ UnknownEvent = Class.new(StandardError)
+
+ class << self
+ # Each event gets a unique Redis key
+ def redis_key(event)
+ raise UnknownEvent, event unless KNOWN_EVENTS.include?(event.to_s)
+
+ "USAGE_DESIGN_MANAGEMENT_DESIGNS_#{event}".upcase
+ end
+
+ def count(event)
+ increment(redis_key(event))
+ end
+
+ def read(event)
+ total_count(redis_key(event))
+ end
+
+ def totals
+ KNOWN_EVENTS.map { |event| [counter_key(event), read(event)] }.to_h
+ end
+
+ def fallback_totals
+ KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h
+ end
+
+ private
+
+ def counter_key(event)
+ "design_management_designs_#{event}".to_sym
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb
index 672450ec82b..7a76180cb08 100644
--- a/lib/gitlab/usage_data_counters/note_counter.rb
+++ b/lib/gitlab/usage_data_counters/note_counter.rb
@@ -25,12 +25,20 @@ module Gitlab::UsageDataCounters
def totals
COUNTABLE_TYPES.map do |countable_type|
- [:"#{countable_type.underscore}_comment", read(:create, countable_type)]
+ [counter_key(countable_type), read(:create, countable_type)]
end.to_h
end
+ def fallback_totals
+ COUNTABLE_TYPES.map { |counter_key| [counter_key(counter_key), -1] }.to_h
+ end
+
private
+ def counter_key(countable_type)
+ "#{countable_type.underscore}_comment".to_sym
+ end
+
def countable?(noteable_type)
COUNTABLE_TYPES.include?(noteable_type.to_s)
end
diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb
index 5f0735347e1..b9e3a5c0104 100644
--- a/lib/gitlab/usage_data_counters/search_counter.rb
+++ b/lib/gitlab/usage_data_counters/search_counter.rb
@@ -21,6 +21,10 @@ module Gitlab
navbar_searches: total_navbar_searches_count
}
end
+
+ def fallback_totals
+ { navbar_searches: -1 }
+ end
end
end
end
diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb
index c012a6c96df..00fcd42a9af 100644
--- a/lib/gitlab/usage_data_counters/web_ide_counter.rb
+++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb
@@ -4,54 +4,52 @@ module Gitlab
module UsageDataCounters
class WebIdeCounter
extend RedisCounter
-
- COMMITS_COUNT_KEY = 'WEB_IDE_COMMITS_COUNT'
- MERGE_REQUEST_COUNT_KEY = 'WEB_IDE_MERGE_REQUESTS_COUNT'
- VIEWS_COUNT_KEY = 'WEB_IDE_VIEWS_COUNT'
- PREVIEW_COUNT_KEY = 'WEB_IDE_PREVIEWS_COUNT'
+ KNOWN_EVENTS = %i[commits views merge_requests previews terminals pipelines].freeze
+ PREFIX = 'web_ide'
class << self
def increment_commits_count
- increment(COMMITS_COUNT_KEY)
- end
-
- def total_commits_count
- total_count(COMMITS_COUNT_KEY)
+ increment(redis_key('commits'))
end
def increment_merge_requests_count
- increment(MERGE_REQUEST_COUNT_KEY)
+ increment(redis_key('merge_requests'))
end
- def total_merge_requests_count
- total_count(MERGE_REQUEST_COUNT_KEY)
+ def increment_views_count
+ increment(redis_key('views'))
end
- def increment_views_count
- increment(VIEWS_COUNT_KEY)
+ def increment_terminals_count
+ increment(redis_key('terminals'))
end
- def total_views_count
- total_count(VIEWS_COUNT_KEY)
+ def increment_pipelines_count
+ increment(redis_key('pipelines'))
end
def increment_previews_count
return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled?
- increment(PREVIEW_COUNT_KEY)
+ increment(redis_key('previews'))
+ end
+
+ def totals
+ KNOWN_EVENTS.map { |event| [counter_key(event), total_count(redis_key(event))] }.to_h
end
- def total_previews_count
- total_count(PREVIEW_COUNT_KEY)
+ def fallback_totals
+ KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h
end
- def totals
- {
- web_ide_commits: total_commits_count,
- web_ide_views: total_views_count,
- web_ide_merge_requests: total_merge_requests_count,
- web_ide_previews: total_previews_count
- }
+ private
+
+ def redis_key(event)
+ "#{PREFIX}_#{event}_count".upcase
+ end
+
+ def counter_key(event)
+ "#{PREFIX}_#{event}".to_sym
end
end
end
diff --git a/lib/gitlab/user_access_snippet.rb b/lib/gitlab/user_access_snippet.rb
index bfed86c4df4..dcd45f9350d 100644
--- a/lib/gitlab/user_access_snippet.rb
+++ b/lib/gitlab/user_access_snippet.rb
@@ -17,7 +17,14 @@ module Gitlab
@project = snippet&.project
end
+ def allowed?
+ return true if snippet_migration?
+
+ super
+ end
+
def can_do_action?(action)
+ return true if snippet_migration?
return false unless can_access_git?
permission_cache[action] =
@@ -35,7 +42,10 @@ module Gitlab
end
def can_push_to_branch?(ref)
+ return true if snippet_migration?
+
super
+
return false unless snippet
return false unless can_do_action?(:update_snippet)
@@ -45,5 +55,9 @@ module Gitlab
def can_merge_to_branch?(ref)
false
end
+
+ def snippet_migration?
+ user&.migration_bot? && snippet
+ end
end
end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index 2e8a3ca4242..d46601fa2e8 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -75,12 +75,12 @@ module Gitlab
str.gsub(/\r?\n/, '')
end
- def to_boolean(value)
+ def to_boolean(value, default: nil)
return value if [true, false].include?(value)
return true if value =~ /^(true|t|yes|y|1|on)$/i
return false if value =~ /^(false|f|no|n|0|off)$/i
- nil
+ default
end
def boolean_to_yes_no(bool)
@@ -123,7 +123,7 @@ module Gitlab
end
def ms_to_round_sec(ms)
- (ms.to_f / 1000).round(2)
+ (ms.to_f / 1000).round(6)
end
# Used in EE
diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb
index 0680cefd249..febe489f1f8 100644
--- a/lib/gitlab/utils/measuring.rb
+++ b/lib/gitlab/utils/measuring.rb
@@ -5,38 +5,51 @@ require 'prometheus/pid_provider'
module Gitlab
module Utils
class Measuring
- def initialize(logger: Logger.new($stdout))
- @logger = logger
+ class << self
+ attr_writer :logger
+
+ def logger
+ @logger ||= Logger.new(STDOUT)
+ end
+ end
+
+ def initialize(base_log_data = {})
+ @base_log_data = base_log_data
end
def with_measuring
- logger.info "Measuring enabled..."
+ result = nil
with_gc_stats do
with_count_queries do
with_measure_time do
- yield
+ result = yield
end
end
end
- logger.info "Memory usage: #{Gitlab::Metrics::System.memory_usage.to_f / 1024 / 1024} MiB"
- logger.info "Label: #{::Prometheus::PidProvider.worker_id}"
+ log_info(
+ gc_stats: gc_stats,
+ time_to_finish: time_to_finish,
+ number_of_sql_calls: sql_calls_count,
+ memory_usage: "#{Gitlab::Metrics::System.memory_usage_rss.to_f / 1024 / 1024} MiB",
+ label: ::Prometheus::PidProvider.worker_id
+ )
+
+ result
end
private
- attr_reader :logger
+ attr_reader :gc_stats, :time_to_finish, :sql_calls_count, :base_log_data
def with_count_queries(&block)
- count = 0
+ @sql_calls_count = 0
counter_f = ->(_name, _started, _finished, _unique_id, payload) {
- count += 1 unless payload[:name].in? %w[CACHE SCHEMA]
+ @sql_calls_count += 1 unless payload[:name].in? %w[CACHE SCHEMA]
}
ActiveSupport::Notifications.subscribed(counter_f, "sql.active_record", &block)
-
- logger.info "Number of sql calls: #{count}"
end
def with_gc_stats
@@ -44,33 +57,22 @@ module Gitlab
stats_before = GC.stat
yield
stats_after = GC.stat
- stats_diff = stats_after.map do |key, after_value|
+ @gc_stats = stats_after.map do |key, after_value|
before_value = stats_before[key]
[key, before: before_value, after: after_value, diff: after_value - before_value]
end.to_h
- logger.info "GC stats:"
- logger.info JSON.pretty_generate(stats_diff)
end
def with_measure_time
- timing = Benchmark.realtime do
+ @time_to_finish = Benchmark.realtime do
yield
end
-
- logger.info "Time to finish: #{duration_in_numbers(timing)}"
end
- def duration_in_numbers(duration_in_seconds)
- milliseconds = duration_in_seconds.in_milliseconds % 1.second.in_milliseconds
- seconds = duration_in_seconds % 1.minute
- minutes = (duration_in_seconds / 1.minute) % (1.hour / 1.minute)
- hours = duration_in_seconds / 1.hour
-
- if hours == 0
- "%02d:%02d:%03d" % [minutes, seconds, milliseconds]
- else
- "%02d:%02d:%02d:%03d" % [hours, minutes, seconds, milliseconds]
- end
+ def log_info(details)
+ details = base_log_data.merge(details)
+ details = details.to_yaml if ActiveSupport::Logger.logger_outputs_to?(Measuring.logger, STDOUT)
+ Measuring.logger.info(details)
end
end
end
diff --git a/lib/gitlab/wiki_pages.rb b/lib/gitlab/wiki_pages.rb
index 47f9aa1117f..dee885e74d1 100644
--- a/lib/gitlab/wiki_pages.rb
+++ b/lib/gitlab/wiki_pages.rb
@@ -11,5 +11,8 @@ module Gitlab
# through the GitLab web interface and API:
MAX_TITLE_BYTES = 245 # reserving 10 bytes for the file extension
MAX_DIRECTORY_BYTES = 255
+
+ # Limit the number of pages displayed in the wiki sidebar.
+ MAX_SIDEBAR_PAGES = 15
end
end
diff --git a/lib/gitlab/with_request_store.rb b/lib/gitlab/with_request_store.rb
index d6c05e1e256..d13cd9a72f7 100644
--- a/lib/gitlab/with_request_store.rb
+++ b/lib/gitlab/with_request_store.rb
@@ -2,12 +2,24 @@
module Gitlab
module WithRequestStore
- def with_request_store
+ def with_request_store(&block)
+ # Skip enabling the request store if it was already active. Whatever
+ # instantiated the request store first is responsible for clearing it
+ return yield if RequestStore.active?
+
+ enabling_request_store(&block)
+ end
+
+ private
+
+ def enabling_request_store
RequestStore.begin!
yield
ensure
RequestStore.end!
RequestStore.clear!
end
+
+ extend self
end
end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index b375602a5fe..c91d1b05440 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -62,9 +62,6 @@ module Gitlab
end
def send_git_archive(repository, ref:, format:, append_sha:, path: nil)
- path_enabled = Feature.enabled?(:git_archive_path, default_enabled: true)
- path = nil unless path_enabled
-
format ||= 'tar.gz'
format = format.downcase
@@ -78,12 +75,7 @@ module Gitlab
raise "Repository or ref not found" if metadata.empty?
- params =
- if path_enabled
- send_git_archive_params(repository, metadata, path, archive_format(format))
- else
- metadata
- end
+ params = send_git_archive_params(repository, metadata, path, archive_format(format))
# If present, DisableCache must be a Boolean. Otherwise
# workhorse ignores it.
@@ -138,8 +130,7 @@ module Gitlab
]
end
- def send_artifacts_entry(build, entry)
- file = build.artifacts_file
+ def send_artifacts_entry(file, entry)
archive = file.file_storage? ? file.path : file.url
params = {
@@ -213,7 +204,7 @@ module Gitlab
# This is the outermost encoding of a senddata: header. It is safe for
# inclusion in HTTP response headers
def encode(hash)
- Base64.urlsafe_encode64(JSON.dump(hash))
+ Base64.urlsafe_encode64(Gitlab::Json.dump(hash))
end
# This is for encoding individual fields inside the senddata JSON that
diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb
index ed248e29211..7d4d4d9d13a 100644
--- a/lib/gitlab/x509/signature.rb
+++ b/lib/gitlab/x509/signature.rb
@@ -22,6 +22,10 @@ module Gitlab
X509Certificate.safe_create!(certificate_attributes) unless verified_signature.nil?
end
+ def user
+ User.find_by_any_email(@email)
+ end
+
def verified_signature
strong_memoize(:verified_signature) { verified_signature? }
end
diff --git a/lib/gitlab/x509/tag.rb b/lib/gitlab/x509/tag.rb
new file mode 100644
index 00000000000..48582c17764
--- /dev/null
+++ b/lib/gitlab/x509/tag.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+require 'openssl'
+require 'digest'
+
+module Gitlab
+ module X509
+ class Tag
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(raw_tag)
+ @raw_tag = raw_tag
+ end
+
+ def signature
+ signature = X509::Signature.new(signature_text, signed_text, @raw_tag.tagger.email, Time.at(@raw_tag.tagger.date.seconds))
+
+ return if signature.verified_signature.nil?
+
+ signature
+ end
+
+ private
+
+ def signature_text
+ @raw_tag.message.slice(@raw_tag.message.index("-----BEGIN SIGNED MESSAGE-----")..-1)
+ rescue
+ nil
+ end
+
+ def signed_text
+ # signed text is reconstructed as long as there is no specific gitaly function
+ %{object #{@raw_tag.target_commit.id}
+type commit
+tag #{@raw_tag.name}
+tagger #{@raw_tag.tagger.name} <#{@raw_tag.tagger.email}> #{@raw_tag.tagger.date.seconds} #{@raw_tag.tagger.timezone}
+
+#{@raw_tag.message.gsub(/-----BEGIN SIGNED MESSAGE-----(.*)-----END SIGNED MESSAGE-----/m, "")}}
+ end
+ end
+ end
+end
diff --git a/lib/gitlab_danger.rb b/lib/gitlab_danger.rb
index ee0951f18ca..1c1763454a5 100644
--- a/lib/gitlab_danger.rb
+++ b/lib/gitlab_danger.rb
@@ -12,6 +12,7 @@ class GitlabDanger
database
commit_messages
telemetry
+ utility_css
].freeze
CI_ONLY_RULES ||= %w[
@@ -19,7 +20,6 @@ class GitlabDanger
changelog
specs
roulette
- gitlab_ui_wg
ce_ee_vue_templates
].freeze
diff --git a/lib/google_api/auth.rb b/lib/google_api/auth.rb
index 56f056fd869..319e5d2063c 100644
--- a/lib/google_api/auth.rb
+++ b/lib/google_api/auth.rb
@@ -37,6 +37,10 @@ module GoogleApi
Gitlab::Auth::OAuth::Provider.config_for('google_oauth2')
end
+ def client_options
+ config.args.client_options.deep_symbolize_keys
+ end
+
def client
return @client if defined?(@client)
@@ -49,7 +53,8 @@ module GoogleApi
config.app_secret,
site: 'https://accounts.google.com',
token_url: '/o/oauth2/token',
- authorize_url: '/o/oauth2/auth'
+ authorize_url: '/o/oauth2/auth',
+ **client_options
)
end
end
diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb
index 293d0c563c5..a9551ffbd30 100644
--- a/lib/mattermost/client.rb
+++ b/lib/mattermost/client.rb
@@ -49,7 +49,7 @@ module Mattermost
end
def json_response(response)
- json_response = JSON.parse(response.body)
+ json_response = Gitlab::Json.parse(response.body, legacy_mode: true)
unless response.success?
raise Mattermost::ClientError.new(json_response['message'] || 'Undefined error')
diff --git a/lib/quality/helm3_client.rb b/lib/quality/helm3_client.rb
index f5eb0834386..afea73cbc50 100644
--- a/lib/quality/helm3_client.rb
+++ b/lib/quality/helm3_client.rb
@@ -17,10 +17,6 @@ module Quality
@revision ||= self[:revision].to_i
end
- def status
- @status ||= self[:status].downcase
- end
-
def last_update
@last_update ||= Time.parse(self[:last_update])
end
@@ -29,7 +25,7 @@ module Quality
# A single page of data and the corresponding page number.
Page = Struct.new(:releases, :number)
- def initialize(namespace:, tiller_namespace: nil)
+ def initialize(namespace:)
@namespace = namespace
end
diff --git a/lib/quality/helm_client.rb b/lib/quality/helm_client.rb
deleted file mode 100644
index fc4e1ca2d18..00000000000
--- a/lib/quality/helm_client.rb
+++ /dev/null
@@ -1,114 +0,0 @@
-# frozen_string_literal: true
-
-require 'time'
-require_relative '../gitlab/popen' unless defined?(Gitlab::Popen)
-
-module Quality
- class HelmClient
- CommandFailedError = Class.new(StandardError)
-
- attr_reader :tiller_namespace, :namespace
-
- RELEASE_JSON_ATTRIBUTES = %w[Name Revision Updated Status Chart AppVersion Namespace].freeze
-
- Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do
- def revision
- @revision ||= self[:revision].to_i
- end
-
- def last_update
- @last_update ||= Time.parse(self[:last_update])
- end
- end
-
- # A single page of data and the corresponding page number.
- Page = Struct.new(:releases, :number)
-
- def initialize(tiller_namespace:, namespace:)
- @tiller_namespace = tiller_namespace
- @namespace = namespace
- end
-
- def releases(args: [])
- each_release(args)
- end
-
- def delete(release_name:)
- run_command([
- 'delete',
- %(--tiller-namespace "#{tiller_namespace}"),
- '--purge',
- release_name
- ])
- end
-
- private
-
- def run_command(command)
- final_command = ['helm', *command].join(' ')
- puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
-
- result = Gitlab::Popen.popen_with_detail([final_command])
-
- if result.status.success?
- result.stdout.chomp.freeze
- else
- raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
- end
- end
-
- def raw_releases(args = [])
- command = [
- 'list',
- %(--namespace "#{namespace}"),
- %(--tiller-namespace "#{tiller_namespace}" --output json),
- *args
- ]
- json = JSON.parse(run_command(command))
-
- releases = json['Releases'].map do |json_release|
- Release.new(*json_release.values_at(*RELEASE_JSON_ATTRIBUTES))
- end
-
- [releases, json['Next']]
- rescue JSON::ParserError => ex
- puts "Ignoring this JSON parsing error: #{ex}" # rubocop:disable Rails/Output
- [[], nil]
- end
-
- # Fetches data from Helm and yields a Page object for every page
- # of data, without loading all of them into memory.
- #
- # method - The Octokit method to use for getting the data.
- # args - Arguments to pass to the `helm list` command.
- def each_releases_page(args, &block)
- return to_enum(__method__, args) unless block_given?
-
- page = 1
- offset = ''
-
- loop do
- final_args = args.dup
- final_args << "--offset #{offset}" unless offset.to_s.empty?
- collection, offset = raw_releases(final_args)
-
- yield Page.new(collection, page += 1)
-
- break if offset.to_s.empty?
- end
- end
-
- # Iterates over all of the releases.
- #
- # args - Any arguments to pass to the `helm list` command.
- def each_release(args, &block)
- return to_enum(__method__, args) unless block_given?
-
- each_releases_page(args) do |page|
- page.releases.each do |release|
- yield release
- end
- end
- end
- end
-end
diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb
index bbd8b4dcc3f..97b86fa8c2e 100644
--- a/lib/quality/test_level.rb
+++ b/lib/quality/test_level.rb
@@ -14,6 +14,7 @@ module Quality
],
unit: %w[
bin
+ channels
config
db
dependencies
diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb
index bf15130d17e..37e4e16e87e 100644
--- a/lib/rspec_flaky/listener.rb
+++ b/lib/rspec_flaky/listener.rb
@@ -40,7 +40,7 @@ module RspecFlaky
new_flaky_examples = flaky_examples - suite_flaky_examples
if new_flaky_examples.any?
Rails.logger.warn "\nNew flaky examples detected:\n"
- Rails.logger.warn JSON.pretty_generate(new_flaky_examples.to_h)
+ Rails.logger.warn Gitlab::Json.pretty_generate(new_flaky_examples.to_h)
RspecFlaky::Report.new(new_flaky_examples).write(RspecFlaky::Config.new_flaky_examples_report_path)
# write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path)
diff --git a/lib/static_model.rb b/lib/static_model.rb
index 86bf8d62f9a..27805817f4d 100644
--- a/lib/static_model.rb
+++ b/lib/static_model.rb
@@ -40,10 +40,6 @@ module StaticModel
end
def ==(other)
- if other.is_a? ::StaticModel
- id == other.id
- else
- super
- end
+ other.present? && other.is_a?(self.class) && id == other.id
end
end
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index 5e8e2ab9c25..8e9f220ec85 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -23,11 +23,20 @@
# An example defaults file can be found in lib/support/init.d/gitlab.default.example
###
-
### Environment variables
-RAILS_ENV="production"
-USE_UNICORN=""
-SIDEKIQ_WORKERS=1
+RAILS_ENV=${RAILS_ENV:-'production'}
+SIDEKIQ_WORKERS=${SIDEKIQ_WORKERS:-1}
+USE_WEB_SERVER=${USE_WEB_SERVER:-'puma'}
+
+case "${USE_WEB_SERVER}" in
+ puma|unicorn)
+ use_web_server="$USE_WEB_SERVER"
+ ;;
+ *)
+ echo "Unsupported web server '${USE_WEB_SERVER}' (Allowed: 'puma', 'unicorn')" 1>&2
+ exit 1
+ ;;
+esac
# Script variable names should be lower-case not to conflict with
# internal /bin/sh variables such as PATH, EDITOR or SHELL.
@@ -36,7 +45,7 @@ app_root="/home/$app_user/gitlab"
pid_path="$app_root/tmp/pids"
socket_path="$app_root/tmp/sockets"
rails_socket="$socket_path/gitlab.socket"
-web_server_pid_path="$pid_path/unicorn.pid"
+web_server_pid_path="$pid_path/$use_web_server.pid"
mail_room_enabled=false
mail_room_pid_path="$pid_path/mail_room.pid"
gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse 2> /dev/null && pwd)
@@ -67,13 +76,6 @@ if ! cd "$app_root" ; then
echo "Failed to cd into $app_root, exiting!"; exit 1
fi
-# Select the web server to use
-if [ -z "$USE_UNICORN" ]; then
- use_web_server="puma"
-else
- use_web_server="unicorn"
-fi
-
if [ -z "$SIDEKIQ_WORKERS" ]; then
sidekiq_pid_path="$pid_path/sidekiq.pid"
else
diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example
index bb271b16836..1b499467ad6 100644
--- a/lib/support/init.d/gitlab.default.example
+++ b/lib/support/init.d/gitlab.default.example
@@ -6,7 +6,7 @@
RAILS_ENV="production"
# Uncomment the line below to enable the Unicorn web server instead of Puma.
-# USE_UNICORN=1
+# use_web_server="unicorn"
# app_user defines the user that GitLab is run as.
# The default is "git".
@@ -26,8 +26,8 @@ pid_path="$app_root/tmp/pids"
socket_path="$app_root/tmp/sockets"
# web_server_pid_path defines the path in which to create the pid file fo the web_server
-# The default is "$pid_path/unicorn.pid"
-web_server_pid_path="$pid_path/unicorn.pid"
+# The default is "$pid_path/puma.pid"
+web_server_pid_path="$pid_path/puma.pid"
# sidekiq_pid_path defines the path in which to create the pid file for sidekiq
# The default is "$pid_path/sidekiq.pid"
diff --git a/lib/system_check/app/hashed_storage_all_projects_check.rb b/lib/system_check/app/hashed_storage_all_projects_check.rb
new file mode 100644
index 00000000000..7539309fbf4
--- /dev/null
+++ b/lib/system_check/app/hashed_storage_all_projects_check.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SystemCheck
+ module App
+ class HashedStorageAllProjectsCheck < SystemCheck::BaseCheck
+ set_name 'All projects are in hashed storage?'
+
+ def check?
+ !Project.with_unmigrated_storage.exists?
+ end
+
+ def show_error
+ try_fixing_it(
+ "Please migrate all projects to hashed storage#{' on the primary' if Gitlab.ee? && Gitlab::Geo.secondary?}",
+ "as legacy storage is deprecated in 13.0 and support will be removed in 14.0."
+ )
+
+ for_more_information('doc/administration/repository_storage_types.md')
+ end
+ end
+ end
+end
diff --git a/lib/system_check/app/hashed_storage_enabled_check.rb b/lib/system_check/app/hashed_storage_enabled_check.rb
new file mode 100644
index 00000000000..b7c1791b740
--- /dev/null
+++ b/lib/system_check/app/hashed_storage_enabled_check.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module SystemCheck
+ module App
+ class HashedStorageEnabledCheck < SystemCheck::BaseCheck
+ set_name 'GitLab configured to store new projects in hashed storage?'
+
+ def check?
+ Gitlab::CurrentSettings.current_application_settings.hashed_storage_enabled
+ end
+
+ def show_error
+ try_fixing_it(
+ "Please enable the setting",
+ "`Use hashed storage paths for newly created and renamed projects`",
+ "in GitLab's Admin panel to avoid security issues and ensure data integrity."
+ )
+
+ for_more_information('doc/administration/repository_storage_types.md')
+ end
+ end
+ end
+end
diff --git a/lib/system_check/rake_task/app_task.rb b/lib/system_check/rake_task/app_task.rb
index aec7e5f416e..99c93edd12d 100644
--- a/lib/system_check/rake_task/app_task.rb
+++ b/lib/system_check/rake_task/app_task.rb
@@ -31,7 +31,9 @@ module SystemCheck
SystemCheck::App::GitVersionCheck,
SystemCheck::App::GitUserDefaultSSHConfigCheck,
SystemCheck::App::ActiveUsersCheck,
- SystemCheck::App::AuthorizedKeysPermissionCheck
+ SystemCheck::App::AuthorizedKeysPermissionCheck,
+ SystemCheck::App::HashedStorageEnabledCheck,
+ SystemCheck::App::HashedStorageAllProjectsCheck
]
end
end
diff --git a/lib/tasks/file_hooks.rake b/lib/tasks/file_hooks.rake
index 66d382db612..f767d63fe0d 100644
--- a/lib/tasks/file_hooks.rake
+++ b/lib/tasks/file_hooks.rake
@@ -4,6 +4,11 @@ namespace :file_hooks do
puts 'Validating file hooks from /file_hooks and /plugins directories'
Gitlab::FileHook.files.each do |file|
+ if File.dirname(file).ends_with?('plugins')
+ puts 'DEPRECATED: /plugins directory is deprecated and will be removed in 14.0. ' \
+ 'Please move your files into /file_hooks directory.'
+ end
+
success, message = Gitlab::FileHook.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA)
if success
diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
index 8cf7c9e89f0..3833689e07e 100644
--- a/lib/tasks/gemojione.rake
+++ b/lib/tasks/gemojione.rake
@@ -7,7 +7,7 @@ namespace :gemojione do
aliases = {}
index_file = File.join(Rails.root, 'fixtures', 'emojis', 'index.json')
- index = JSON.parse(File.read(index_file))
+ index = Gitlab::Json.parse(File.read(index_file))
index.each_pair do |key, data|
data['aliases'].each do |a|
@@ -19,7 +19,7 @@ namespace :gemojione do
out = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json')
File.open(out, 'w') do |handle|
- handle.write(JSON.pretty_generate(aliases, indent: ' ', space: '', space_before: ''))
+ handle.write(Gitlab::Json.pretty_generate(aliases, indent: ' ', space: '', space_before: ''))
end
end
@@ -58,7 +58,7 @@ namespace :gemojione do
out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json')
File.open(out, 'w') do |handle|
- handle.write(JSON.pretty_generate(resultant_emoji_map))
+ handle.write(Gitlab::Json.pretty_generate(resultant_emoji_map))
end
end
diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake
index ee47f71af93..fc55d9704d1 100644
--- a/lib/tasks/gitlab/gitaly.rake
+++ b/lib/tasks/gitlab/gitaly.rake
@@ -13,10 +13,9 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
version = Gitlab::GitalyClient.expected_server_version
- checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
-
- command = %w[/usr/bin/env -u RUBYOPT -u BUNDLE_GEMFILE]
+ checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
+ command = []
_, status = Gitlab::Popen.popen(%w[which gmake])
command << (status.zero? ? 'gmake' : 'make')
@@ -31,7 +30,7 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
Dir.chdir(args.dir) do
# In CI we run scripts/gitaly-test-build instead of this command
unless ENV['CI'].present?
- Bundler.with_original_env { run_command!(command) }
+ Bundler.with_original_env { Gitlab::Popen.popen(command, nil, { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }) }
end
end
end
diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake
index c9c212fbe4d..4bdc62c9319 100644
--- a/lib/tasks/gitlab/import_export/export.rake
+++ b/lib/tasks/gitlab/import_export/export.rake
@@ -3,12 +3,12 @@
# Export project to archive
#
# @example
-# bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz, true]"
+# bundle exec rake "gitlab:import_export:export[root, root, project_to_export, /path/to/file.tar.gz]"
#
namespace :gitlab do
namespace :import_export do
desc 'GitLab | Import/Export | EXPERIMENTAL | Export large project archives'
- task :export, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args|
+ task :export, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args|
# Load it here to avoid polluting Rake tasks with Sidekiq test warnings
require 'sidekiq/testing'
@@ -18,6 +18,7 @@ namespace :gitlab do
warn_user_is_not_gitlab
if ENV['EXPORT_DEBUG'].present?
+ Gitlab::Utils::Measuring.logger = logger
ActiveRecord::Base.logger = logger
logger.level = Logger::DEBUG
else
@@ -29,7 +30,6 @@ namespace :gitlab do
project_path: args.project_path,
username: args.username,
file_path: args.archive_path,
- measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled),
logger: logger
)
diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake
index 7e2162a7774..2702b530334 100644
--- a/lib/tasks/gitlab/import_export/import.rake
+++ b/lib/tasks/gitlab/import_export/import.rake
@@ -7,12 +7,12 @@
# 2. Performs Sidekiq job synchronously
#
# @example
-# bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz, true]"
+# bundle exec rake "gitlab:import_export:import[root, root, imported_project, /path/to/file.tar.gz]"
#
namespace :gitlab do
namespace :import_export do
desc 'GitLab | Import/Export | EXPERIMENTAL | Import large project archives'
- task :import, [:username, :namespace_path, :project_path, :archive_path, :measurement_enabled] => :gitlab_environment do |_t, args|
+ task :import, [:username, :namespace_path, :project_path, :archive_path] => :gitlab_environment do |_t, args|
# Load it here to avoid polluting Rake tasks with Sidekiq test warnings
require 'sidekiq/testing'
@@ -22,6 +22,7 @@ namespace :gitlab do
warn_user_is_not_gitlab
if ENV['IMPORT_DEBUG'].present?
+ Gitlab::Utils::Measuring.logger = logger
ActiveRecord::Base.logger = logger
logger.level = Logger::DEBUG
else
@@ -33,7 +34,6 @@ namespace :gitlab do
project_path: args.project_path,
username: args.username,
file_path: args.archive_path,
- measurement_enabled: Gitlab::Utils.to_boolean(args.measurement_enabled),
logger: logger
)
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 6586699f8ba..d6e62a5c550 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -12,7 +12,7 @@ namespace :gitlab do
gitlab_url += '/' unless gitlab_url.end_with?('/')
target_dir = Gitlab.config.gitlab_shell.path
- checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir)
+ checkout_or_clone_version(version: default_version, repo: args.repo, target_dir: target_dir, clone_opts: %w[--depth 1])
# Make sure we're on the right tag
Dir.chdir(target_dir) do
diff --git a/lib/tasks/gitlab/snippets.rake b/lib/tasks/gitlab/snippets.rake
new file mode 100644
index 00000000000..c391cecfdbc
--- /dev/null
+++ b/lib/tasks/gitlab/snippets.rake
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+ namespace :snippets do
+ DEFAULT_LIMIT = 100
+
+ # @example
+ # bundle exec rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4
+ # bundle exec rake gitlab:snippets:migrate SNIPPET_IDS=1,2,3,4 LIMIT=50
+ desc 'GitLab | Migrate specific snippets to git'
+ task :migrate, [:ids] => :environment do |_, args|
+ unless ENV['SNIPPET_IDS'].presence
+ raise "Please supply the list of ids through the SNIPPET_IDS env var"
+ end
+
+ raise "Invalid limit value" if limit.zero?
+
+ if migration_running?
+ raise "There are already snippet migrations running. Please wait until they are finished."
+ end
+
+ ids = parse_snippet_ids!
+
+ puts "Starting the migration..."
+ Gitlab::BackgroundMigration::BackfillSnippetRepositories.new.perform_by_ids(ids)
+
+ list_non_migrated = non_migrated_snippets.where(id: ids)
+
+ if list_non_migrated.exists?
+ puts "The following snippets couldn't be migrated:"
+ puts list_non_migrated.pluck(:id).join(',')
+ else
+ puts "All snippets were migrated successfully"
+ end
+ end
+
+ def parse_snippet_ids!
+ ids = ENV['SNIPPET_IDS'].delete(' ').split(',').map do |id|
+ id.to_i.tap do |value|
+ raise "Invalid id provided" if value.zero?
+ end
+ end
+
+ if ids.size > limit
+ raise "The number of ids provided is higher than #{limit}. You can update this limit by using the env var `LIMIT`"
+ end
+
+ ids
+ end
+
+ # @example
+ # bundle exec rake gitlab:snippets:migration_status
+ desc 'GitLab | Show whether there are snippet background migrations running'
+ task migration_status: :environment do
+ if migration_running?
+ puts "There are snippet migrations running"
+ else
+ puts "There are no snippet migrations running"
+ end
+ end
+
+ def migration_running?
+ Sidekiq::ScheduledSet.new.any? { |r| r.klass == 'BackgroundMigrationWorker' && r.args[0] == 'BackfillSnippetRepositories' }
+ end
+
+ # @example
+ # bundle exec rake gitlab:snippets:list_non_migrated
+ # bundle exec rake gitlab:snippets:list_non_migrated LIMIT=50
+ desc 'GitLab | Show non migrated snippets'
+ task list_non_migrated: :environment do
+ raise "Invalid limit value" if limit.zero?
+
+ non_migrated_count = non_migrated_snippets.count
+ if non_migrated_count.zero?
+ puts "All snippets have been successfully migrated"
+ else
+ puts "There are #{non_migrated_count} snippets that haven't been migrated. Showing a batch of ids of those snippets:\n"
+ puts non_migrated_snippets.limit(limit).pluck(:id).join(',')
+ end
+ end
+
+ def non_migrated_snippets
+ @non_migrated_snippets ||= Snippet.select(:id).where.not(id: SnippetRepository.select(:snippet_id))
+ end
+
+ # There are problems with the specs if we memoize this value
+ def limit
+ ENV['LIMIT'] ? ENV['LIMIT'].to_i : DEFAULT_LIMIT
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/track_deployment.rake b/lib/tasks/gitlab/track_deployment.rake
deleted file mode 100644
index 6f101aea303..00000000000
--- a/lib/tasks/gitlab/track_deployment.rake
+++ /dev/null
@@ -1,9 +0,0 @@
-namespace :gitlab do
- desc 'GitLab | Tracks a deployment in GitLab Performance Monitoring'
- task track_deployment: :environment do
- metric = Gitlab::Metrics::Metric
- .new('deployments', version: Gitlab::VERSION)
-
- Gitlab::Metrics.submit_metrics([metric.to_hash])
- end
-end
diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake
index bae3e4e8001..53343c8f8ff 100644
--- a/lib/tasks/gitlab/workhorse.rake
+++ b/lib/tasks/gitlab/workhorse.rake
@@ -12,7 +12,7 @@ namespace :gitlab do
version = Gitlab::Workhorse.version
- checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir)
+ checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir, clone_opts: %w[--depth 1])
_, status = Gitlab::Popen.popen(%w[which gmake])
command = status.zero? ? 'gmake' : 'make'
diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake
deleted file mode 100644
index d74878835fd..00000000000
--- a/lib/tasks/sidekiq.rake
+++ /dev/null
@@ -1,38 +0,0 @@
-namespace :sidekiq do
- def deprecation_warning!
- warn <<~WARNING
- This task is deprecated and will be removed in 13.0 as it is thought to be unused.
-
- If you are using this task, please comment on the below issue:
- https://gitlab.com/gitlab-org/gitlab/issues/196731
- WARNING
- end
-
- desc '[DEPRECATED] GitLab | Sidekiq | Stop sidekiq'
- task :stop do
- deprecation_warning!
-
- system(*%w(bin/background_jobs stop))
- end
-
- desc '[DEPRECATED] GitLab | Sidekiq | Start sidekiq'
- task :start do
- deprecation_warning!
-
- system(*%w(bin/background_jobs start))
- end
-
- desc '[DEPRECATED] GitLab | Sidekiq | Restart sidekiq'
- task :restart do
- deprecation_warning!
-
- system(*%w(bin/background_jobs restart))
- end
-
- desc '[DEPRECATED] GitLab | Sidekiq | Start sidekiq with launchd on Mac OS X'
- task :launchd do
- deprecation_warning!
-
- system(*%w(bin/background_jobs start_silent))
- end
-end