summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-06-18 11:18:50 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-06-18 11:18:50 +0000
commit8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781 (patch)
treea77e7fe7a93de11213032ed4ab1f33a3db51b738 /lib
parent00b35af3db1abfe813a778f643dad221aad51fca (diff)
downloadgitlab-ce-8c7f4e9d5f36cff46365a7f8c4b9c21578c1e781.tar.gz
Add latest changes from gitlab-org/gitlab@13-1-stable-ee
Diffstat (limited to 'lib')
-rw-r--r--lib/api/api.rb2
-rw-r--r--lib/api/deploy_keys.rb5
-rw-r--r--lib/api/discussions.rb2
-rw-r--r--lib/api/entities/bridge.rb9
-rw-r--r--lib/api/entities/container_registry.rb1
-rw-r--r--lib/api/entities/group_detail.rb3
-rw-r--r--lib/api/entities/merge_request_basic.rb8
-rw-r--r--lib/api/entities/project.rb4
-rw-r--r--lib/api/entities/releases/evidence.rb2
-rw-r--r--lib/api/entities/releases/link.rb1
-rw-r--r--lib/api/entities/resource_milestone_event.rb20
-rw-r--r--lib/api/entities/runner_details.rb7
-rw-r--r--lib/api/entities/shared_group_with_group.rb17
-rw-r--r--lib/api/entities/shared_group_with_project.rb (renamed from lib/api/entities/shared_group.rb)2
-rw-r--r--lib/api/entities/ssh_key.rb3
-rw-r--r--lib/api/entities/user_with_admin.rb3
-rw-r--r--lib/api/features.rb4
-rw-r--r--lib/api/group_container_repositories.rb3
-rw-r--r--lib/api/group_export.rb6
-rw-r--r--lib/api/group_import.rb9
-rw-r--r--lib/api/groups.rb57
-rw-r--r--lib/api/helpers.rb2
-rw-r--r--lib/api/helpers/issues_helpers.rb2
-rw-r--r--lib/api/helpers/notes_helpers.rb2
-rw-r--r--lib/api/helpers/projects_helpers.rb2
-rw-r--r--lib/api/helpers/runner.rb32
-rw-r--r--lib/api/helpers/services_helpers.rb12
-rw-r--r--lib/api/issues.rb9
-rw-r--r--lib/api/jobs.rb26
-rw-r--r--lib/api/lsif_data.rb40
-rw-r--r--lib/api/merge_requests.rb9
-rw-r--r--lib/api/project_container_repositories.rb9
-rw-r--r--lib/api/project_export.rb4
-rw-r--r--lib/api/project_import.rb5
-rw-r--r--lib/api/project_repository_storage_moves.rb57
-rw-r--r--lib/api/projects.rb4
-rw-r--r--lib/api/release/links.rb2
-rw-r--r--lib/api/releases.rb11
-rw-r--r--lib/api/repositories.rb4
-rw-r--r--lib/api/resource_label_events.rb5
-rw-r--r--lib/api/resource_milestone_events.rb54
-rw-r--r--lib/api/runner.rb18
-rw-r--r--lib/api/search.rb15
-rw-r--r--lib/api/settings.rb6
-rw-r--r--lib/api/suggestions.rb43
-rw-r--r--lib/api/terraform/state.rb10
-rw-r--r--lib/api/todos.rb10
-rw-r--r--lib/api/users.rb19
-rw-r--r--lib/api/validations/validators/file_path.rb2
-rw-r--r--lib/api/validations/validators/untrusted_regexp.rb19
-rw-r--r--lib/api/wikis.rb2
-rw-r--r--lib/backup/files.rb3
-rw-r--r--lib/banzai/filter/ascii_doc_sanitization_filter.rb3
-rw-r--r--lib/banzai/filter/design_reference_filter.rb121
-rw-r--r--lib/banzai/filter/external_issue_reference_filter.rb36
-rw-r--r--lib/banzai/filter/gollum_tags_filter.rb20
-rw-r--r--lib/banzai/filter/issue_reference_filter.rb14
-rw-r--r--lib/banzai/filter/iteration_reference_filter.rb16
-rw-r--r--lib/banzai/filter/label_reference_filter.rb15
-rw-r--r--lib/banzai/filter/reference_filter.rb6
-rw-r--r--lib/banzai/filter/repository_link_filter.rb4
-rw-r--r--lib/banzai/filter/wiki_link_filter.rb10
-rw-r--r--lib/banzai/pipeline/gfm_pipeline.rb1
-rw-r--r--lib/banzai/reference_parser/iteration_parser.rb22
-rw-r--r--lib/event_filter.rb10
-rw-r--r--lib/extracts_path.rb124
-rw-r--r--lib/extracts_ref.rb128
-rw-r--r--lib/feature.rb105
-rw-r--r--lib/gitaly/server.rb9
-rw-r--r--lib/gitlab.rb9
-rw-r--r--lib/gitlab/alert_management/alert_params.rb3
-rw-r--r--lib/gitlab/alert_management/fingerprint.rb27
-rw-r--r--lib/gitlab/alerting/alert.rb11
-rw-r--r--lib/gitlab/alerting/notification_payload_parser.rb7
-rw-r--r--lib/gitlab/analytics/cycle_analytics/base_query_builder.rb56
-rw-r--r--lib/gitlab/analytics/cycle_analytics/median.rb4
-rw-r--r--lib/gitlab/analytics/cycle_analytics/records_fetcher.rb20
-rw-r--r--lib/gitlab/application_rate_limiter.rb9
-rw-r--r--lib/gitlab/auth/auth_finders.rb1
-rw-r--r--lib/gitlab/auth/ldap/person.rb2
-rw-r--r--lib/gitlab/auth/o_auth/provider.rb4
-rw-r--r--lib/gitlab/background_migration/.rubocop.yml2
-rw-r--r--lib/gitlab/background_migration/backfill_project_repositories.rb2
-rw-r--r--lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb2
-rw-r--r--lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb13
-rw-r--r--lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb2
-rw-r--r--lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb2
-rw-r--r--lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb2
-rw-r--r--lib/gitlab/background_migration/populate_untracked_uploads.rb2
-rw-r--r--lib/gitlab/background_migration/reset_merge_status.rb2
-rw-r--r--lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb2
-rw-r--r--lib/gitlab/badge/coverage/report.rb10
-rw-r--r--lib/gitlab/badge/coverage/template.rb14
-rw-r--r--lib/gitlab/badge/pipeline/status.rb8
-rw-r--r--lib/gitlab/badge/pipeline/template.rb14
-rw-r--r--lib/gitlab/badge/template.rb2
-rw-r--r--lib/gitlab/bitbucket_import/importer.rb2
-rw-r--r--lib/gitlab/cache/import/caching.rb11
-rw-r--r--lib/gitlab/ci/build/releaser.rb24
-rw-r--r--lib/gitlab/ci/build/step.rb13
-rw-r--r--lib/gitlab/ci/config/entry/job.rb2
-rw-r--r--lib/gitlab/ci/config/entry/reports.rb7
-rw-r--r--lib/gitlab/ci/features.rb30
-rw-r--r--lib/gitlab/ci/parsers/terraform/tfplan.rb24
-rw-r--r--lib/gitlab/ci/pipeline/chain/command.rb19
-rw-r--r--lib/gitlab/ci/pipeline/chain/metrics.rb35
-rw-r--r--lib/gitlab/ci/pipeline/chain/seed.rb1
-rw-r--r--lib/gitlab/ci/pipeline/chain/sequence.rb1
-rw-r--r--lib/gitlab/ci/reports/terraform_reports.rb8
-rw-r--r--lib/gitlab/ci/status/bridge/failed.rb8
-rw-r--r--lib/gitlab/ci/status/core.rb2
-rw-r--r--lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml13
-rw-r--r--lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml18
-rw-r--r--lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml3
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml18
-rw-r--r--lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml7
-rw-r--r--lib/gitlab/ci/templates/Rust.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml9
-rw-r--r--lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml24
-rw-r--r--lib/gitlab/ci/templates/Terraform.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml17
-rw-r--r--lib/gitlab/ci/trace.rb2
-rw-r--r--lib/gitlab/ci/yaml_processor.rb2
-rw-r--r--lib/gitlab/code_navigation_path.rb2
-rw-r--r--lib/gitlab/config/loader/yaml.rb8
-rw-r--r--lib/gitlab/contributions_calendar.rb8
-rw-r--r--lib/gitlab/cycle_analytics/summary/commit.rb2
-rw-r--r--lib/gitlab/cycle_analytics/summary/deploy.rb2
-rw-r--r--lib/gitlab/cycle_analytics/summary/issue.rb2
-rw-r--r--lib/gitlab/cycle_analytics/summary/value.rb12
-rw-r--r--lib/gitlab/danger/emoji_checker.rb6
-rw-r--r--lib/gitlab/danger/helper.rb2
-rw-r--r--lib/gitlab/danger/request_helper.rb2
-rw-r--r--lib/gitlab/danger/roulette.rb67
-rw-r--r--lib/gitlab/data_builder/alert.rb27
-rw-r--r--lib/gitlab/data_builder/note.rb2
-rw-r--r--lib/gitlab/database/custom_structure.rb44
-rw-r--r--lib/gitlab/database/migration_helpers.rb14
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers.rb116
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb140
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb198
-rw-r--r--lib/gitlab/database/schema_cleaner.rb18
-rw-r--r--lib/gitlab/database/schema_helpers.rb36
-rw-r--r--lib/gitlab/dependency_linker.rb4
-rw-r--r--lib/gitlab/dependency_linker/go_mod_linker.rb34
-rw-r--r--lib/gitlab/dependency_linker/go_sum_linker.rb38
-rw-r--r--lib/gitlab/diff/file.rb8
-rw-r--r--lib/gitlab/diff/formatters/base_formatter.rb11
-rw-r--r--lib/gitlab/diff/position.rb13
-rw-r--r--lib/gitlab/doctor/secrets.rb87
-rw-r--r--lib/gitlab/error_tracking.rb3
-rw-r--r--lib/gitlab/error_tracking/processor/sidekiq_processor.rb66
-rw-r--r--lib/gitlab/exception_log_formatter.rb10
-rw-r--r--lib/gitlab/experimentation.rb15
-rw-r--r--lib/gitlab/export/logger.rb11
-rw-r--r--lib/gitlab/file_detector.rb2
-rw-r--r--lib/gitlab/gfm/uploads_rewriter.rb15
-rw-r--r--lib/gitlab/git/commit.rb15
-rw-r--r--lib/gitlab/git/diff.rb2
-rw-r--r--lib/gitlab/git/rugged_impl/use_rugged.rb5
-rw-r--r--lib/gitlab/git_access.rb37
-rw-r--r--lib/gitlab/git_access_project.rb44
-rw-r--r--lib/gitlab/git_access_snippet.rb13
-rw-r--r--lib/gitlab/git_access_wiki.rb8
-rw-r--r--lib/gitlab/gitaly_client.rb12
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb9
-rw-r--r--lib/gitlab/github_import/bulk_importing.rb2
-rw-r--r--lib/gitlab/github_import/importer/diff_note_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/issue_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/label_links_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/lfs_objects_importer.rb5
-rw-r--r--lib/gitlab/github_import/importer/note_importer.rb2
-rw-r--r--lib/gitlab/github_import/importer/pull_requests_importer.rb6
-rw-r--r--lib/gitlab/gl_repository.rb19
-rw-r--r--lib/gitlab/gl_repository/identifier.rb74
-rw-r--r--lib/gitlab/gl_repository/repo_type.rb27
-rw-r--r--lib/gitlab/golang.rb99
-rw-r--r--lib/gitlab/graphql/authorize/authorize_field_service.rb14
-rw-r--r--lib/gitlab/graphql/authorize/instrumentation.rb6
-rw-r--r--lib/gitlab/graphql/filterable_array.rb14
-rw-r--r--lib/gitlab/graphql/loaders/full_path_model_loader.rb26
-rw-r--r--lib/gitlab/graphql/pagination/connections.rb4
-rw-r--r--lib/gitlab/graphql/pagination/filterable_array_connection.rb17
-rw-r--r--lib/gitlab/graphql/pagination/keyset/connection.rb100
-rw-r--r--lib/gitlab/import/database_helpers.rb2
-rw-r--r--lib/gitlab/import/set_async_jid.rb7
-rw-r--r--lib/gitlab/import_export/attributes_finder.rb2
-rw-r--r--lib/gitlab/import_export/attributes_permitter.rb105
-rw-r--r--lib/gitlab/import_export/importer.rb18
-rw-r--r--lib/gitlab/import_export/json/streaming_serializer.rb15
-rw-r--r--lib/gitlab/import_export/legacy_relation_tree_saver.rb8
-rw-r--r--lib/gitlab/import_export/members_mapper.rb2
-rw-r--r--lib/gitlab/import_export/merge_request_parser.rb8
-rw-r--r--lib/gitlab/import_export/project/import_export.yml2
-rw-r--r--lib/gitlab/import_export/repo_restorer.rb2
-rw-r--r--lib/gitlab/import_export/saver.rb21
-rw-r--r--lib/gitlab/import_export/snippet_repo_restorer.rb13
-rw-r--r--lib/gitlab/import_export/snippets_repo_restorer.rb6
-rw-r--r--lib/gitlab/import_export/version_checker.rb6
-rw-r--r--lib/gitlab/instrumentation/elasticsearch_transport.rb68
-rw-r--r--lib/gitlab/instrumentation/redis.rb81
-rw-r--r--lib/gitlab/instrumentation/redis_base.rb102
-rw-r--r--lib/gitlab/instrumentation/redis_interceptor.rb86
-rw-r--r--lib/gitlab/instrumentation/redis_payload.rb37
-rw-r--r--lib/gitlab/instrumentation_helper.rb54
-rw-r--r--lib/gitlab/issuable_metadata.rb101
-rw-r--r--lib/gitlab/jira/http_client.rb7
-rw-r--r--lib/gitlab/jira_import.rb40
-rw-r--r--lib/gitlab/jira_import/base_importer.rb2
-rw-r--r--lib/gitlab/jira_import/issues_importer.rb32
-rw-r--r--lib/gitlab/kubernetes/helm.rb6
-rw-r--r--lib/gitlab/kubernetes/helm/base_command.rb31
-rw-r--r--lib/gitlab/kubernetes/helm/client_command.rb4
-rw-r--r--lib/gitlab/kubernetes/helm/delete_command.rb14
-rw-r--r--lib/gitlab/kubernetes/helm/init_command.rb16
-rw-r--r--lib/gitlab/kubernetes/helm/install_command.rb15
-rw-r--r--lib/gitlab/kubernetes/helm/patch_command.rb16
-rw-r--r--lib/gitlab/kubernetes/helm/reset_command.rb15
-rw-r--r--lib/gitlab/kubernetes/network_policy.rb45
-rw-r--r--lib/gitlab/lfs_token.rb2
-rw-r--r--lib/gitlab/lograge/custom_options.rb18
-rw-r--r--lib/gitlab/looping_batcher.rb99
-rw-r--r--lib/gitlab/metrics/dashboard/stages/url_validator.rb19
-rw-r--r--lib/gitlab/metrics/elasticsearch_rack_middleware.rb41
-rw-r--r--lib/gitlab/metrics/methods.rb2
-rw-r--r--lib/gitlab/metrics/redis_rack_middleware.rb39
-rw-r--r--lib/gitlab/metrics/samplers/base_sampler.rb4
-rw-r--r--lib/gitlab/metrics/samplers/puma_sampler.rb2
-rw-r--r--lib/gitlab/metrics/samplers/ruby_sampler.rb3
-rw-r--r--lib/gitlab/metrics/sidekiq_middleware.rb15
-rw-r--r--lib/gitlab/metrics/subscribers/active_record.rb18
-rw-r--r--lib/gitlab/metrics/transaction.rb10
-rw-r--r--lib/gitlab/middleware/handle_ip_spoof_attack_error.rb33
-rw-r--r--lib/gitlab/monitor/demo_projects.rb27
-rw-r--r--lib/gitlab/pagination/keyset/request_context.rb4
-rw-r--r--lib/gitlab/pagination/offset_pagination.rb8
-rw-r--r--lib/gitlab/phabricator_import/cache/map.rb2
-rw-r--r--lib/gitlab/phabricator_import/worker_state.rb2
-rw-r--r--lib/gitlab/process_memory_cache/helper.rb51
-rw-r--r--lib/gitlab/project_search_results.rb26
-rw-r--r--lib/gitlab/project_template.rb10
-rw-r--r--lib/gitlab/prometheus/query_variables.rb14
-rw-r--r--lib/gitlab/prometheus_client.rb13
-rw-r--r--lib/gitlab/quick_actions/merge_request_actions.rb17
-rw-r--r--lib/gitlab/redis/cache.rb4
-rw-r--r--lib/gitlab/redis/queues.rb4
-rw-r--r--lib/gitlab/redis/shared_state.rb4
-rw-r--r--lib/gitlab/redis/wrapper.rb6
-rw-r--r--lib/gitlab/reference_extractor.rb2
-rw-r--r--lib/gitlab/regex.rb90
-rw-r--r--lib/gitlab/routing.rb42
-rw-r--r--lib/gitlab/rugged_instrumentation.rb7
-rw-r--r--lib/gitlab/search_context.rb162
-rw-r--r--lib/gitlab/search_results.rb2
-rw-r--r--lib/gitlab/setup_helper.rb4
-rw-r--r--lib/gitlab/sidekiq_config.rb4
-rw-r--r--lib/gitlab/sidekiq_config/cli_methods.rb7
-rw-r--r--lib/gitlab/sidekiq_config/dummy_worker.rb3
-rw-r--r--lib/gitlab/sidekiq_config/worker.rb5
-rw-r--r--lib/gitlab/sidekiq_logging/json_formatter.rb18
-rw-r--r--lib/gitlab/sidekiq_logging/structured_logger.rb7
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb3
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb45
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb26
-rw-r--r--lib/gitlab/sidekiq_middleware/server_metrics.rb42
-rw-r--r--lib/gitlab/slash_commands/presenters/help.rb2
-rw-r--r--lib/gitlab/snippet_search_results.rb2
-rw-r--r--lib/gitlab/sourcegraph.rb2
-rw-r--r--lib/gitlab/suggestions/commit_message.rb54
-rw-r--r--lib/gitlab/suggestions/file_suggestion.rb107
-rw-r--r--lib/gitlab/suggestions/suggestion_set.rb120
-rw-r--r--lib/gitlab/themes.rb3
-rw-r--r--lib/gitlab/url_builder.rb23
-rw-r--r--lib/gitlab/usage_data.rb169
-rw-r--r--lib/gitlab/usage_data_concerns/topology.rb137
-rw-r--r--lib/gitlab/usage_data_counters/base_counter.rb6
-rw-r--r--lib/gitlab/usage_data_counters/designs_counter.rb2
-rw-r--r--lib/gitlab/usage_data_counters/search_counter.rb24
-rw-r--r--lib/gitlab/utils.rb21
-rw-r--r--lib/gitlab/utils/log_limited_array.rb4
-rw-r--r--lib/gitlab/utils/usage_data.rb110
-rw-r--r--lib/gitlab/web_ide/config.rb44
-rw-r--r--lib/gitlab/web_ide/config/entry/global.rb29
-rw-r--r--lib/gitlab/web_ide/config/entry/terminal.rb75
-rw-r--r--lib/milestone_array.rb42
-rw-r--r--lib/object_storage/direct_upload.rb30
-rw-r--r--lib/peek/views/bullet_detailed.rb47
-rw-r--r--lib/peek/views/elasticsearch.rb47
-rw-r--r--lib/peek/views/gitaly.rb6
-rw-r--r--lib/peek/views/redis_detailed.rb7
-rw-r--r--lib/quality/test_level.rb1
-rwxr-xr-xlib/support/init.d/gitlab58
-rw-r--r--lib/tasks/gemojione.rake16
-rw-r--r--lib/tasks/gitlab/container_registry.rake35
-rw-r--r--lib/tasks/gitlab/db.rake35
-rw-r--r--lib/tasks/gitlab/doctor/secrets.rake12
-rw-r--r--lib/tasks/gitlab/features.rake2
-rw-r--r--lib/tasks/gitlab/shell.rake15
305 files changed, 5230 insertions, 1407 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb
index b8135539cda..fb67258f331 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -159,7 +159,6 @@ module API
mount ::API::Keys
mount ::API::Labels
mount ::API::Lint
- mount ::API::LsifData
mount ::API::Markdown
mount ::API::Members
mount ::API::MergeRequestDiffs
@@ -170,6 +169,7 @@ module API
mount ::API::Notes
mount ::API::Discussions
mount ::API::ResourceLabelEvents
+ mount ::API::ResourceMilestoneEvents
mount ::API::NotificationSettings
mount ::API::Pages
mount ::API::PagesDomains
diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb
index e86bcc19b2b..11340e91aae 100644
--- a/lib/api/deploy_keys.rb
+++ b/lib/api/deploy_keys.rb
@@ -25,7 +25,8 @@ module API
get "deploy_keys" do
authenticated_as_admin!
- present paginate(DeployKey.all), with: Entities::SSHKey
+ deploy_keys = DeployKey.all.preload_users
+ present paginate(deploy_keys), with: Entities::SSHKey
end
params do
@@ -42,7 +43,7 @@ module API
end
# rubocop: disable CodeReuse/ActiveRecord
get ":id/deploy_keys" do
- keys = user_project.deploy_keys_projects.preload(:deploy_key)
+ keys = user_project.deploy_keys_projects.preload(deploy_key: [:user])
present paginate(keys), with: Entities::DeployKeysProject
end
diff --git a/lib/api/discussions.rb b/lib/api/discussions.rb
index 0dd1850e526..7b453ada41c 100644
--- a/lib/api/discussions.rb
+++ b/lib/api/discussions.rb
@@ -78,6 +78,8 @@ module API
optional :line_range, type: Hash, desc: 'Multi-line start and end' do
requires :start_line_code, type: String, desc: 'Start line code for multi-line note'
requires :end_line_code, type: String, desc: 'End line code for multi-line note'
+ requires :start_line_type, type: String, desc: 'Start line type for multi-line note'
+ requires :end_line_type, type: String, desc: 'End line type for multi-line note'
end
end
end
diff --git a/lib/api/entities/bridge.rb b/lib/api/entities/bridge.rb
new file mode 100644
index 00000000000..8f0ee69399a
--- /dev/null
+++ b/lib/api/entities/bridge.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class Bridge < Entities::JobBasic
+ expose :downstream_pipeline, with: Entities::PipelineBasic
+ end
+ end
+end
diff --git a/lib/api/entities/container_registry.rb b/lib/api/entities/container_registry.rb
index 6250f35c7cb..cff627ab50a 100644
--- a/lib/api/entities/container_registry.rb
+++ b/lib/api/entities/container_registry.rb
@@ -16,6 +16,7 @@ module API
expose :project_id
expose :location
expose :created_at
+ expose :tags_count, if: -> (_, options) { options[:tags_count] }
expose :tags, using: Tag, if: -> (_, options) { options[:tags] }
end
diff --git a/lib/api/entities/group_detail.rb b/lib/api/entities/group_detail.rb
index e03047a6e75..93dc41da81d 100644
--- a/lib/api/entities/group_detail.rb
+++ b/lib/api/entities/group_detail.rb
@@ -3,6 +3,9 @@
module API
module Entities
class GroupDetail < Group
+ expose :shared_with_groups do |group, options|
+ SharedGroupWithGroup.represent(group.shared_with_group_links.public_or_visible_to_user(group, options[:current_user]))
+ end
expose :runners_token, if: lambda { |group, options| options[:user_can_admin_group] }
expose :projects, using: Entities::Project do |group, options|
projects = GroupProjectsFinder.new(
diff --git a/lib/api/entities/merge_request_basic.rb b/lib/api/entities/merge_request_basic.rb
index 1a89a83a619..1643f267938 100644
--- a/lib/api/entities/merge_request_basic.rb
+++ b/lib/api/entities/merge_request_basic.rb
@@ -6,19 +6,15 @@ module API
expose :merged_by, using: Entities::UserBasic do |merge_request, _options|
merge_request.metrics&.merged_by
end
-
expose :merged_at do |merge_request, _options|
merge_request.metrics&.merged_at
end
-
expose :closed_by, using: Entities::UserBasic do |merge_request, _options|
merge_request.metrics&.latest_closed_by
end
-
expose :closed_at do |merge_request, _options|
merge_request.metrics&.latest_closed_at
end
-
expose :title_html, if: -> (_, options) { options[:render_html] } do |entity|
MarkupHelper.markdown_field(entity, :title)
end
@@ -33,7 +29,6 @@ module API
merge_request.assignee
end
expose :author, :assignees, using: Entities::UserBasic
-
expose :source_project_id, :target_project_id
expose :labels do |merge_request, options|
if options[:with_labels_details]
@@ -85,11 +80,8 @@ module API
end
expose :squash
-
expose :task_completion_status
-
expose :cannot_be_merged?, as: :has_conflicts
-
expose :mergeable_discussions_state?, as: :blocking_discussions_resolved
end
end
diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb
index 39cd2d610e4..55a57501858 100644
--- a/lib/api/entities/project.rb
+++ b/lib/api/entities/project.rb
@@ -90,9 +90,10 @@ module API
expose :build_coverage_regex
expose :ci_config_path, if: -> (project, options) { Ability.allowed?(options[:current_user], :download_code, project) }
expose :shared_with_groups do |project, options|
- SharedGroup.represent(project.project_group_links, options)
+ SharedGroupWithProject.represent(project.project_group_links, options)
end
expose :only_allow_merge_if_pipeline_succeeds
+ expose :allow_merge_on_skipped_pipeline
expose :request_access_enabled
expose :only_allow_merge_if_all_discussions_are_resolved
expose :remove_source_branch_after_merge
@@ -119,6 +120,7 @@ module API
# MR describing the solution: https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20555
super(projects_relation).preload(:group)
.preload(:ci_cd_settings)
+ .preload(:project_setting)
.preload(:container_expiration_policy)
.preload(:auto_devops)
.preload(project_group_links: { group: :route },
diff --git a/lib/api/entities/releases/evidence.rb b/lib/api/entities/releases/evidence.rb
index 25b2bf6bf6f..01603a71dbf 100644
--- a/lib/api/entities/releases/evidence.rb
+++ b/lib/api/entities/releases/evidence.rb
@@ -6,7 +6,7 @@ module API
class Evidence < Grape::Entity
include ::API::Helpers::Presentable
- expose :summary_sha, as: :sha
+ expose :sha
expose :filepath
expose :collected_at
end
diff --git a/lib/api/entities/releases/link.rb b/lib/api/entities/releases/link.rb
index f4edb83bd58..654df2e2caf 100644
--- a/lib/api/entities/releases/link.rb
+++ b/lib/api/entities/releases/link.rb
@@ -9,6 +9,7 @@ module API
expose :url
expose :direct_asset_url
expose :external?, as: :external
+ expose :link_type
def direct_asset_url
return object.url unless object.filepath
diff --git a/lib/api/entities/resource_milestone_event.rb b/lib/api/entities/resource_milestone_event.rb
new file mode 100644
index 00000000000..26dc6620cbe
--- /dev/null
+++ b/lib/api/entities/resource_milestone_event.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class ResourceMilestoneEvent < Grape::Entity
+ expose :id
+ expose :user, using: Entities::UserBasic
+ expose :created_at
+ expose :resource_type do |event, _options|
+ event.issuable.class.name
+ end
+ expose :resource_id do |event, _options|
+ event.issuable.id
+ end
+ expose :milestone, using: Entities::Milestone
+ expose :action
+ expose :state
+ end
+ end
+end
diff --git a/lib/api/entities/runner_details.rb b/lib/api/entities/runner_details.rb
index 1dd8543d595..0afe298ef64 100644
--- a/lib/api/entities/runner_details.rb
+++ b/lib/api/entities/runner_details.rb
@@ -11,13 +11,6 @@ module API
expose :version, :revision, :platform, :architecture
expose :contacted_at
- # Will be removed: https://gitlab.com/gitlab-org/gitlab/-/issues/217105
- expose(:token, if: ->(runner, options) do
- return false if ::Feature.enabled?(:hide_token_from_runners_api, default_enabled: true)
-
- options[:current_user].admin? || !runner.instance_type?
- end)
-
# rubocop: disable CodeReuse/ActiveRecord
expose :projects, with: Entities::BasicProjectDetails do |runner, options|
if options[:current_user].admin?
diff --git a/lib/api/entities/shared_group_with_group.rb b/lib/api/entities/shared_group_with_group.rb
new file mode 100644
index 00000000000..1ca879182eb
--- /dev/null
+++ b/lib/api/entities/shared_group_with_group.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module API
+ module Entities
+ class SharedGroupWithGroup < Grape::Entity
+ expose :shared_with_group_id, as: :group_id
+ expose :group_name do |group_link|
+ group_link.shared_with_group.name
+ end
+ expose :group_full_path do |group_link|
+ group_link.shared_with_group.full_path
+ end
+ expose :group_access, as: :group_access_level
+ expose :expires_at
+ end
+ end
+end
diff --git a/lib/api/entities/shared_group.rb b/lib/api/entities/shared_group_with_project.rb
index 862e73e07f0..d91bee31b04 100644
--- a/lib/api/entities/shared_group.rb
+++ b/lib/api/entities/shared_group_with_project.rb
@@ -2,7 +2,7 @@
module API
module Entities
- class SharedGroup < Grape::Entity
+ class SharedGroupWithProject < Grape::Entity
expose :group_id
expose :group_name do |group_link, options|
group_link.group.name
diff --git a/lib/api/entities/ssh_key.rb b/lib/api/entities/ssh_key.rb
index aae216173c7..e1554730cb6 100644
--- a/lib/api/entities/ssh_key.rb
+++ b/lib/api/entities/ssh_key.rb
@@ -3,7 +3,8 @@
module API
module Entities
class SSHKey < Grape::Entity
- expose :id, :title, :key, :created_at, :expires_at
+ expose :id, :title, :created_at, :expires_at
+ expose :publishable_key, as: :key
end
end
end
diff --git a/lib/api/entities/user_with_admin.rb b/lib/api/entities/user_with_admin.rb
index d3df12200ff..c225ade6eb6 100644
--- a/lib/api/entities/user_with_admin.rb
+++ b/lib/api/entities/user_with_admin.rb
@@ -4,8 +4,7 @@ module API
module Entities
class UserWithAdmin < UserPublic
expose :admin?, as: :is_admin
+ expose :note
end
end
end
-
-API::Entities::UserWithAdmin.prepend_if_ee('EE::API::Entities::UserWithAdmin', with_descendants: true)
diff --git a/lib/api/features.rb b/lib/api/features.rb
index f507919b055..3fb3fc92e42 100644
--- a/lib/api/features.rb
+++ b/lib/api/features.rb
@@ -61,7 +61,7 @@ module API
mutually_exclusive :key, :project
end
post ':name' do
- feature = Feature.get(params[:name])
+ feature = Feature.get(params[:name]) # rubocop:disable Gitlab/AvoidFeatureGet
targets = gate_targets(params)
value = gate_value(params)
key = gate_key(params)
@@ -92,7 +92,7 @@ module API
desc 'Remove the gate value for the given feature'
delete ':name' do
- Feature.get(params[:name]).remove
+ Feature.remove(params[:name])
no_content!
end
diff --git a/lib/api/group_container_repositories.rb b/lib/api/group_container_repositories.rb
index 7f95b411b36..d34317b5271 100644
--- a/lib/api/group_container_repositories.rb
+++ b/lib/api/group_container_repositories.rb
@@ -20,6 +20,7 @@ module API
params do
use :pagination
optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included'
+ optional :tags_count, type: Boolean, default: false, desc: 'Determines if the tags count should be included'
end
get ':id/registry/repositories' do
repositories = ContainerRepositoriesFinder.new(
@@ -28,7 +29,7 @@ module API
track_event('list_repositories')
- present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags]
+ present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count]
end
end
diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb
index 8ca5dfa082e..d3010b6d147 100644
--- a/lib/api/group_export.rb
+++ b/lib/api/group_export.rb
@@ -2,6 +2,8 @@
module API
class GroupExport < Grape::API
+ helpers Helpers::RateLimiter
+
before do
not_found! unless Feature.enabled?(:group_import_export, user_group, default_enabled: true)
@@ -16,6 +18,8 @@ module API
detail 'This feature was introduced in GitLab 12.5.'
end
get ':id/export/download' do
+ check_rate_limit! :group_download_export, [current_user, user_group]
+
if user_group.export_file_exists?
present_carrierwave_file!(user_group.export_file)
else
@@ -27,6 +31,8 @@ module API
detail 'This feature was introduced in GitLab 12.5.'
end
post ':id/export' do
+ check_rate_limit! :group_export, [current_user]
+
export_service = ::Groups::ImportExport::ExportService.new(group: user_group, user: current_user)
if export_service.async_execute
diff --git a/lib/api/group_import.rb b/lib/api/group_import.rb
index ec51c2f44c3..afcbc24d3ce 100644
--- a/lib/api/group_import.rb
+++ b/lib/api/group_import.rb
@@ -2,8 +2,6 @@
module API
class GroupImport < Grape::API
- MAXIMUM_FILE_SIZE = 50.megabytes.freeze
-
helpers Helpers::FileUploadHelpers
helpers do
@@ -40,7 +38,10 @@ module API
status 200
content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
- ImportExportUploader.workhorse_authorize(has_length: false, maximum_size: MAXIMUM_FILE_SIZE)
+ ImportExportUploader.workhorse_authorize(
+ has_length: false,
+ maximum_size: Gitlab::CurrentSettings.max_import_size.megabytes
+ )
end
desc 'Create a new group import' do
@@ -69,7 +70,7 @@ module API
group = ::Groups::CreateService.new(current_user, group_params).execute
if group.persisted?
- GroupImportWorker.perform_async(current_user.id, group.id) # rubocop:disable CodeReuse/Worker
+ ::Groups::ImportExport::ImportService.new(group: group, user: current_user).async_execute
accepted!
else
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 353c8b4b242..6e07bb46721 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -23,13 +23,20 @@ module API
optional :order_by, type: String, values: %w[name path id], default: 'name', desc: 'Order by name, path or id'
optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)'
optional :min_access_level, type: Integer, values: Gitlab::Access.all_values, desc: 'Minimum access level of authenticated user'
+ optional :top_level_only, type: Boolean, desc: 'Only include top level groups'
use :pagination
end
# rubocop: disable CodeReuse/ActiveRecord
def find_groups(params, parent_id = nil)
find_params = params.slice(:all_available, :custom_attributes, :owned, :min_access_level)
- find_params[:parent] = find_group!(parent_id) if parent_id
+
+ find_params[:parent] = if params[:top_level_only]
+ [nil]
+ elsif parent_id
+ find_group!(parent_id)
+ end
+
find_params[:all_available] =
find_params.fetch(:all_available, current_user&.can_read_all_resources?)
@@ -144,6 +151,7 @@ module API
end
group = create_group
+ group.preload_shared_group_links
if group.persisted?
present group, with: Entities::GroupDetail, current_user: current_user
@@ -168,6 +176,8 @@ module API
end
put ':id' do
group = find_group!(params[:id])
+ group.preload_shared_group_links
+
authorize! :admin_group, group
if update_group(group)
@@ -186,6 +196,7 @@ module API
end
get ":id" do
group = find_group!(params[:id])
+ group.preload_shared_group_links
options = {
with: params[:with_projects] ? Entities::GroupDetail : Entities::Group,
@@ -292,6 +303,7 @@ module API
post ":id/projects/:project_id", requirements: { project_id: /.+/ } do
authenticated_as_admin!
group = find_group!(params[:id])
+ group.preload_shared_group_links
project = find_project!(params[:project_id])
result = ::Projects::TransferService.new(project, current_user).execute(group)
@@ -301,6 +313,49 @@ module API
render_api_error!("Failed to transfer project #{project.errors.messages}", 400)
end
end
+
+ desc 'Share a group with a group' do
+ success Entities::GroupDetail
+ end
+ params do
+ requires :group_id, type: Integer, desc: 'The ID of the group to share'
+ requires :group_access, type: Integer, values: Gitlab::Access.all_values, desc: 'The group access level'
+ optional :expires_at, type: Date, desc: 'Share expiration date'
+ end
+ post ":id/share" do
+ shared_group = find_group!(params[:id])
+ shared_with_group = find_group!(params[:group_id])
+
+ group_link_create_params = {
+ shared_group_access: params[:group_access],
+ expires_at: params[:expires_at]
+ }
+
+ result = ::Groups::GroupLinks::CreateService.new(shared_with_group, current_user, group_link_create_params).execute(shared_group)
+ shared_group.preload_shared_group_links
+
+ if result[:status] == :success
+ present shared_group, with: Entities::GroupDetail, current_user: current_user
+ else
+ render_api_error!(result[:message], result[:http_status])
+ end
+ end
+
+ params do
+ requires :group_id, type: Integer, desc: 'The ID of the shared group'
+ end
+ # rubocop: disable CodeReuse/ActiveRecord
+ delete ":id/share/:group_id" do
+ shared_group = find_group!(params[:id])
+
+ link = shared_group.shared_with_group_links.find_by(shared_with_group_id: params[:group_id])
+ not_found!('Group Link') unless link
+
+ ::Groups::GroupLinks::DestroyService.new(shared_group, current_user).execute(link)
+
+ no_content!
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
end
end
end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index c6f6dc255d4..bbdb45da3b1 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -437,7 +437,7 @@ module API
if report_exception?(exception)
define_params_for_grape_middleware
Gitlab::ErrorTracking.with_context(current_user) do
- Gitlab::ErrorTracking.track_exception(exception, params)
+ Gitlab::ErrorTracking.track_exception(exception)
end
end
diff --git a/lib/api/helpers/issues_helpers.rb b/lib/api/helpers/issues_helpers.rb
index e272b13f3ae..638b31cc7ba 100644
--- a/lib/api/helpers/issues_helpers.rb
+++ b/lib/api/helpers/issues_helpers.rb
@@ -24,6 +24,8 @@ module API
:discussion_locked,
:due_date,
:labels,
+ :add_labels,
+ :remove_labels,
:milestone_id,
:state_event,
:title
diff --git a/lib/api/helpers/notes_helpers.rb b/lib/api/helpers/notes_helpers.rb
index c85a38fc18b..f88624ed63e 100644
--- a/lib/api/helpers/notes_helpers.rb
+++ b/lib/api/helpers/notes_helpers.rb
@@ -133,7 +133,7 @@ module API
if resolved
parent = noteable_parent(noteable)
- ::Discussions::ResolveService.new(parent, current_user, merge_request: noteable).execute(discussion)
+ ::Discussions::ResolveService.new(parent, current_user, one_or_more_discussions: discussion).execute
else
discussion.unresolve!
end
diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb
index 5afdb34da97..8a115d42929 100644
--- a/lib/api/helpers/projects_helpers.rb
+++ b/lib/api/helpers/projects_helpers.rb
@@ -44,6 +44,7 @@ module API
optional :public_builds, type: Boolean, desc: 'Perform public builds'
optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed'
+ optional :allow_merge_on_skipped_pipeline, type: Boolean, desc: 'Allow to merge if pipeline is skipped'
optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved'
optional :tag_list, type: Array[String], desc: 'The list of tags for a project'
# TODO: remove rubocop disable - https://gitlab.com/gitlab-org/gitlab/issues/14960
@@ -92,6 +93,7 @@ module API
def self.update_params_at_least_one_of
[
+ :allow_merge_on_skipped_pipeline,
:autoclose_referenced_issues,
:auto_devops_enabled,
:auto_devops_deploy_strategy,
diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb
index 1f1253c8542..293d7ed9a6a 100644
--- a/lib/api/helpers/runner.rb
+++ b/lib/api/helpers/runner.rb
@@ -3,6 +3,8 @@
module API
module Helpers
module Runner
+ include Gitlab::Utils::StrongMemoize
+
prepend_if_ee('EE::API::Helpers::Runner') # rubocop: disable Cop/InjectEnterpriseEditionModule
JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'
@@ -16,7 +18,7 @@ module API
forbidden! unless current_runner
current_runner
- .update_cached_info(get_runner_details_from_request)
+ .heartbeat(get_runner_details_from_request)
end
def get_runner_details_from_request
@@ -31,31 +33,35 @@ module API
end
def current_runner
- @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s)
+ strong_memoize(:current_runner) do
+ ::Ci::Runner.find_by_token(params[:token].to_s)
+ end
end
- def validate_job!(job)
- not_found! unless job
+ def authenticate_job!(require_running: true)
+ job = current_job
- yield if block_given?
+ not_found! unless job
+ forbidden! unless job_token_valid?(job)
- project = job.project
- forbidden!('Project has been deleted!') if project.nil? || project.pending_delete?
+ forbidden!('Project has been deleted!') if job.project.nil? || job.project.pending_delete?
forbidden!('Job has been erased!') if job.erased?
- end
- def authenticate_job!
- job = current_job
+ if require_running
+ job_forbidden!(job, 'Job is not running') unless job.running?
+ end
- validate_job!(job) do
- forbidden! unless job_token_valid?(job)
+ if Gitlab::Ci::Features.job_heartbeats_runner?(job.project)
+ job.runner&.heartbeat(get_runner_ip)
end
job
end
def current_job
- @current_job ||= Ci::Build.find_by_id(params[:id])
+ strong_memoize(:current_job) do
+ Ci::Build.find_by_id(params[:id])
+ end
end
def job_token_valid?(job)
diff --git a/lib/api/helpers/services_helpers.rb b/lib/api/helpers/services_helpers.rb
index 02e60ff5db5..3d6039cacaa 100644
--- a/lib/api/helpers/services_helpers.rb
+++ b/lib/api/helpers/services_helpers.rb
@@ -583,6 +583,18 @@ module API
name: :api_url,
type: String,
desc: 'Prometheus API Base URL, like http://prometheus.example.com/'
+ },
+ {
+ required: true,
+ name: :google_iap_audience_client_id,
+ type: String,
+ desc: 'Client ID of the IAP secured resource (looks like IAP_CLIENT_ID.apps.googleusercontent.com)'
+ },
+ {
+ required: true,
+ name: :google_iap_service_account_json,
+ type: String,
+ desc: 'Contents of the credentials.json file of your service account, like: { "type": "service_account", "project_id": ... }'
}
],
'pushover' => [
diff --git a/lib/api/issues.rb b/lib/api/issues.rb
index be50c3e0381..2374ac11f4a 100644
--- a/lib/api/issues.rb
+++ b/lib/api/issues.rb
@@ -5,7 +5,6 @@ module API
include PaginationParams
helpers Helpers::IssuesHelpers
helpers Helpers::RateLimiter
- helpers ::Gitlab::IssuableMetadata
before { authenticate_non_get! }
@@ -67,6 +66,8 @@ module API
optional :assignee_id, type: Integer, desc: '[Deprecated] The ID of a user to assign issue'
optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue'
optional :labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names'
+ optional :add_labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names'
+ optional :remove_labels, type: Array[String], coerce_with: Validations::Types::LabelsList.coerce, desc: 'Comma-separated list of label names'
optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY'
optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential'
optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked"
@@ -106,7 +107,7 @@ module API
with: Entities::Issue,
with_labels_details: declared_params[:with_labels_details],
current_user: current_user,
- issuable_metadata: issuable_meta_data(issues, 'Issue', current_user),
+ issuable_metadata: Gitlab::IssuableMetadata.new(current_user, issues).data,
include_subscribed: false
}
@@ -132,7 +133,7 @@ module API
with: Entities::Issue,
with_labels_details: declared_params[:with_labels_details],
current_user: current_user,
- issuable_metadata: issuable_meta_data(issues, 'Issue', current_user),
+ issuable_metadata: Gitlab::IssuableMetadata.new(current_user, issues).data,
include_subscribed: false,
group: user_group
}
@@ -169,7 +170,7 @@ module API
with_labels_details: declared_params[:with_labels_details],
current_user: current_user,
project: user_project,
- issuable_metadata: issuable_meta_data(issues, 'Issue', current_user),
+ issuable_metadata: Gitlab::IssuableMetadata.new(current_user, issues).data,
include_subscribed: false
}
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb
index 59f0dbe8a9b..61a7fc107ef 100644
--- a/lib/api/jobs.rb
+++ b/lib/api/jobs.rb
@@ -70,6 +70,32 @@ module API
end
# rubocop: enable CodeReuse/ActiveRecord
+ desc 'Get pipeline bridge jobs' do
+ success Entities::Bridge
+ end
+ params do
+ requires :pipeline_id, type: Integer, desc: 'The pipeline ID'
+ use :optional_scope
+ use :pagination
+ end
+ # rubocop: disable CodeReuse/ActiveRecord
+ get ':id/pipelines/:pipeline_id/bridges' do
+ authorize!(:read_build, user_project)
+ pipeline = user_project.ci_pipelines.find(params[:pipeline_id])
+ authorize!(:read_pipeline, pipeline)
+
+ bridges = pipeline.bridges
+ bridges = filter_builds(bridges, params[:scope])
+ bridges = bridges.preload(
+ :metadata,
+ downstream_pipeline: [project: [:route, { namespace: :route }]],
+ project: [:namespace]
+ )
+
+ present paginate(bridges), with: Entities::Bridge
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
desc 'Get a specific job of a project' do
success Entities::Job
end
diff --git a/lib/api/lsif_data.rb b/lib/api/lsif_data.rb
deleted file mode 100644
index a673ccb4af0..00000000000
--- a/lib/api/lsif_data.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module API
- class LsifData < Grape::API
- MAX_FILE_SIZE = 10.megabytes
-
- before do
- not_found! if Feature.disabled?(:code_navigation, user_project)
- end
-
- params do
- requires :id, type: String, desc: 'The ID of a project'
- requires :commit_id, type: String, desc: 'The ID of a commit'
- end
- resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
- segment ':id/commits/:commit_id' do
- params do
- requires :paths, type: Array, desc: 'The paths of the files'
- end
- get 'lsif/info' do
- authorize! :download_code, user_project
-
- artifact =
- Ci::JobArtifact
- .with_file_types(['lsif'])
- .for_sha(params[:commit_id], @project.id)
- .last
-
- not_found! unless artifact
- authorize! :read_pipeline, artifact.job.pipeline
- file_too_large! if artifact.file.cached_size > MAX_FILE_SIZE
-
- service = ::Projects::LsifDataService.new(artifact.file, @project, params[:commit_id])
-
- params[:paths].to_h { |path| [path, service.execute(path)] }
- end
- end
- end
- end
-end
diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb
index ff4ad85115b..773a451d3a8 100644
--- a/lib/api/merge_requests.rb
+++ b/lib/api/merge_requests.rb
@@ -8,7 +8,6 @@ module API
before { authenticate_non_get! }
- helpers ::Gitlab::IssuableMetadata
helpers Helpers::MergeRequestsHelpers
# EE::API::MergeRequests would override the following helpers
@@ -92,10 +91,8 @@ module API
if params[:view] == 'simple'
options[:with] = Entities::MergeRequestSimple
else
- options[:issuable_metadata] = issuable_meta_data(merge_requests, 'MergeRequest', current_user)
- if Feature.enabled?(:mr_list_api_skip_merge_status_recheck, default_enabled: true)
- options[:skip_merge_status_recheck] = !declared_params[:with_merge_status_recheck]
- end
+ options[:issuable_metadata] = Gitlab::IssuableMetadata.new(current_user, merge_requests).data
+ options[:skip_merge_status_recheck] = !declared_params[:with_merge_status_recheck]
end
options
@@ -478,7 +475,7 @@ module API
squash_commit_message: params[:squash_commit_message],
should_remove_source_branch: params[:should_remove_source_branch],
sha: params[:sha] || merge_request.diff_head_sha
- )
+ ).compact
if immediately_mergeable
::MergeRequests::MergeService
diff --git a/lib/api/project_container_repositories.rb b/lib/api/project_container_repositories.rb
index 555fd98b451..2a0099018d9 100644
--- a/lib/api/project_container_repositories.rb
+++ b/lib/api/project_container_repositories.rb
@@ -21,6 +21,7 @@ module API
params do
use :pagination
optional :tags, type: Boolean, default: false, desc: 'Determines if tags should be included'
+ optional :tags_count, type: Boolean, default: false, desc: 'Determines if the tags count should be included'
end
get ':id/registry/repositories' do
repositories = ContainerRepositoriesFinder.new(
@@ -29,7 +30,7 @@ module API
track_event( 'list_repositories')
- present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags]
+ present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count]
end
desc 'Delete repository' do
@@ -69,11 +70,11 @@ module API
end
params do
requires :repository_id, type: Integer, desc: 'The ID of the repository'
- optional :name_regex_delete, type: String, desc: 'The tag name regexp to delete, specify .* to delete all'
- optional :name_regex, type: String, desc: 'The tag name regexp to delete, specify .* to delete all'
+ optional :name_regex_delete, type: String, untrusted_regexp: true, desc: 'The tag name regexp to delete, specify .* to delete all'
+ optional :name_regex, type: String, untrusted_regexp: true, desc: 'The tag name regexp to delete, specify .* to delete all'
# require either name_regex (deprecated) or name_regex_delete, it is ok to have both
at_least_one_of :name_regex, :name_regex_delete
- optional :name_regex_keep, type: String, desc: 'The tag name regexp to retain'
+ optional :name_regex_keep, type: String, untrusted_regexp: true, desc: 'The tag name regexp to retain'
optional :keep_n, type: Integer, desc: 'Keep n of latest tags with matching name'
optional :older_than, type: String, desc: 'Delete older than: 1h, 1d, 1month'
end
diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index 9fd9d13a20c..4b35f245b8c 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -25,7 +25,7 @@ module API
detail 'This feature was introduced in GitLab 10.6.'
end
get ':id/export/download' do
- check_rate_limit! :project_download_export, [current_user, :project_download_export, user_project]
+ check_rate_limit! :project_download_export, [current_user, user_project]
if user_project.export_file_exists?
present_carrierwave_file!(user_project.export_file)
@@ -45,7 +45,7 @@ module API
end
end
post ':id/export' do
- check_rate_limit! :project_export, [current_user, :project_export, user_project]
+ check_rate_limit! :project_export, [current_user]
project_export_params = declared_params(include_missing: false)
after_export_params = project_export_params.delete(:upload) || {}
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index 0e83686cab2..17d08d14a20 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -30,7 +30,10 @@ module API
status 200
content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
- ImportExportUploader.workhorse_authorize(has_length: false, maximum_size: MAXIMUM_FILE_SIZE)
+ ImportExportUploader.workhorse_authorize(
+ has_length: false,
+ maximum_size: Gitlab::CurrentSettings.max_import_size.megabytes
+ )
end
params do
diff --git a/lib/api/project_repository_storage_moves.rb b/lib/api/project_repository_storage_moves.rb
index 1a63e984fbf..5de623102fb 100644
--- a/lib/api/project_repository_storage_moves.rb
+++ b/lib/api/project_repository_storage_moves.rb
@@ -24,11 +24,64 @@ module API
detail 'This feature was introduced in GitLab 13.0.'
success Entities::ProjectRepositoryStorageMove
end
- get ':id' do
- storage_move = ProjectRepositoryStorageMove.find(params[:id])
+ params do
+ requires :repository_storage_move_id, type: Integer, desc: 'The ID of a project repository storage move'
+ end
+ get ':repository_storage_move_id' do
+ storage_move = ProjectRepositoryStorageMove.find(params[:repository_storage_move_id])
+
+ present storage_move, with: Entities::ProjectRepositoryStorageMove, current_user: current_user
+ end
+ end
+
+ params do
+ requires :id, type: String, desc: 'The ID of a project'
+ end
+ resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc 'Get a list of all project repository storage moves' do
+ detail 'This feature was introduced in GitLab 13.1.'
+ success Entities::ProjectRepositoryStorageMove
+ end
+ params do
+ use :pagination
+ end
+ get ':id/repository_storage_moves' do
+ storage_moves = user_project.repository_storage_moves.with_projects.order_created_at_desc
+
+ present paginate(storage_moves), with: Entities::ProjectRepositoryStorageMove, current_user: current_user
+ end
+
+ desc 'Get a project repository storage move' do
+ detail 'This feature was introduced in GitLab 13.1.'
+ success Entities::ProjectRepositoryStorageMove
+ end
+ params do
+ requires :repository_storage_move_id, type: Integer, desc: 'The ID of a project repository storage move'
+ end
+ get ':id/repository_storage_moves/:repository_storage_move_id' do
+ storage_move = user_project.repository_storage_moves.find(params[:repository_storage_move_id])
present storage_move, with: Entities::ProjectRepositoryStorageMove, current_user: current_user
end
+
+ desc 'Schedule a project repository storage move' do
+ detail 'This feature was introduced in GitLab 13.1.'
+ success Entities::ProjectRepositoryStorageMove
+ end
+ params do
+ requires :destination_storage_name, type: String, desc: 'The destination storage shard'
+ end
+ post ':id/repository_storage_moves' do
+ storage_move = user_project.repository_storage_moves.build(
+ declared_params.merge(source_storage_name: user_project.repository_storage)
+ )
+
+ if storage_move.schedule
+ present storage_move, with: Entities::ProjectRepositoryStorageMove, current_user: current_user
+ else
+ render_validation_error!(storage_move)
+ end
+ end
end
end
end
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index f305da681c4..e00fb61f478 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -502,7 +502,9 @@ module API
link = user_project.project_group_links.find_by(group_id: params[:group_id])
not_found!('Group Link') unless link
- destroy_conditionally!(link)
+ destroy_conditionally!(link) do
+ ::Projects::GroupLinks::DestroyService.new(user_project, current_user).execute(link)
+ end
end
# rubocop: enable CodeReuse/ActiveRecord
diff --git a/lib/api/release/links.rb b/lib/api/release/links.rb
index f72230c084c..07c27f39539 100644
--- a/lib/api/release/links.rb
+++ b/lib/api/release/links.rb
@@ -40,6 +40,7 @@ module API
requires :name, type: String, desc: 'The name of the link'
requires :url, type: String, desc: 'The URL of the link'
optional :filepath, type: String, desc: 'The filepath of the link'
+ optional :link_type, type: String, desc: 'The link type'
end
post 'links' do
authorize! :create_release, release
@@ -75,6 +76,7 @@ module API
optional :name, type: String, desc: 'The name of the link'
optional :url, type: String, desc: 'The URL of the link'
optional :filepath, type: String, desc: 'The filepath of the link'
+ optional :link_type, type: String, desc: 'The link type'
at_least_one_of :name, :url
end
put do
diff --git a/lib/api/releases.rb b/lib/api/releases.rb
index 95b3e90323c..a5bb1a44f1f 100644
--- a/lib/api/releases.rb
+++ b/lib/api/releases.rb
@@ -67,7 +67,6 @@ module API
if result[:status] == :success
log_release_created_audit_event(result[:release])
- create_evidence!
present result[:release], with: Entities::Release, current_user: current_user
else
@@ -169,16 +168,6 @@ module API
def log_release_milestones_updated_audit_event
# This is a separate method so that EE can extend its behaviour
end
-
- def create_evidence!
- return if release.historical_release?
-
- if release.upcoming_release?
- CreateEvidenceWorker.perform_at(release.released_at, release.id) # rubocop:disable CodeReuse/Worker
- else
- CreateEvidenceWorker.perform_async(release.id) # rubocop:disable CodeReuse/Worker
- end
- end
end
end
end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 0b2df85f61f..bf4f08ce390 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -6,6 +6,8 @@ module API
class Repositories < Grape::API
include PaginationParams
+ helpers ::API::Helpers::HeadersHelpers
+
before { authorize! :download_code, user_project }
params do
@@ -67,6 +69,8 @@ module API
get ':id/repository/blobs/:sha/raw' do
assign_blob_vars!
+ no_cache_headers
+
send_git_blob @repo, @blob
end
diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb
index f7f7c881f4a..1fa6898b92c 100644
--- a/lib/api/resource_label_events.rb
+++ b/lib/api/resource_label_events.rb
@@ -27,10 +27,9 @@ module API
get ":id/#{eventables_str}/:eventable_id/resource_label_events" do
eventable = find_noteable(eventable_type, params[:eventable_id])
- opts = { page: params[:page], per_page: params[:per_page] }
- events = ResourceLabelEventFinder.new(current_user, eventable, opts).execute
+ events = eventable.resource_label_events.inc_relations
- present paginate(events), with: Entities::ResourceLabelEvent
+ present ResourceLabelEvent.visible_to_user?(current_user, paginate(events)), with: Entities::ResourceLabelEvent
end
desc "Get a single #{eventable_type.to_s.downcase} resource label event" do
diff --git a/lib/api/resource_milestone_events.rb b/lib/api/resource_milestone_events.rb
new file mode 100644
index 00000000000..30ff5a9b4be
--- /dev/null
+++ b/lib/api/resource_milestone_events.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module API
+ class ResourceMilestoneEvents < Grape::API
+ include PaginationParams
+ helpers ::API::Helpers::NotesHelpers
+
+ before { authenticate! }
+
+ [Issue, MergeRequest].each do |eventable_type|
+ parent_type = eventable_type.parent_class.to_s.underscore
+ eventables_str = eventable_type.to_s.underscore.pluralize
+
+ params do
+ requires :id, type: String, desc: "The ID of a #{parent_type}"
+ end
+ resource parent_type.pluralize.to_sym, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+ desc "Get a list of #{eventable_type.to_s.downcase} resource milestone events" do
+ success Entities::ResourceMilestoneEvent
+ end
+ params do
+ requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable'
+ use :pagination
+ end
+
+ get ":id/#{eventables_str}/:eventable_id/resource_milestone_events" do
+ eventable = find_noteable(eventable_type, params[:eventable_id])
+
+ opts = { page: params[:page], per_page: params[:per_page] }
+ events = ResourceMilestoneEventFinder.new(current_user, eventable, opts).execute
+
+ present paginate(events), with: Entities::ResourceMilestoneEvent
+ end
+
+ desc "Get a single #{eventable_type.to_s.downcase} resource milestone event" do
+ success Entities::ResourceMilestoneEvent
+ end
+ params do
+ requires :event_id, type: String, desc: 'The ID of a resource milestone event'
+ requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable'
+ end
+ get ":id/#{eventables_str}/:eventable_id/resource_milestone_events/:event_id" do
+ eventable = find_noteable(eventable_type, params[:eventable_id])
+
+ event = eventable.resource_milestone_events.find(params[:event_id])
+
+ not_found!('ResourceMilestoneEvent') unless can?(current_user, :read_milestone, event.milestone_parent)
+
+ present event, with: Entities::ResourceMilestoneEvent
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index 9095aba7340..5f08ebe4a06 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -154,7 +154,6 @@ module API
end
put '/:id' do
job = authenticate_job!
- job_forbidden!(job, 'Job is not running') unless job.running?
job.trace.set(params[:trace]) if params[:trace]
@@ -182,7 +181,6 @@ module API
end
patch '/:id/trace' do
job = authenticate_job!
- job_forbidden!(job, 'Job is not running') unless job.running?
error!('400 Missing header Content-Range', 400) unless request.headers.key?('Content-Range')
content_range = request.headers['Content-Range']
@@ -220,6 +218,8 @@ module API
requires :id, type: Integer, desc: %q(Job's ID)
optional :token, type: String, desc: %q(Job's authentication token)
optional :filesize, type: Integer, desc: %q(Artifacts filesize)
+ optional :artifact_type, type: String, desc: %q(The type of artifact),
+ default: 'archive', values: Ci::JobArtifact.file_types.keys
end
post '/:id/artifacts/authorize' do
not_allowed! unless Gitlab.config.artifacts.enabled
@@ -227,18 +227,15 @@ module API
Gitlab::Workhorse.verify_api_request!(headers)
job = authenticate_job!
- forbidden!('Job is not running') unless job.running?
- max_size = max_artifacts_size(job)
+ service = Ci::AuthorizeJobArtifactService.new(job, params, max_size: max_artifacts_size(job))
- if params[:filesize]
- file_size = params[:filesize].to_i
- file_too_large! unless file_size < max_size
- end
+ forbidden! if service.forbidden?
+ file_too_large! if service.too_large?
status 200
content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
- JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_size)
+ service.headers
end
desc 'Upload artifacts for job' do
@@ -265,7 +262,6 @@ module API
require_gitlab_workhorse!
job = authenticate_job!
- forbidden!('Job is not running!') unless job.running?
artifacts = params[:file]
metadata = params[:metadata]
@@ -292,7 +288,7 @@ module API
optional :direct_download, default: false, type: Boolean, desc: %q(Perform direct download from remote storage instead of proxying artifacts)
end
get '/:id/artifacts' do
- job = authenticate_job!
+ job = authenticate_job!(require_running: false)
present_carrierwave_file!(job.artifacts_file, supports_direct_download: params[:direct_download])
end
diff --git a/lib/api/search.rb b/lib/api/search.rb
index 3d2d4527e30..ac00d3682a0 100644
--- a/lib/api/search.rb
+++ b/lib/api/search.rb
@@ -20,6 +20,13 @@ module API
users: Entities::UserBasic
}.freeze
+ SCOPE_PRELOAD_METHOD = {
+ merge_requests: :with_api_entity_associations,
+ projects: :with_api_entity_associations,
+ issues: :with_api_entity_associations,
+ milestones: :with_api_entity_associations
+ }.freeze
+
def search(additional_params = {})
search_params = {
scope: params[:scope],
@@ -29,7 +36,9 @@ module API
per_page: params[:per_page]
}.merge(additional_params)
- results = SearchService.new(current_user, search_params).search_objects
+ results = SearchService.new(current_user, search_params).search_objects(preload_method)
+
+ Gitlab::UsageDataCounters::SearchCounter.count(:all_searches)
paginate(results)
end
@@ -42,6 +51,10 @@ module API
SCOPE_ENTITY[params[:scope].to_sym]
end
+ def preload_method
+ SCOPE_PRELOAD_METHOD[params[:scope].to_sym]
+ end
+
def verify_search_scope!(resource:)
# In EE we have additional validation requirements for searches.
# Defining this method here as a noop allows us to easily extend it in
diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index e3a8f0671ef..0bf5eed26b4 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -83,6 +83,7 @@ module API
desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com'
optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts"
optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB'
+ optional :max_import_size, type: Integer, desc: 'Maximum import size in MB'
optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB'
optional :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.'
optional :password_authentication_enabled, type: Boolean, desc: 'Flag indicating if password authentication is enabled for the web interface' # support legacy names, can be removed in v5
@@ -113,6 +114,7 @@ module API
end
optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues."
optional :repository_storages, type: Array[String], desc: 'Storage paths for new projects'
+ optional :repository_storages_weighted, type: Hash, desc: 'Storage paths for new projects with a weighted value between 0 and 100'
optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to set up Two-factor authentication'
given require_two_factor_authentication: ->(val) { val } do
requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication'
@@ -132,6 +134,10 @@ module API
given sourcegraph_enabled: ->(val) { val } do
requires :sourcegraph_url, type: String, desc: 'The configured Sourcegraph instance URL'
end
+ optional :spam_check_endpoint_enabled, type: Boolean, desc: 'Enable Spam Check via external API endpoint'
+ given spam_check_endpoint_enabled: ->(val) { val } do
+ requires :spam_check_endpoint_url, type: String, desc: 'The URL of the external Spam Check service endpoint'
+ end
optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.'
optional :usage_ping_enabled, type: Boolean, desc: 'Every week GitLab will report license usage back to GitLab, Inc.'
optional :instance_statistics_visibility_private, type: Boolean, desc: 'When set to `true` Instance statistics will only be available to admins'
diff --git a/lib/api/suggestions.rb b/lib/api/suggestions.rb
index d008d1b9e97..05aaa8a6f41 100644
--- a/lib/api/suggestions.rb
+++ b/lib/api/suggestions.rb
@@ -14,18 +14,51 @@ module API
put ':id/apply' do
suggestion = Suggestion.find_by_id(params[:id])
- not_found! unless suggestion
- authorize! :apply_suggestion, suggestion
+ if suggestion
+ apply_suggestions(suggestion, current_user)
+ else
+ render_api_error!(_('Suggestion is not applicable as the suggestion was not found.'), :not_found)
+ end
+ end
+
+ desc 'Apply multiple suggestion patches in the Merge Request where they were created' do
+ success Entities::Suggestion
+ end
+ params do
+ requires :ids, type: Array[String], desc: "An array of suggestion ID's"
+ end
+ put 'batch_apply' do
+ ids = params[:ids]
+
+ suggestions = Suggestion.id_in(ids)
- result = ::Suggestions::ApplyService.new(current_user).execute(suggestion)
+ if suggestions.size == ids.length
+ apply_suggestions(suggestions, current_user)
+ else
+ render_api_error!(_('Suggestions are not applicable as one or more suggestions were not found.'), :not_found)
+ end
+ end
+ end
+
+ helpers do
+ def apply_suggestions(suggestions, current_user)
+ authorize_suggestions(*suggestions)
+
+ result = ::Suggestions::ApplyService.new(current_user, *suggestions).execute
if result[:status] == :success
- present suggestion, with: Entities::Suggestion, current_user: current_user
+ present suggestions, with: Entities::Suggestion, current_user: current_user
else
- http_status = result[:http_status] || 400
+ http_status = result[:http_status] || :bad_request
render_api_error!(result[:message], http_status)
end
end
+
+ def authorize_suggestions(*suggestions)
+ suggestions.each do |suggestion|
+ authorize! :apply_suggestion, suggestion
+ end
+ end
end
end
end
diff --git a/lib/api/terraform/state.rb b/lib/api/terraform/state.rb
index 5141d1fd499..e7c9627c753 100644
--- a/lib/api/terraform/state.rb
+++ b/lib/api/terraform/state.rb
@@ -32,7 +32,7 @@ module API
end
desc 'Get a terraform state by its name'
- route_setting :authentication, basic_auth_personal_access_token: true
+ route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
get do
remote_state_handler.find_with_lock do |state|
no_content! unless state.file.exists?
@@ -44,7 +44,7 @@ module API
end
desc 'Add a new terraform state or update an existing one'
- route_setting :authentication, basic_auth_personal_access_token: true
+ route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
post do
data = request.body.read
no_content! if data.empty?
@@ -57,7 +57,7 @@ module API
end
desc 'Delete a terraform state of a certain name'
- route_setting :authentication, basic_auth_personal_access_token: true
+ route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
delete do
remote_state_handler.handle_with_lock do |state|
state.destroy!
@@ -66,7 +66,7 @@ module API
end
desc 'Lock a terraform state of a certain name'
- route_setting :authentication, basic_auth_personal_access_token: true
+ route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
params do
requires :ID, type: String, limit: 255, desc: 'Terraform state lock ID'
requires :Operation, type: String, desc: 'Terraform operation'
@@ -103,7 +103,7 @@ module API
end
desc 'Unlock a terraform state of a certain name'
- route_setting :authentication, basic_auth_personal_access_token: true
+ route_setting :authentication, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth
params do
optional :ID, type: String, limit: 255, desc: 'Terraform state lock ID'
end
diff --git a/lib/api/todos.rb b/lib/api/todos.rb
index 02b8bb55274..e36ddf21277 100644
--- a/lib/api/todos.rb
+++ b/lib/api/todos.rb
@@ -6,8 +6,6 @@ module API
before { authenticate! }
- helpers ::Gitlab::IssuableMetadata
-
ISSUABLE_TYPES = {
'merge_requests' => ->(iid) { find_merge_request_with_access(iid) },
'issues' => ->(iid) { find_project_issue(iid) }
@@ -65,7 +63,7 @@ module API
next unless collection
targets = collection.map(&:target)
- options[type] = { issuable_metadata: issuable_meta_data(targets, type, current_user) }
+ options[type] = { issuable_metadata: Gitlab::IssuableMetadata.new(current_user, targets).data }
end
end
end
@@ -91,16 +89,18 @@ module API
requires :id, type: Integer, desc: 'The ID of the todo being marked as done'
end
post ':id/mark_as_done' do
- TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user)
todo = current_user.todos.find(params[:id])
+ TodoService.new.resolve_todo(todo, current_user, resolved_by_action: :api_done)
+
present todo, with: Entities::Todo, current_user: current_user
end
desc 'Mark all todos as done'
post '/mark_as_done' do
todos = find_todos
- TodoService.new.mark_todos_as_done(todos, current_user)
+
+ TodoService.new.resolve_todos(todos, current_user, resolved_by_action: :api_all_done)
no_content!
end
diff --git a/lib/api/users.rb b/lib/api/users.rb
index c986414c223..3d8ae09edf1 100644
--- a/lib/api/users.rb
+++ b/lib/api/users.rb
@@ -55,6 +55,7 @@ module API
optional :theme_id, type: Integer, desc: 'The GitLab theme for the user'
optional :color_scheme_id, type: Integer, desc: 'The color scheme for the file viewer'
optional :private_profile, type: Boolean, desc: 'Flag indicating the user has a private profile'
+ optional :note, type: String, desc: 'Admin note for this user'
all_or_none_of :extern_uid, :provider
use :optional_params_ee
@@ -254,6 +255,7 @@ module API
requires :id, type: Integer, desc: 'The ID of the user'
requires :key, type: String, desc: 'The new SSH key'
requires :title, type: String, desc: 'The title of the new SSH key'
+ optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)'
end
# rubocop: disable CodeReuse/ActiveRecord
post ":id/keys" do
@@ -262,9 +264,9 @@ module API
user = User.find_by(id: params.delete(:id))
not_found!('User') unless user
- key = user.keys.new(declared_params(include_missing: false))
+ key = ::Keys::CreateService.new(current_user, declared_params(include_missing: false).merge(user: user)).execute
- if key.save
+ if key.persisted?
present key, with: Entities::SSHKey
else
render_validation_error!(key)
@@ -283,7 +285,8 @@ module API
user = find_user(params[:user_id])
not_found!('User') unless user && can?(current_user, :read_user, user)
- present paginate(user.keys), with: Entities::SSHKey
+ keys = user.keys.preload_users
+ present paginate(keys), with: Entities::SSHKey
end
desc 'Delete an existing SSH key from a specified user. Available only for admins.' do
@@ -303,7 +306,10 @@ module API
key = user.keys.find_by(id: params[:key_id])
not_found!('Key') unless key
- destroy_conditionally!(key)
+ destroy_conditionally!(key) do |key|
+ destroy_service = ::Keys::DestroyService.new(current_user)
+ destroy_service.execute(key)
+ end
end
# rubocop: enable CodeReuse/ActiveRecord
@@ -695,7 +701,9 @@ module API
use :pagination
end
get "keys" do
- present paginate(current_user.keys), with: Entities::SSHKey
+ keys = current_user.keys.preload_users
+
+ present paginate(keys), with: Entities::SSHKey
end
desc 'Get a single key owned by currently authenticated user' do
@@ -719,6 +727,7 @@ module API
params do
requires :key, type: String, desc: 'The new SSH key'
requires :title, type: String, desc: 'The title of the new SSH key'
+ optional :expires_at, type: DateTime, desc: 'The expiration date of the SSH key in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ)'
end
post "keys" do
key = current_user.keys.new(declared_params)
diff --git a/lib/api/validations/validators/file_path.rb b/lib/api/validations/validators/file_path.rb
index 93a20e5bf7d..fee71373170 100644
--- a/lib/api/validations/validators/file_path.rb
+++ b/lib/api/validations/validators/file_path.rb
@@ -8,7 +8,7 @@ module API
path = params[attr_name]
Gitlab::Utils.check_path_traversal!(path)
- rescue StandardError
+ rescue ::Gitlab::Utils::PathTraversalAttackError
raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)],
message: "should be a valid file path"
end
diff --git a/lib/api/validations/validators/untrusted_regexp.rb b/lib/api/validations/validators/untrusted_regexp.rb
new file mode 100644
index 00000000000..ec623684e67
--- /dev/null
+++ b/lib/api/validations/validators/untrusted_regexp.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module API
+ module Validations
+ module Validators
+ class UntrustedRegexp < Grape::Validations::Base
+ def validate_param!(attr_name, params)
+ value = params[attr_name]
+ return unless value
+
+ Gitlab::UntrustedRegexp.new(value)
+ rescue RegexpError => e
+ message = "is an invalid regexp: #{e.message}"
+ raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message
+ end
+ end
+ end
+ end
+end
diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb
index 884e3019a2d..c1bf3a64923 100644
--- a/lib/api/wikis.rb
+++ b/lib/api/wikis.rb
@@ -24,7 +24,7 @@ module API
params :common_wiki_page_params do
optional :format,
type: String,
- values: ProjectWiki::MARKUPS.values.map(&:to_s),
+ values: Wiki::MARKUPS.values.map(&:to_s),
default: 'markdown',
desc: 'Format of a wiki page. Available formats are markdown, rdoc, asciidoc and org'
end
diff --git a/lib/backup/files.rb b/lib/backup/files.rb
index 098f2da6d88..5e784dadb14 100644
--- a/lib/backup/files.rb
+++ b/lib/backup/files.rb
@@ -78,7 +78,8 @@ module Backup
return if status.compact.all?(&:success?)
regex = /^g?tar: \.: Cannot mkdir: No such file or directory$/
- raise Backup::Error, 'Backup failed' unless err_r.read =~ regex
+ error = err_r.read
+ raise Backup::Error, "Backup failed. #{error}" unless error =~ regex
end
end
end
diff --git a/lib/banzai/filter/ascii_doc_sanitization_filter.rb b/lib/banzai/filter/ascii_doc_sanitization_filter.rb
index e41f7d8488a..a1a204ec652 100644
--- a/lib/banzai/filter/ascii_doc_sanitization_filter.rb
+++ b/lib/banzai/filter/ascii_doc_sanitization_filter.rb
@@ -18,6 +18,7 @@ module Banzai
# Classes used by Asciidoctor to style components
ADMONITION_CLASSES = %w(fa icon-note icon-tip icon-warning icon-caution icon-important).freeze
+ ALIGNMENT_BUILTINS_CLASSES = %w(text-center text-left text-right text-justify).freeze
CALLOUT_CLASSES = ['conum'].freeze
CHECKLIST_CLASSES = %w(fa fa-check-square-o fa-square-o).freeze
LIST_CLASSES = %w(checklist none no-bullet unnumbered unstyled).freeze
@@ -28,7 +29,7 @@ module Banzai
ELEMENT_CLASSES_WHITELIST = {
span: %w(big small underline overline line-through).freeze,
- div: ['admonitionblock'].freeze,
+ div: ALIGNMENT_BUILTINS_CLASSES + ['admonitionblock'].freeze,
td: ['icon'].freeze,
i: ADMONITION_CLASSES + CALLOUT_CLASSES + CHECKLIST_CLASSES,
ul: LIST_CLASSES,
diff --git a/lib/banzai/filter/design_reference_filter.rb b/lib/banzai/filter/design_reference_filter.rb
new file mode 100644
index 00000000000..7455dfe00ef
--- /dev/null
+++ b/lib/banzai/filter/design_reference_filter.rb
@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ class DesignReferenceFilter < AbstractReferenceFilter
+ FEATURE_FLAG = :design_management_reference_filter_gfm_pipeline
+
+ class Identifier
+ include Comparable
+ attr_reader :issue_iid, :filename
+
+ def initialize(issue_iid:, filename:)
+ @issue_iid = issue_iid
+ @filename = filename
+ end
+
+ def as_composite_id(id_for_iid)
+ id = id_for_iid[issue_iid]
+ return unless id
+
+ { issue_id: id, filename: filename }
+ end
+
+ def <=>(other)
+ return unless other.is_a?(Identifier)
+
+ [issue_iid, filename] <=> [other.issue_iid, other.filename]
+ end
+ alias_method :eql?, :==
+
+ def hash
+ [issue_iid, filename].hash
+ end
+ end
+
+ self.reference_type = :design
+
+ # This filter must be enabled by setting the
+ # design_management_reference_filter_gfm_pipeline flag
+ def call
+ return doc unless enabled?
+
+ super
+ end
+
+ def find_object(project, identifier)
+ records_per_parent[project][identifier]
+ end
+
+ def parent_records(project, identifiers)
+ return [] unless project.design_management_enabled?
+
+ iids = identifiers.map(&:issue_iid).to_set
+ issues = project.issues.where(iid: iids)
+ id_for_iid = issues.index_by(&:iid).transform_values(&:id)
+ issue_by_id = issues.index_by(&:id)
+
+ designs(identifiers, id_for_iid).each do |d|
+ issue = issue_by_id[d.issue_id]
+ # optimisation: assign values we have already fetched
+ d.project = project
+ d.issue = issue
+ end
+ end
+
+ def relation_for_paths(paths)
+ super.includes(:route, :namespace, :group)
+ end
+
+ def parent_type
+ :project
+ end
+
+ # optimisation to reuse the parent_per_reference query information
+ def parent_from_ref(ref)
+ parent_per_reference[ref || current_parent_path]
+ end
+
+ def url_for_object(design, project)
+ path_options = { vueroute: design.filename }
+ Gitlab::Routing.url_helpers.designs_project_issue_path(project, design.issue, path_options)
+ end
+
+ def data_attributes_for(_text, _project, design, **_kwargs)
+ super.merge(issue: design.issue_id)
+ end
+
+ def self.object_class
+ ::DesignManagement::Design
+ end
+
+ def self.object_sym
+ :design
+ end
+
+ def self.parse_symbol(raw, match_data)
+ filename = match_data[:url_filename]
+ iid = match_data[:issue].to_i
+ Identifier.new(filename: CGI.unescape(filename), issue_iid: iid)
+ end
+
+ def record_identifier(design)
+ Identifier.new(filename: design.filename, issue_iid: design.issue.iid)
+ end
+
+ private
+
+ def designs(identifiers, id_for_iid)
+ identifiers
+ .map { |identifier| identifier.as_composite_id(id_for_iid) }
+ .compact
+ .in_groups_of(100, false) # limitation of by_issue_id_and_filename, so we batch
+ .flat_map { |ids| DesignManagement::Design.by_issue_id_and_filename(ids) }
+ end
+
+ def enabled?
+ Feature.enabled?(FEATURE_FLAG, parent)
+ end
+ end
+ end
+end
diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb
index 8159dcfed72..74bc102320c 100644
--- a/lib/banzai/filter/external_issue_reference_filter.rb
+++ b/lib/banzai/filter/external_issue_reference_filter.rb
@@ -54,6 +54,8 @@ module Banzai
doc
end
+ private
+
# Replace `JIRA-123` issue references in text with links to the referenced
# issue's details page.
#
@@ -63,27 +65,31 @@ module Banzai
# Returns a String with `JIRA-123` references replaced with links. All
# links have `gfm` and `gfm-issue` class names attached for styling.
def issue_link_filter(text, link_content: nil)
- project = context[:project]
-
self.class.references_in(text, issue_reference_pattern) do |match, id|
- ExternalIssue.new(id, project)
-
- url = url_for_issue(id, project, only_path: context[:only_path])
-
- title = "Issue in #{project.external_issue_tracker.title}"
+ url = url_for_issue(id)
klass = reference_class(:issue)
data = data_attribute(project: project.id, external_issue: id)
-
content = link_content || match
%(<a href="#{url}" #{data}
- title="#{escape_once(title)}"
+ title="#{escape_once(issue_title)}"
class="#{klass}">#{content}</a>)
end
end
- def url_for_issue(*args)
- IssuesHelper.url_for_issue(*args)
+ def url_for_issue(issue_id)
+ return '' if project.nil?
+
+ url = if only_path?
+ project.external_issue_tracker.issue_path(issue_id)
+ else
+ project.external_issue_tracker.issue_url(issue_id)
+ end
+
+ # Ensure we return a valid URL to prevent possible XSS.
+ URI.parse(url).to_s
+ rescue URI::InvalidURIError
+ ''
end
def default_issues_tracker?
@@ -94,7 +100,13 @@ module Banzai
external_issues_cached(:external_issue_reference_pattern)
end
- private
+ def project
+ context[:project]
+ end
+
+ def issue_title
+ "Issue in #{project.external_issue_tracker.title}"
+ end
def external_issues_cached(attribute)
cached_attributes = Gitlab::SafeRequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} }
diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb
index dec4ec871f1..033e3d2c33e 100644
--- a/lib/banzai/filter/gollum_tags_filter.rb
+++ b/lib/banzai/filter/gollum_tags_filter.rb
@@ -30,7 +30,7 @@ module Banzai
# Note: the table of contents tag is now handled by TableOfContentsTagFilter
#
# Context options:
- # :project_wiki (required) - Current project wiki.
+ # :wiki [Wiki] (required) - Current wiki instance.
#
class GollumTagsFilter < HTML::Pipeline::Filter
include ActionView::Helpers::TagHelper
@@ -100,8 +100,8 @@ module Banzai
if url?(content)
path = content
- elsif file = project_wiki.find_file(content)
- path = ::File.join project_wiki_base_path, file.path
+ elsif file = wiki.find_file(content)
+ path = ::File.join(wiki_base_path, file.path)
end
if path
@@ -134,25 +134,25 @@ module Banzai
if url?(reference)
reference
else
- ::File.join(project_wiki_base_path, reference)
+ ::File.join(wiki_base_path, reference)
end
content_tag(:a, name || reference, href: href, class: 'gfm')
end
- def project_wiki
- context[:project_wiki]
+ def wiki
+ context[:wiki]
end
- def project_wiki_base_path
- project_wiki && project_wiki.wiki_base_path
+ def wiki_base_path
+ wiki&.wiki_base_path
end
- # Ensure that a :project_wiki key exists in context
+ # Ensure that a :wiki key exists in context
#
# Note that while the key might exist, its value could be nil!
def validate
- needs :project_wiki
+ needs :wiki
end
end
end
diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb
index 37e66387f2e..dc4b865bfb6 100644
--- a/lib/banzai/filter/issue_reference_filter.rb
+++ b/lib/banzai/filter/issue_reference_filter.rb
@@ -18,7 +18,9 @@ module Banzai
end
def url_for_object(issue, project)
- IssuesHelper.url_for_issue(issue.iid, project, only_path: context[:only_path], internal: true)
+ return issue_path(issue, project) if only_path?
+
+ issue_url(issue, project)
end
def projects_relation_for_paths(paths)
@@ -35,6 +37,14 @@ module Banzai
private
+ def issue_path(issue, project)
+ Gitlab::Routing.url_helpers.namespace_project_issue_path(namespace_id: project.namespace, project_id: project, id: issue.iid)
+ end
+
+ def issue_url(issue, project)
+ Gitlab::Routing.url_helpers.namespace_project_issue_url(namespace_id: project.namespace, project_id: project, id: issue.iid)
+ end
+
def design_link_extras(issue, path)
if path == '/designs' && read_designs?(issue)
['designs']
@@ -44,7 +54,7 @@ module Banzai
end
def read_designs?(issue)
- Ability.allowed?(current_user, :read_design, issue)
+ issue.project.design_management_enabled?
end
end
end
diff --git a/lib/banzai/filter/iteration_reference_filter.rb b/lib/banzai/filter/iteration_reference_filter.rb
new file mode 100644
index 00000000000..9d2b533e6da
--- /dev/null
+++ b/lib/banzai/filter/iteration_reference_filter.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Banzai
+ module Filter
+ # The actual filter is implemented in the EE mixin
+ class IterationReferenceFilter < AbstractReferenceFilter
+ self.reference_type = :iteration
+
+ def self.object_class
+ Iteration
+ end
+ end
+ end
+end
+
+Banzai::Filter::IterationReferenceFilter.prepend_if_ee('EE::Banzai::Filter::IterationReferenceFilter')
diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb
index 60ffb178393..7cda4699ae6 100644
--- a/lib/banzai/filter/label_reference_filter.rb
+++ b/lib/banzai/filter/label_reference_filter.rb
@@ -71,13 +71,16 @@ module Banzai
end
def url_for_object(label, parent)
- h = Gitlab::Routing.url_helpers
+ label_url_method =
+ if context[:label_url_method]
+ context[:label_url_method]
+ elsif parent.is_a?(Project)
+ :project_issues_url
+ end
- if parent.is_a?(Project)
- h.project_issues_url(parent, label_name: label.name, only_path: context[:only_path])
- elsif context[:label_url_method]
- h.public_send(context[:label_url_method], parent, label_name: label.name, only_path: context[:only_path]) # rubocop:disable GitlabSecurity/PublicSend
- end
+ return unless label_url_method
+
+ Gitlab::Routing.url_helpers.public_send(label_url_method, parent, label_name: label.name, only_path: context[:only_path]) # rubocop:disable GitlabSecurity/PublicSend
end
def object_link_text(object, matches)
diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb
index 38bbed3cf72..9e932ccf9f8 100644
--- a/lib/banzai/filter/reference_filter.rb
+++ b/lib/banzai/filter/reference_filter.rb
@@ -142,6 +142,12 @@ module Banzai
def element_node?(node)
node.is_a?(Nokogiri::XML::Element)
end
+
+ private
+
+ def only_path?
+ context[:only_path]
+ end
end
end
end
diff --git a/lib/banzai/filter/repository_link_filter.rb b/lib/banzai/filter/repository_link_filter.rb
index 24900217560..66b9aac3e7e 100644
--- a/lib/banzai/filter/repository_link_filter.rb
+++ b/lib/banzai/filter/repository_link_filter.rb
@@ -10,7 +10,7 @@ module Banzai
# :commit
# :current_user
# :project
- # :project_wiki
+ # :wiki
# :ref
# :requested_path
# :system_note
@@ -53,7 +53,7 @@ module Banzai
def linkable_files?
strong_memoize(:linkable_files) do
- context[:project_wiki].nil? && repository.try(:exists?) && !repository.empty?
+ context[:wiki].nil? && repository.try(:exists?) && !repository.empty?
end
end
diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb
index 205f777bc90..44f13612fde 100644
--- a/lib/banzai/filter/wiki_link_filter.rb
+++ b/lib/banzai/filter/wiki_link_filter.rb
@@ -6,12 +6,12 @@ module Banzai
# Rewrite rules are documented in the `WikiPipeline` spec.
#
# Context options:
- # :project_wiki
+ # :wiki
class WikiLinkFilter < HTML::Pipeline::Filter
include Gitlab::Utils::SanitizeNodeLink
def call
- return doc unless project_wiki?
+ return doc unless wiki?
doc.search('a:not(.gfm)').each { |el| process_link(el.attribute('href'), el) }
@@ -33,8 +33,8 @@ module Banzai
remove_unsafe_links({ node: node }, remove_invalid_links: false)
end
- def project_wiki?
- !context[:project_wiki].nil?
+ def wiki?
+ !context[:wiki].nil?
end
def process_link_attr(html_attr)
@@ -46,7 +46,7 @@ module Banzai
end
def apply_rewrite_rules(link_string)
- Rewriter.new(link_string, wiki: context[:project_wiki], slug: context[:page_slug]).apply_rules
+ Rewriter.new(link_string, wiki: context[:wiki], slug: context[:page_slug]).apply_rules
end
end
end
diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb
index 329bbb270bd..2ea5fd3388a 100644
--- a/lib/banzai/pipeline/gfm_pipeline.rb
+++ b/lib/banzai/pipeline/gfm_pipeline.rb
@@ -56,6 +56,7 @@ module Banzai
[
Filter::UserReferenceFilter,
Filter::ProjectReferenceFilter,
+ Filter::DesignReferenceFilter,
Filter::IssueReferenceFilter,
Filter::ExternalIssueReferenceFilter,
Filter::MergeRequestReferenceFilter,
diff --git a/lib/banzai/reference_parser/iteration_parser.rb b/lib/banzai/reference_parser/iteration_parser.rb
new file mode 100644
index 00000000000..45253fa1977
--- /dev/null
+++ b/lib/banzai/reference_parser/iteration_parser.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Banzai
+ module ReferenceParser
+ # The actual parser is implemented in the EE mixin
+ class IterationParser < BaseParser
+ self.reference_type = :iteration
+
+ def references_relation
+ Iteration
+ end
+
+ private
+
+ def can_read_reference?(_user, _ref_project, _node)
+ false
+ end
+ end
+ end
+end
+
+Banzai::ReferenceParser::IterationParser.prepend_if_ee('::EE::Banzai::ReferenceParser::IterationParser')
diff --git a/lib/event_filter.rb b/lib/event_filter.rb
index 8cb0b1441df..538727dc422 100644
--- a/lib/event_filter.rb
+++ b/lib/event_filter.rb
@@ -27,15 +27,15 @@ class EventFilter
case filter
when PUSH
- events.where(action: Event::PUSHED)
+ events.pushed_action
when MERGED
- events.where(action: Event::MERGED)
+ events.merged_action
when COMMENTS
- events.where(action: Event::COMMENTED)
+ events.commented_action
when TEAM
- events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED])
+ events.where(action: [:joined, :left, :expired])
when ISSUE
- events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED], target_type: 'Issue')
+ events.where(action: [:created, :updated, :closed, :reopened], target_type: 'Issue')
when WIKI
wiki_events(events)
else
diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb
index 44a9c7ea536..4c537eeaa89 100644
--- a/lib/extracts_path.rb
+++ b/lib/extracts_path.rb
@@ -3,79 +3,8 @@
# Module providing methods for dealing with separating a tree-ish string and a
# file path string when combined in a request parameter
module ExtractsPath
- # Raised when given an invalid file path
- InvalidPathError = Class.new(StandardError)
-
- # Given a string containing both a Git tree-ish, such as a branch or tag, and
- # a filesystem path joined by forward slashes, attempts to separate the two.
- #
- # Expects a @project instance variable to contain the active project. This is
- # used to check the input against a list of valid repository refs.
- #
- # Examples
- #
- # # No @project available
- # extract_ref('master')
- # # => ['', '']
- #
- # extract_ref('master')
- # # => ['master', '']
- #
- # extract_ref("f4b14494ef6abf3d144c28e4af0c20143383e062/CHANGELOG")
- # # => ['f4b14494ef6abf3d144c28e4af0c20143383e062', 'CHANGELOG']
- #
- # extract_ref("v2.0.0/README.md")
- # # => ['v2.0.0', 'README.md']
- #
- # extract_ref('master/app/models/project.rb')
- # # => ['master', 'app/models/project.rb']
- #
- # extract_ref('issues/1234/app/models/project.rb')
- # # => ['issues/1234', 'app/models/project.rb']
- #
- # # Given an invalid branch, we fall back to just splitting on the first slash
- # extract_ref('non/existent/branch/README.md')
- # # => ['non', 'existent/branch/README.md']
- #
- # Returns an Array where the first value is the tree-ish and the second is the
- # path
- def extract_ref(id)
- pair = ['', '']
-
- return pair unless @project # rubocop:disable Gitlab/ModuleWithInstanceVariables
-
- if id =~ /^(\h{40})(.+)/
- # If the ref appears to be a SHA, we're done, just split the string
- pair = $~.captures
- else
- # Otherwise, attempt to detect the ref using a list of the project's
- # branches and tags
-
- # Append a trailing slash if we only get a ref and no file path
- unless id.ends_with?('/')
- id = [id, '/'].join
- end
-
- valid_refs = ref_names.select { |v| id.start_with?("#{v}/") }
-
- if valid_refs.empty?
- # No exact ref match, so just try our best
- pair = id.match(%r{([^/]+)(.*)}).captures
- else
- # There is a distinct possibility that multiple refs prefix the ID.
- # Use the longest match to maximize the chance that we have the
- # right ref.
- best_match = valid_refs.max_by(&:length)
- # Partition the string into the ref and the path, ignoring the empty first value
- pair = id.partition(best_match)[1..-1]
- end
- end
-
- # Remove ending slashes from path
- pair[1].gsub!(%r{^/|/$}, '')
-
- pair
- end
+ extend ::Gitlab::Utils::Override
+ include ExtractsRef
# If we have an ID of 'foo.atom', and the controller provides Atom and HTML
# formats, then we have to check if the request was for the Atom version of
@@ -90,34 +19,17 @@ module ExtractsPath
valid_refs.max_by(&:length)
end
- # Assigns common instance variables for views working with Git tree-ish objects
- #
- # Assignments are:
- #
- # - @id - A string representing the joined ref and path
- # - @ref - A string representing the ref (e.g., the branch, tag, or commit SHA)
- # - @path - A string representing the filesystem path
- # - @commit - A Commit representing the commit from the given ref
- #
- # If the :id parameter appears to be requesting a specific response format,
- # that will be handled as well.
- #
- # If there is no path and the ref doesn't exist in the repo, try to resolve
- # the ref without an '.atom' suffix. If _that_ ref is found, set the request's
- # format to Atom manually.
+ # Extends the method to handle if there is no path and the ref doesn't
+ # exist in the repo, try to resolve the ref without an '.atom' suffix.
+ # If _that_ ref is found, set the request's format to Atom manually.
#
# Automatically renders `not_found!` if a valid tree path could not be
# resolved (e.g., when a user inserts an invalid path or ref).
+ #
# rubocop:disable Gitlab/ModuleWithInstanceVariables
+ override :assign_ref_vars
def assign_ref_vars
- @id = get_id
- @ref, @path = extract_ref(@id)
- @repo = @project.repository
- @ref.strip!
-
- raise InvalidPathError if @ref.match?(/\s/)
-
- @commit = @repo.commit(@ref)
+ super
if @path.empty? && !@commit && @id.ends_with?('.atom')
@id = @ref = extract_ref_without_atom(@id)
@@ -135,10 +47,6 @@ module ExtractsPath
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
- def tree
- @tree ||= @repo.tree(@commit.id, @path) # rubocop:disable Gitlab/ModuleWithInstanceVariables
- end
-
def lfs_blob_ids
blob_ids = tree.blobs.map(&:id)
@@ -146,21 +54,13 @@ module ExtractsPath
# the current Blob in order to determine if it's a LFS object
blob_ids = Array.wrap(@repo.blob_at(@commit.id, @path)&.id) if blob_ids.empty? # rubocop:disable Gitlab/ModuleWithInstanceVariables
- @lfs_blob_ids = Gitlab::Git::Blob.batch_lfs_pointers(@project.repository, blob_ids).map(&:id) # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ @lfs_blob_ids = Gitlab::Git::Blob.batch_lfs_pointers(repository_container.repository, blob_ids).map(&:id) # rubocop:disable Gitlab/ModuleWithInstanceVariables
end
private
- # overridden in subclasses, do not remove
- def get_id
- id = [params[:id] || params[:ref]]
- id << "/" + params[:path] unless params[:path].blank?
- id.join
- end
-
- def ref_names
- return [] unless @project # rubocop:disable Gitlab/ModuleWithInstanceVariables
-
- @ref_names ||= @project.repository.ref_names # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ override :repository_container
+ def repository_container
+ @project
end
end
diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb
new file mode 100644
index 00000000000..346ed6b6f60
--- /dev/null
+++ b/lib/extracts_ref.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+# Module providing methods for dealing with separating a tree-ish string and a
+# file path string when combined in a request parameter
+# Can be extended for different types of repository object, e.g. Project or Snippet
+module ExtractsRef
+ InvalidPathError = Class.new(StandardError)
+
+ # Given a string containing both a Git tree-ish, such as a branch or tag, and
+ # a filesystem path joined by forward slashes, attempts to separate the two.
+ #
+ # Expects a repository_container method that returns the active repository object. This is
+ # used to check the input against a list of valid repository refs.
+ #
+ # Examples
+ #
+ # # No repository_container available
+ # extract_ref('master')
+ # # => ['', '']
+ #
+ # extract_ref('master')
+ # # => ['master', '']
+ #
+ # extract_ref("f4b14494ef6abf3d144c28e4af0c20143383e062/CHANGELOG")
+ # # => ['f4b14494ef6abf3d144c28e4af0c20143383e062', 'CHANGELOG']
+ #
+ # extract_ref("v2.0.0/README.md")
+ # # => ['v2.0.0', 'README.md']
+ #
+ # extract_ref('master/app/models/project.rb')
+ # # => ['master', 'app/models/project.rb']
+ #
+ # extract_ref('issues/1234/app/models/project.rb')
+ # # => ['issues/1234', 'app/models/project.rb']
+ #
+ # # Given an invalid branch, we fall back to just splitting on the first slash
+ # extract_ref('non/existent/branch/README.md')
+ # # => ['non', 'existent/branch/README.md']
+ #
+ # Returns an Array where the first value is the tree-ish and the second is the
+ # path
+ def extract_ref(id)
+ pair = ['', '']
+
+ return pair unless repository_container
+
+ if id =~ /^(\h{40})(.+)/
+ # If the ref appears to be a SHA, we're done, just split the string
+ pair = $~.captures
+ else
+ # Otherwise, attempt to detect the ref using a list of the repository_container's
+ # branches and tags
+
+ # Append a trailing slash if we only get a ref and no file path
+ unless id.ends_with?('/')
+ id = [id, '/'].join
+ end
+
+ valid_refs = ref_names.select { |v| id.start_with?("#{v}/") }
+
+ if valid_refs.empty?
+ # No exact ref match, so just try our best
+ pair = id.match(%r{([^/]+)(.*)}).captures
+ else
+ # There is a distinct possibility that multiple refs prefix the ID.
+ # Use the longest match to maximize the chance that we have the
+ # right ref.
+ best_match = valid_refs.max_by(&:length)
+ # Partition the string into the ref and the path, ignoring the empty first value
+ pair = id.partition(best_match)[1..-1]
+ end
+ end
+
+ pair[0] = pair[0].strip
+
+ # Remove ending slashes from path
+ pair[1].gsub!(%r{^/|/$}, '')
+
+ pair
+ end
+
+ # Assigns common instance variables for views working with Git tree-ish objects
+ #
+ # Assignments are:
+ #
+ # - @id - A string representing the joined ref and path
+ # - @ref - A string representing the ref (e.g., the branch, tag, or commit SHA)
+ # - @path - A string representing the filesystem path
+ # - @commit - A Commit representing the commit from the given ref
+ #
+ # If the :id parameter appears to be requesting a specific response format,
+ # that will be handled as well.
+ #
+ # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ def assign_ref_vars
+ @id = get_id
+ @ref, @path = extract_ref(@id)
+ @repo = repository_container.repository
+
+ raise InvalidPathError if @ref.match?(/\s/)
+
+ @commit = @repo.commit(@ref)
+ end
+ # rubocop:enable Gitlab/ModuleWithInstanceVariables
+
+ def tree
+ @tree ||= @repo.tree(@commit.id, @path) # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ end
+
+ private
+
+ # overridden in subclasses, do not remove
+ def get_id
+ id = [params[:id] || params[:ref]]
+ id << "/" + params[:path] unless params[:path].blank?
+ id.join
+ end
+
+ def ref_names
+ return [] unless repository_container
+
+ @ref_names ||= repository_container.repository.ref_names # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ end
+
+ def repository_container
+ raise NotImplementedError
+ end
+end
diff --git a/lib/feature.rb b/lib/feature.rb
index dc7e8da8f35..d995e0a988f 100644
--- a/lib/feature.rb
+++ b/lib/feature.rb
@@ -4,8 +4,6 @@ require 'flipper/adapters/active_record'
require 'flipper/adapters/active_support_cache_store'
class Feature
- prepend_if_ee('EE::Feature') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
# Classes to override flipper table names
class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature
# Using `self.table_name` won't work. ActiveRecord bug?
@@ -20,6 +18,8 @@ class Feature
superclass.table_name = 'feature_gates'
end
+ InvalidFeatureFlagError = Class.new(Exception) # rubocop:disable Lint/InheritException
+
class << self
delegate :group, to: :flipper
@@ -34,37 +34,58 @@ class Feature
def persisted_names
return [] unless Gitlab::Database.exists?
- Gitlab::SafeRequestStore[:flipper_persisted_names] ||=
- begin
- # We saw on GitLab.com, this database request was called 2300
- # times/s. Let's cache it for a minute to avoid that load.
- Gitlab::ProcessMemoryCache.cache_backend.fetch('flipper:persisted_names', expires_in: 1.minute) do
- FlipperFeature.feature_names
+ if Gitlab::Utils.to_boolean(ENV['FF_LEGACY_PERSISTED_NAMES'])
+ # To be removed:
+ # This uses a legacy persisted names that are know to work (always)
+ Gitlab::SafeRequestStore[:flipper_persisted_names] ||=
+ begin
+ # We saw on GitLab.com, this database request was called 2300
+ # times/s. Let's cache it for a minute to avoid that load.
+ Gitlab::ProcessMemoryCache.cache_backend.fetch('flipper:persisted_names', expires_in: 1.minute) do
+ FlipperFeature.feature_names
+ end.to_set
end
- end
+ else
+ # This loads names of all stored feature flags
+ # and returns a stable Set in the following order:
+ # - Memoized: using Gitlab::SafeRequestStore or @flipper
+ # - L1: using Process cache
+ # - L2: using Redis cache
+ # - DB: using a single SQL query
+ flipper.adapter.features
+ end
end
- def persisted?(feature)
+ def persisted_name?(feature_name)
# Flipper creates on-memory features when asked for a not-yet-created one.
# If we want to check if a feature has been actually set, we look for it
# on the persisted features list.
- persisted_names.include?(feature.name.to_s)
+ persisted_names.include?(feature_name.to_s)
end
# use `default_enabled: true` to default the flag to being `enabled`
# unless set explicitly. The default is `disabled`
+ # TODO: remove the `default_enabled:` and read it from the `defintion_yaml`
+ # check: https://gitlab.com/gitlab-org/gitlab/-/issues/30228
def enabled?(key, thing = nil, default_enabled: false)
+ if check_feature_flags_definition?
+ if thing && !thing.respond_to?(:flipper_id)
+ raise InvalidFeatureFlagError,
+ "The thing '#{thing.class.name}' for feature flag '#{key}' needs to include `FeatureGate` or implement `flipper_id`"
+ end
+ end
+
# During setup the database does not exist yet. So we haven't stored a value
# for the feature yet and return the default.
return default_enabled unless Gitlab::Database.exists?
- feature = Feature.get(key)
+ feature = get(key)
# If we're not default enabling the flag or the feature has been set, always evaluate.
# `persisted?` can potentially generate DB queries and also checks for inclusion
# in an array of feature names (177 at last count), possibly reducing performance by half.
# So we only perform the `persisted` check if `default_enabled: true`
- !default_enabled || Feature.persisted?(feature) ? feature.enabled?(thing) : true
+ !default_enabled || Feature.persisted_name?(feature.name) ? feature.enabled?(thing) : true
end
def disabled?(key, thing = nil, default_enabled: false)
@@ -88,23 +109,31 @@ class Feature
get(key).disable_group(group)
end
- def remove(key)
- feature = get(key)
- return unless persisted?(feature)
+ def enable_percentage_of_time(key, percentage)
+ get(key).enable_percentage_of_time(percentage)
+ end
- feature.remove
+ def disable_percentage_of_time(key)
+ get(key).disable_percentage_of_time
end
- def flipper
- if Gitlab::SafeRequestStore.active?
- Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance
- else
- @flipper ||= build_flipper_instance
- end
+ def enable_percentage_of_actors(key, percentage)
+ get(key).enable_percentage_of_actors(percentage)
end
- def build_flipper_instance
- Flipper.new(flipper_adapter).tap { |flip| flip.memoize = true }
+ def disable_percentage_of_actors(key)
+ get(key).disable_percentage_of_actors
+ end
+
+ def remove(key)
+ return unless persisted_name?(key)
+
+ get(key).remove
+ end
+
+ def reset
+ Gitlab::SafeRequestStore.delete(:flipper) if Gitlab::SafeRequestStore.active?
+ @flipper = nil
end
# This method is called from config/initializers/flipper.rb and can be used
@@ -113,7 +142,17 @@ class Feature
def register_feature_groups
end
- def flipper_adapter
+ private
+
+ def flipper
+ if Gitlab::SafeRequestStore.active?
+ Gitlab::SafeRequestStore[:flipper] ||= build_flipper_instance
+ else
+ @flipper ||= build_flipper_instance
+ end
+ end
+
+ def build_flipper_instance
active_record_adapter = Flipper::Adapters::ActiveRecord.new(
feature_class: FlipperFeature,
gate_class: FlipperGate)
@@ -127,10 +166,20 @@ class Feature
# Thread-local L1 cache: use a short timeout since we don't have a
# way to expire this cache all at once
- Flipper::Adapters::ActiveSupportCacheStore.new(
+ flipper_adapter = Flipper::Adapters::ActiveSupportCacheStore.new(
redis_cache_adapter,
l1_cache_backend,
expires_in: 1.minute)
+
+ Flipper.new(flipper_adapter).tap do |flip|
+ flip.memoize = true
+ end
+ end
+
+ def check_feature_flags_definition?
+ # We want to check feature flags usage only when
+ # running in development or test environment
+ Gitlab.dev_or_test_env?
end
def l1_cache_backend
@@ -186,3 +235,5 @@ class Feature
end
end
end
+
+Feature.prepend_if_ee('EE::Feature')
diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb
index 89a836e629f..a816dd89e9c 100644
--- a/lib/gitaly/server.rb
+++ b/lib/gitaly/server.rb
@@ -3,6 +3,7 @@
module Gitaly
class Server
SHA_VERSION_REGEX = /\A\d+\.\d+\.\d+-\d+-g([a-f0-9]{8})\z/.freeze
+ DEFAULT_REPLICATION_FACTOR = 1
class << self
def all
@@ -16,6 +17,10 @@ module Gitaly
def filesystems
all.map(&:filesystem_type).compact.uniq
end
+
+ def gitaly_clusters
+ all.count { |g| g.replication_factor > DEFAULT_REPLICATION_FACTOR }
+ end
end
attr_reader :storage
@@ -73,6 +78,10 @@ module Gitaly
"Error getting the address: #{e.message}"
end
+ def replication_factor
+ storage_status&.replication_factor
+ end
+
private
def storage_status
diff --git a/lib/gitlab.rb b/lib/gitlab.rb
index f2bff51df38..43785d165fb 100644
--- a/lib/gitlab.rb
+++ b/lib/gitlab.rb
@@ -36,6 +36,7 @@ module Gitlab
end
COM_URL = 'https://gitlab.com'
+ STAGING_COM_URL = 'https://staging.gitlab.com'
APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}.freeze
SUBDOMAIN_REGEX = %r{\Ahttps://[a-z0-9]+\.gitlab\.com\z}.freeze
VERSION = File.read(root.join("VERSION")).strip.freeze
@@ -47,6 +48,10 @@ module Gitlab
Gitlab.config.gitlab.url == COM_URL || gl_subdomain?
end
+ def self.staging?
+ Gitlab.config.gitlab.url == STAGING_COM_URL
+ end
+
def self.canary?
Gitlab::Utils.to_boolean(ENV['CANARY'])
end
@@ -75,6 +80,10 @@ module Gitlab
Rails.env.development? || com?
end
+ def self.dev_or_test_env?
+ Rails.env.development? || Rails.env.test?
+ end
+
def self.ee?
@is_ee ||=
# We use this method when the Rails environment is not loaded. This
diff --git a/lib/gitlab/alert_management/alert_params.rb b/lib/gitlab/alert_management/alert_params.rb
index 982479784a9..789a4fe246a 100644
--- a/lib/gitlab/alert_management/alert_params.rb
+++ b/lib/gitlab/alert_management/alert_params.rb
@@ -20,7 +20,8 @@ module Gitlab
hosts: Array(annotations[:hosts]),
payload: payload,
started_at: parsed_payload['startsAt'],
- severity: annotations[:severity]
+ severity: annotations[:severity],
+ fingerprint: annotations[:fingerprint]
}
end
diff --git a/lib/gitlab/alert_management/fingerprint.rb b/lib/gitlab/alert_management/fingerprint.rb
new file mode 100644
index 00000000000..6ab47c88ca1
--- /dev/null
+++ b/lib/gitlab/alert_management/fingerprint.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module AlertManagement
+ class Fingerprint
+ def self.generate(data)
+ new.generate(data)
+ end
+
+ def generate(data)
+ return unless data.present?
+
+ if data.is_a?(Array)
+ data = flatten_array(data)
+ end
+
+ Digest::SHA1.hexdigest(data.to_s)
+ end
+
+ private
+
+ def flatten_array(array)
+ array.flatten.map!(&:to_s).join
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb
index d859ca89418..dad3dabb4fc 100644
--- a/lib/gitlab/alerting/alert.rb
+++ b/lib/gitlab/alerting/alert.rb
@@ -106,7 +106,7 @@ module Gitlab
end
def gitlab_fingerprint
- Digest::SHA1.hexdigest(plain_gitlab_fingerprint)
+ Gitlab::AlertManagement::Fingerprint.generate(plain_gitlab_fingerprint)
end
def valid?
@@ -121,9 +121,9 @@ module Gitlab
def plain_gitlab_fingerprint
if gitlab_managed?
- [metric_id, starts_at].join('/')
+ [metric_id, starts_at_raw].join('/')
else # self managed
- [starts_at, title, full_query].join('/')
+ [starts_at_raw, title, full_query].join('/')
end
end
@@ -173,7 +173,10 @@ module Gitlab
value = payload&.dig(field)
return unless value
- Time.rfc3339(value)
+ # value is a rfc3339 timestamp
+ # Timestamps from Prometheus and Alertmanager are UTC RFC3339 timestamps like: '2018-03-12T09:06:00Z' (Z represents 0 offset or UTC)
+ # .utc sets the datetime zone to `UTC`
+ Time.rfc3339(value).utc
rescue ArgumentError
end
diff --git a/lib/gitlab/alerting/notification_payload_parser.rb b/lib/gitlab/alerting/notification_payload_parser.rb
index c79d69613f3..d98b9296347 100644
--- a/lib/gitlab/alerting/notification_payload_parser.rb
+++ b/lib/gitlab/alerting/notification_payload_parser.rb
@@ -35,6 +35,10 @@ module Gitlab
payload[:severity].presence || DEFAULT_SEVERITY
end
+ def fingerprint
+ Gitlab::AlertManagement::Fingerprint.generate(payload[:fingerprint])
+ end
+
def annotations
primary_params
.reverse_merge(flatten_secondary_params)
@@ -49,7 +53,8 @@ module Gitlab
'monitoring_tool' => payload[:monitoring_tool],
'service' => payload[:service],
'hosts' => hosts.presence,
- 'severity' => severity
+ 'severity' => severity,
+ 'fingerprint' => fingerprint
}
end
diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
index 9ea20a4d6a4..4dec71b35e8 100644
--- a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
+++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb
@@ -8,21 +8,24 @@ module Gitlab
delegate :subject_class, to: :stage
- # rubocop: disable CodeReuse/ActiveRecord
+ FINDER_CLASSES = {
+ MergeRequest.to_s => MergeRequestsFinder,
+ Issue.to_s => IssuesFinder
+ }.freeze
def initialize(stage:, params: {})
@stage = stage
- @params = params
+ @params = build_finder_params(params)
end
+ # rubocop: disable CodeReuse/ActiveRecord
def build
- query = subject_class
- query = filter_by_parent_model(query)
- query = filter_by_time_range(query)
+ query = finder.execute
query = stage.start_event.apply_query_customization(query)
query = stage.end_event.apply_query_customization(query)
query.where(duration_condition)
end
+ # rubocop: enable CodeReuse/ActiveRecord
private
@@ -32,38 +35,33 @@ module Gitlab
stage.end_event.timestamp_projection.gteq(stage.start_event.timestamp_projection)
end
- def filter_by_parent_model(query)
- if parent_class.eql?(Project)
- if subject_class.eql?(Issue)
- query.where(project_id: stage.parent_id)
- elsif subject_class.eql?(MergeRequest)
- query.where(target_project_id: stage.parent_id)
- else
- raise ArgumentError, "unknown subject_class: #{subject_class}"
- end
- else
- raise ArgumentError, "unknown parent_class: #{parent_class}"
- end
+ def finder
+ FINDER_CLASSES.fetch(subject_class.to_s).new(params[:current_user], params)
end
- def filter_by_time_range(query)
- from = params.fetch(:from, 30.days.ago)
- to = params[:to]
-
- query = query.where(subject_table[:created_at].gteq(from))
- query = query.where(subject_table[:created_at].lteq(to)) if to
- query
+ def parent_class
+ stage.parent.class
end
- def subject_table
- subject_class.arel_table
+ def build_finder_params(params)
+ {}.tap do |finder_params|
+ finder_params[:current_user] = params[:current_user]
+
+ add_parent_model_params!(finder_params)
+ add_time_range_params!(finder_params, params[:from], params[:to])
+ end
end
- def parent_class
- stage.parent.class
+ def add_parent_model_params!(finder_params)
+ raise(ArgumentError, "unknown parent_class: #{parent_class}") unless parent_class.eql?(Project)
+
+ finder_params[:project_id] = stage.parent_id
end
- # rubocop: enable CodeReuse/ActiveRecord
+ def add_time_range_params!(finder_params, from, to)
+ finder_params[:created_after] = from || 30.days.ago
+ finder_params[:created_before] = to if to
+ end
end
end
end
diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb
index 9fcaeadf351..6c0450ac9e5 100644
--- a/lib/gitlab/analytics/cycle_analytics/median.rb
+++ b/lib/gitlab/analytics/cycle_analytics/median.rb
@@ -11,12 +11,14 @@ module Gitlab
@query = query
end
+ # rubocop: disable CodeReuse/ActiveRecord
def seconds
- @query = @query.select(median_duration_in_seconds.as('median'))
+ @query = @query.select(median_duration_in_seconds.as('median')).reorder(nil)
result = execute_query(@query).first || {}
result['median'] || nil
end
+ # rubocop: enable CodeReuse/ActiveRecord
def days
seconds ? seconds.fdiv(1.day) : nil
diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
index e8e269a88f0..e7352a23b99 100644
--- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
+++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb
@@ -12,13 +12,11 @@ module Gitlab
MAPPINGS = {
Issue => {
- finder_class: IssuesFinder,
serializer_class: AnalyticsIssueSerializer,
includes_for_query: { project: [:namespace], author: [] },
columns_for_select: %I[title iid id created_at author_id project_id]
},
MergeRequest => {
- finder_class: MergeRequestsFinder,
serializer_class: AnalyticsMergeRequestSerializer,
includes_for_query: { target_project: [:namespace], author: [] },
columns_for_select: %I[title iid id created_at author_id state_id target_project_id]
@@ -56,27 +54,12 @@ module Gitlab
attr_reader :stage, :query, :params
- def finder_query
- MAPPINGS
- .fetch(subject_class)
- .fetch(:finder_class)
- .new(params.fetch(:current_user), finder_params.fetch(stage.parent.class))
- .execute
- end
-
def columns
MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name|
subject_class.arel_table[column_name]
end
end
- # EE will override this to include Group rules
- def finder_params
- {
- Project => { project_id: stage.parent_id }
- }
- end
-
def default_test_stage?
stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage)
end
@@ -113,8 +96,7 @@ module Gitlab
end
def records
- results = finder_query
- .merge(ordered_and_limited_query)
+ results = ordered_and_limited_query
.select(*columns, round_duration_to_seconds.as('total_time'))
# using preloader instead of includes to avoid AR generating a large column list
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index 2defbd26b98..3277ddd9f49 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -20,15 +20,16 @@ module Gitlab
def rate_limits
{
issues_create: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.issues_create_limit }, interval: 1.minute },
- project_export: { threshold: 1, interval: 5.minutes },
+ project_export: { threshold: 30, interval: 5.minutes },
project_download_export: { threshold: 10, interval: 10.minutes },
project_repositories_archive: { threshold: 5, interval: 1.minute },
- project_generate_new_export: { threshold: 1, interval: 5.minutes },
+ project_generate_new_export: { threshold: 30, interval: 5.minutes },
project_import: { threshold: 30, interval: 5.minutes },
play_pipeline_schedule: { threshold: 1, interval: 1.minute },
show_raw_controller: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute },
- group_export: { threshold: 1, interval: 5.minutes },
- group_download_export: { threshold: 10, interval: 10.minutes }
+ group_export: { threshold: 30, interval: 5.minutes },
+ group_download_export: { threshold: 10, interval: 10.minutes },
+ group_import: { threshold: 30, interval: 5.minutes }
}.freeze
end
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index b7e78189d37..93342fbad51 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -56,6 +56,7 @@ module Gitlab
def find_user_from_job_token
return unless route_authentication_setting[:job_token_allowed]
+ return find_user_from_basic_auth_job if route_authentication_setting[:job_token_allowed] == :basic_auth
token = current_request.params[JOB_TOKEN_PARAM].presence ||
current_request.params[RUNNER_JOB_TOKEN_PARAM].presence ||
diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb
index e4a4900c37a..b3321c0b1fb 100644
--- a/lib/gitlab/auth/ldap/person.rb
+++ b/lib/gitlab/auth/ldap/person.rb
@@ -39,7 +39,7 @@ module Gitlab
*config.attributes['name'],
*config.attributes['email'],
*config.attributes['username']
- ].compact.uniq
+ ].compact.uniq.reject(&:blank?)
end
def self.normalize_dn(dn)
diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb
index 6d699d37a8c..1ca59aa827b 100644
--- a/lib/gitlab/auth/o_auth/provider.rb
+++ b/lib/gitlab/auth/o_auth/provider.rb
@@ -41,10 +41,6 @@ module Gitlab
name.to_s.start_with?('ldap')
end
- def self.ultraauth_provider?(name)
- name.to_s.eql?('ultraauth')
- end
-
def self.sync_profile_from_provider?(provider)
return true if ldap_provider?(provider)
diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml
index 8242821cedc..50112a51675 100644
--- a/lib/gitlab/background_migration/.rubocop.yml
+++ b/lib/gitlab/background_migration/.rubocop.yml
@@ -15,7 +15,7 @@ Metrics/AbcSize:
Metrics/PerceivedComplexity:
Enabled: true
-Metrics/LineLength:
+Layout/LineLength:
Enabled: true
Details: >
Long lines are very hard to read and make it more difficult to review
diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb
index 263546bd132..bc113a1e33d 100644
--- a/lib/gitlab/background_migration/backfill_project_repositories.rb
+++ b/lib/gitlab/background_migration/backfill_project_repositories.rb
@@ -189,7 +189,7 @@ module Gitlab
end
def perform(start_id, stop_id)
- Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id))
+ Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert
end
private
diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb
index c652a5bb3fc..e750b8ca374 100644
--- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb
+++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb
@@ -62,7 +62,7 @@ module Gitlab
class PrometheusService < ActiveRecord::Base
self.inheritance_column = :_type_disabled
self.table_name = 'services'
- default_scope { where(type: type) }
+ default_scope { where(type: type) } # rubocop:disable Cop/DefaultScope
def self.type
'PrometheusService'
diff --git a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb
new file mode 100644
index 00000000000..46921a070c3
--- /dev/null
+++ b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Remove serialized Ruby object in audit_events
+ class FixRubyObjectInAuditEvents
+ def perform(start_id, stop_id)
+ end
+ end
+ end
+end
+
+Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents')
diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
index 899f381e911..d2a9939b9ee 100644
--- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
+++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb
@@ -34,7 +34,7 @@ module Gitlab
end
end
- Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints)
+ Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert
execute("ANALYZE #{TEMP_TABLE}")
diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
index 956f9daa493..2bce5037d03 100644
--- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
+++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb
@@ -65,7 +65,7 @@ module Gitlab
next if service_ids.empty?
migrated_ids += service_ids
- Gitlab::Database.bulk_insert(table, data)
+ Gitlab::Database.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert
end
return if migrated_ids.empty?
diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb
index 35bfc381180..fcbcaacb2d6 100644
--- a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb
+++ b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb
@@ -73,7 +73,7 @@ module Gitlab
end
def insert_into_cluster_kubernetes_namespace(rows)
- Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name,
+ Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, # rubocop:disable Gitlab/BulkInsert
rows,
disable_quote: [:created_at, :updated_at])
end
diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb
index d2924d10225..43698b7955f 100644
--- a/lib/gitlab/background_migration/populate_untracked_uploads.rb
+++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb
@@ -95,7 +95,7 @@ module Gitlab
file.to_h.merge(created_at: 'NOW()')
end
- Gitlab::Database.bulk_insert('uploads',
+ Gitlab::Database.bulk_insert('uploads', # rubocop:disable Gitlab/BulkInsert
rows,
disable_quote: :created_at)
end
diff --git a/lib/gitlab/background_migration/reset_merge_status.rb b/lib/gitlab/background_migration/reset_merge_status.rb
index 447fec8903c..d040b4931be 100644
--- a/lib/gitlab/background_migration/reset_merge_status.rb
+++ b/lib/gitlab/background_migration/reset_merge_status.rb
@@ -7,7 +7,7 @@ module Gitlab
class ResetMergeStatus
def perform(from_id, to_id)
relation = MergeRequest.where(id: from_id..to_id,
- state: 'opened',
+ state_id: 1, # opened
merge_status: 'can_be_merged')
relation.update_all(merge_status: 'unchecked')
diff --git a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb
index cf0f582a2d4..d71a50a0af6 100644
--- a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb
+++ b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb
@@ -25,7 +25,7 @@ module Gitlab
mentions << mention_record unless mention_record.blank?
end
- Gitlab::Database.bulk_insert(
+ Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert
resource_user_mention_model.table_name,
mentions,
return_ids: true,
diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb
index 15cccc6f287..0df6e858bf4 100644
--- a/lib/gitlab/badge/coverage/report.rb
+++ b/lib/gitlab/badge/coverage/report.rb
@@ -7,12 +7,16 @@ module Gitlab
# Test coverage report badge
#
class Report < Badge::Base
- attr_reader :project, :ref, :job
+ attr_reader :project, :ref, :job, :customization
- def initialize(project, ref, job = nil)
+ def initialize(project, ref, opts: { job: nil })
@project = project
@ref = ref
- @job = job
+ @job = opts[:job]
+ @customization = {
+ key_width: opts[:key_width].to_i,
+ key_text: opts[:key_text]
+ }
@pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref)
end
diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb
index 817dc28f84a..6b78825aefd 100644
--- a/lib/gitlab/badge/coverage/template.rb
+++ b/lib/gitlab/badge/coverage/template.rb
@@ -20,10 +20,16 @@ module Gitlab
def initialize(badge)
@entity = badge.entity
@status = badge.status
+ @key_text = badge.customization.dig(:key_text)
+ @key_width = badge.customization.dig(:key_width)
end
def key_text
- @entity.to_s
+ if @key_text && @key_text.size <= MAX_KEY_SIZE
+ @key_text
+ else
+ @entity.to_s
+ end
end
def value_text
@@ -31,7 +37,11 @@ module Gitlab
end
def key_width
- 62
+ if @key_width && @key_width.between?(1, MAX_KEY_SIZE)
+ @key_width
+ else
+ 62
+ end
end
def value_width
diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/badge/pipeline/status.rb
index a403d839517..17f179f027d 100644
--- a/lib/gitlab/badge/pipeline/status.rb
+++ b/lib/gitlab/badge/pipeline/status.rb
@@ -7,11 +7,15 @@ module Gitlab
# Pipeline status badge
#
class Status < Badge::Base
- attr_reader :project, :ref
+ attr_reader :project, :ref, :customization
- def initialize(project, ref)
+ def initialize(project, ref, opts: {})
@project = project
@ref = ref
+ @customization = {
+ key_width: opts[:key_width].to_i,
+ key_text: opts[:key_text]
+ }
@sha = @project.commit(@ref).try(:sha)
end
diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/badge/pipeline/template.rb
index 0d3d44135e7..781897fab4b 100644
--- a/lib/gitlab/badge/pipeline/template.rb
+++ b/lib/gitlab/badge/pipeline/template.rb
@@ -24,10 +24,16 @@ module Gitlab
def initialize(badge)
@entity = badge.entity
@status = badge.status
+ @key_text = badge.customization.dig(:key_text)
+ @key_width = badge.customization.dig(:key_width)
end
def key_text
- @entity.to_s
+ if @key_text && @key_text.size <= MAX_KEY_SIZE
+ @key_text
+ else
+ @entity.to_s
+ end
end
def value_text
@@ -35,7 +41,11 @@ module Gitlab
end
def key_width
- 62
+ if @key_width && @key_width.between?(1, MAX_KEY_SIZE)
+ @key_width
+ else
+ 62
+ end
end
def value_width
diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/badge/template.rb
index ed2ec50b197..97103e3f42c 100644
--- a/lib/gitlab/badge/template.rb
+++ b/lib/gitlab/badge/template.rb
@@ -6,6 +6,8 @@ module Gitlab
# Abstract template class for badges
#
class Template
+ MAX_KEY_SIZE = 128
+
def initialize(badge)
@entity = badge.entity
@status = badge.status
diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index d8f9105d66d..5a9fad3be56 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -43,7 +43,7 @@ module Gitlab
def store_pull_request_error(pull_request, ex)
backtrace = Gitlab::BacktraceCleaner.clean_backtrace(ex.backtrace)
- error = { type: :pull_request, iid: pull_request.iid, errors: ex.message, trace: backtrace, raw_response: pull_request.raw }
+ error = { type: :pull_request, iid: pull_request.iid, errors: ex.message, trace: backtrace, raw_response: pull_request.raw&.to_json }
Gitlab::ErrorTracking.log_exception(ex, error)
diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb
index 7f2d2858149..ec94991157a 100644
--- a/lib/gitlab/cache/import/caching.rb
+++ b/lib/gitlab/cache/import/caching.rb
@@ -113,15 +113,18 @@ module Gitlab
end
end
- # Sets multiple keys to a given value.
+ # Sets multiple keys to given values.
#
# mapping - A Hash mapping the cache keys to their values.
+ # key_prefix - prefix inserted before each key
# timeout - The time after which the cache key should expire.
- def self.write_multiple(mapping, timeout: TIMEOUT)
+ def self.write_multiple(mapping, key_prefix: nil, timeout: TIMEOUT)
Redis::Cache.with do |redis|
- redis.multi do |multi|
+ redis.pipelined do |multi|
mapping.each do |raw_key, value|
- multi.set(cache_key_for(raw_key), value, ex: timeout)
+ key = cache_key_for("#{key_prefix}#{raw_key}")
+
+ multi.set(key, value, ex: timeout)
end
end
end
diff --git a/lib/gitlab/ci/build/releaser.rb b/lib/gitlab/ci/build/releaser.rb
new file mode 100644
index 00000000000..ba6c7857e96
--- /dev/null
+++ b/lib/gitlab/ci/build/releaser.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Build
+ class Releaser
+ BASE_COMMAND = 'release-cli create'
+
+ attr_reader :config
+
+ def initialize(config:)
+ @config = config
+ end
+
+ def script
+ command = BASE_COMMAND.dup
+ config.each { |k, v| command.concat(" --#{k.to_s.dasherize} \"#{v}\"") }
+
+ command
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb
index 48111ae5717..f8550b50905 100644
--- a/lib/gitlab/ci/build/step.rb
+++ b/lib/gitlab/ci/build/step.rb
@@ -20,6 +20,19 @@ module Gitlab
end
end
+ def from_release(job)
+ return unless Gitlab::Ci::Features.release_generation_enabled?
+
+ release = job.options[:release]
+ return unless release
+
+ self.new(:release).tap do |step|
+ step.script = Gitlab::Ci::Build::Releaser.new(config: job.options[:release]).script
+ step.timeout = job.metadata_timeout
+ step.when = WHEN_ON_SUCCESS
+ end
+ end
+
def from_after_script(job)
after_script = job.options[:after_script]
return unless after_script
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index 1ea59491378..66050a7bbe0 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -28,7 +28,7 @@ module Gitlab
in: %i[release],
message: 'release features are not enabled'
},
- unless: -> { Feature.enabled?(:ci_release_generation, default_enabled: false) }
+ unless: -> { Gitlab::Ci::Features.release_generation_enabled? }
with_options allow_nil: true do
validates :allow_failure, boolean: true
diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb
index 1a871e043a6..74736b24d73 100644
--- a/lib/gitlab/ci/config/entry/reports.rb
+++ b/lib/gitlab/ci/config/entry/reports.rb
@@ -12,9 +12,10 @@ module Gitlab
include ::Gitlab::Config::Entry::Attributable
ALLOWED_KEYS =
- %i[junit codequality sast dependency_scanning container_scanning
+ %i[junit codequality sast secret_detection dependency_scanning container_scanning
dast performance license_management license_scanning metrics lsif
- dotenv cobertura terraform accessibility cluster_applications].freeze
+ dotenv cobertura terraform accessibility cluster_applications
+ requirements].freeze
attributes ALLOWED_KEYS
@@ -26,6 +27,7 @@ module Gitlab
validates :junit, array_of_strings_or_string: true
validates :codequality, array_of_strings_or_string: true
validates :sast, array_of_strings_or_string: true
+ validates :secret_detection, array_of_strings_or_string: true
validates :dependency_scanning, array_of_strings_or_string: true
validates :container_scanning, array_of_strings_or_string: true
validates :dast, array_of_strings_or_string: true
@@ -39,6 +41,7 @@ module Gitlab
validates :terraform, array_of_strings_or_string: true
validates :accessibility, array_of_strings_or_string: true
validates :cluster_applications, array_of_strings_or_string: true
+ validates :requirements, array_of_strings_or_string: true
end
end
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index 48f3d4fdd2f..a2eb31369c7 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -7,12 +7,40 @@ module Gitlab
#
module Features
def self.artifacts_exclude_enabled?
- ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: false)
+ ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true)
end
def self.ensure_scheduling_type_enabled?
::Feature.enabled?(:ci_ensure_scheduling_type, default_enabled: true)
end
+
+ def self.job_heartbeats_runner?(project)
+ ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true)
+ end
+
+ def self.instance_level_variables_limit_enabled?
+ ::Feature.enabled?(:ci_instance_level_variables_limit, default_enabled: true)
+ end
+
+ def self.pipeline_fixed_notifications?
+ ::Feature.enabled?(:ci_pipeline_fixed_notifications)
+ end
+
+ def self.instance_variables_ui_enabled?
+ ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true)
+ end
+
+ def self.composite_status?(project)
+ ::Feature.enabled?(:ci_composite_status, project, default_enabled: true)
+ end
+
+ def self.atomic_processing?(project)
+ ::Feature.enabled?(:ci_atomic_processing, project, default_enabled: true)
+ end
+
+ def self.release_generation_enabled?
+ ::Feature.enabled?(:ci_release_generation)
+ end
end
end
end
diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb
index 26a18c6603e..19f724b79af 100644
--- a/lib/gitlab/ci/parsers/terraform/tfplan.rb
+++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb
@@ -8,15 +8,11 @@ module Gitlab
TfplanParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
def parse!(json_data, terraform_reports, artifact:)
- tfplan = Gitlab::Json.parse(json_data).tap do |parsed_data|
- parsed_data['job_path'] = Gitlab::Routing.url_helpers.project_job_path(
- artifact.job.project, artifact.job
- )
- end
+ plan_data = Gitlab::Json.parse(json_data)
- raise TfplanParserError, 'Tfplan missing required key' unless valid_supported_keys?(tfplan)
+ raise TfplanParserError, 'Tfplan missing required key' unless has_required_keys?(plan_data)
- terraform_reports.add_plan(artifact.filename, tfplan)
+ terraform_reports.add_plan(artifact.job.id.to_s, tfplan(plan_data, artifact.job))
rescue JSON::ParserError
raise TfplanParserError, 'JSON parsing failed'
rescue
@@ -25,8 +21,18 @@ module Gitlab
private
- def valid_supported_keys?(tfplan)
- tfplan.keys == %w[create update delete job_path]
+ def has_required_keys?(plan_data)
+ (%w[create update delete] - plan_data.keys).empty?
+ end
+
+ def tfplan(plan_data, artifact_job)
+ {
+ 'create' => plan_data['create'].to_i,
+ 'delete' => plan_data['delete'].to_i,
+ 'job_name' => artifact_job.options.dig(:artifacts, :name).to_s,
+ 'job_path' => Gitlab::Routing.url_helpers.project_job_path(artifact_job.project, artifact_job),
+ 'update' => plan_data['update'].to_i
+ }
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 73187401903..8118e7b2487 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -77,19 +77,18 @@ module Gitlab
bridge&.parent_pipeline
end
- def duration_histogram
- strong_memoize(:duration_histogram) do
- name = :gitlab_ci_pipeline_creation_duration_seconds
- comment = 'Pipeline creation duration'
- labels = {}
- buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0]
-
- Gitlab::Metrics.histogram(name, comment, labels, buckets)
- end
+ def metrics
+ @metrics ||= Chain::Metrics.new
end
def observe_creation_duration(duration)
- duration_histogram.observe({}, duration.seconds)
+ metrics.pipeline_creation_duration_histogram
+ .observe({}, duration.seconds)
+ end
+
+ def observe_pipeline_size(pipeline)
+ metrics.pipeline_size_histogram
+ .observe({ source: pipeline.source.to_s }, pipeline.total_size)
end
end
end
diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb
new file mode 100644
index 00000000000..980ab2de9b0
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/metrics.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Pipeline
+ module Chain
+ class Metrics
+ include Gitlab::Utils::StrongMemoize
+
+ def pipeline_creation_duration_histogram
+ strong_memoize(:pipeline_creation_duration_histogram) do
+ name = :gitlab_ci_pipeline_creation_duration_seconds
+ comment = 'Pipeline creation duration'
+ labels = {}
+ buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0]
+
+ ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+ end
+ end
+
+ def pipeline_size_histogram
+ strong_memoize(:pipeline_size_histogram) do
+ name = :gitlab_ci_pipeline_size_builds
+ comment = 'Pipeline size'
+ labels = { source: nil }
+ buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000]
+
+ ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb
index 2e177cfec7e..e48e79d561b 100644
--- a/lib/gitlab/ci/pipeline/chain/seed.rb
+++ b/lib/gitlab/ci/pipeline/chain/seed.rb
@@ -13,6 +13,7 @@ module Gitlab
# Allocate next IID. This operation must be outside of transactions of pipeline creations.
pipeline.ensure_project_iid!
+ pipeline.ensure_ci_ref!
# Protect the pipeline. This is assigned in Populate instead of
# Build to prevent erroring out on ambiguous refs.
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index a7c671e76d3..204c7725214 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -27,6 +27,7 @@ module Gitlab
yield @pipeline, self if block_given?
@command.observe_creation_duration(Time.now - @start)
+ @command.observe_pipeline_size(@pipeline)
end
end
diff --git a/lib/gitlab/ci/reports/terraform_reports.rb b/lib/gitlab/ci/reports/terraform_reports.rb
index f955d007daf..4b52c25d724 100644
--- a/lib/gitlab/ci/reports/terraform_reports.rb
+++ b/lib/gitlab/ci/reports/terraform_reports.rb
@@ -10,14 +10,6 @@ module Gitlab
@plans = {}
end
- def pick(keys)
- terraform_plans = plans.select do |key|
- keys.include?(key)
- end
-
- { plans: terraform_plans }
- end
-
def add_plan(name, plan)
plans[name] = plan
end
diff --git a/lib/gitlab/ci/status/bridge/failed.rb b/lib/gitlab/ci/status/bridge/failed.rb
index de7446c238c..b0ab0992594 100644
--- a/lib/gitlab/ci/status/bridge/failed.rb
+++ b/lib/gitlab/ci/status/bridge/failed.rb
@@ -5,6 +5,14 @@ module Gitlab
module Status
module Bridge
class Failed < Status::Build::Failed
+ private
+
+ def failure_reason_message
+ [
+ self.class.reasons.fetch(subject.failure_reason.to_sym),
+ subject.options[:downstream_errors]
+ ].flatten.compact.join(', ')
+ end
end
end
end
diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb
index ea773ee9944..4779c8d3d53 100644
--- a/lib/gitlab/ci/status/core.rb
+++ b/lib/gitlab/ci/status/core.rb
@@ -3,7 +3,7 @@
module Gitlab
module Ci
module Status
- # Base abstract class fore core status
+ # Base abstract class for core status
#
class Core
include Gitlab::Routing
diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml
new file mode 100644
index 00000000000..82b2f5c035e
--- /dev/null
+++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml
@@ -0,0 +1,13 @@
+stages:
+ - build
+ - test
+ - review
+ - deploy
+ - production
+
+variables:
+ AUTO_DEVOPS_PLATFORM_TARGET: ECS
+
+include:
+ - template: Jobs/Build.gitlab-ci.yml
+ - template: Jobs/Deploy/ECS.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index 5017037fb5a..e37cd14d1d1 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -13,6 +13,7 @@
# * license_management: LICENSE_MANAGEMENT_DISABLED
# * performance: PERFORMANCE_DISABLED
# * sast: SAST_DISABLED
+# * secret_detection: SECRET_DETECTION_DISABLED
# * dependency_scanning: DEPENDENCY_SCANNING_DISABLED
# * container_scanning: CONTAINER_SCANNING_DISABLED
# * dast: DAST_DISABLED
@@ -160,3 +161,4 @@ include:
- template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
- template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
- template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+ - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml
index 82b2f5c035e..5f4bd631db6 100644
--- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml
@@ -1,3 +1,18 @@
+# This template is deprecated and will be removed as part of GitLab 13.2!
+#
+# If you have referenced this template in your CI pipeline, please
+# update your CI configuration by replacing the following occurrence(s):
+#
+# template: Deploy-ECS.gitlab-ci.yml
+#
+# with
+#
+# template: AWS/Deploy-ECS.gitlab-ci.yml
+#
+# --------------------
+#
+# Documentation: https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-the-aws-elastic-container-service-ecs
+
stages:
- build
- test
@@ -5,6 +20,9 @@ stages:
- deploy
- production
+before_script:
+ - printf '\nWARNING!\nThis job includes "Deploy-ECS.gitlab-ci.yml". Please rename this to "AWS/Deploy-ECS.gitlab-ci.yml".\n'
+
variables:
AUTO_DEVOPS_PLATFORM_TARGET: ECS
diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
index adbf9731e43..9a34f8cb113 100644
--- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml
@@ -1,11 +1,11 @@
performance:
stage: performance
- image: docker:19.03.8
+ image: docker:19.03.11
allow_failure: true
variables:
DOCKER_TLS_CERTDIR: ""
services:
- - docker:19.03.8-dind
+ - docker:19.03.11-dind
script:
- |
if ! docker info &>/dev/null; then
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index 787f07521e0..b5550461482 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -1,10 +1,10 @@
build:
stage: build
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.2"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.3"
variables:
DOCKER_TLS_CERTDIR: ""
services:
- - docker:19.03.8-dind
+ - docker:19.03.11-dind
script:
- |
if [[ -z "$CI_COMMIT_TAG" ]]; then
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 24e75c56a75..bde6f185d3a 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -1,9 +1,9 @@
code_quality:
stage: test
- image: docker:19.03.8
+ image: docker:19.03.11
allow_failure: true
services:
- - docker:19.03.8-dind
+ - docker:19.03.11-dind
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index 5174aed04ba..bab4fae67f0 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.dast-auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.0"
dast_environment_deploy:
extends: .dast-auto-deploy
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 3fbae496896..97b5f3fd7f5 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.0"
include:
- template: Jobs/Deploy/ECS.gitlab-ci.yml
@@ -177,6 +177,7 @@ production_manual:
.manual_rollout_template: &manual_rollout_template
<<: *rollout_template
stage: production
+ resource_group: production
allow_failure: true
rules:
- if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
index 642f0ebeaf7..bb3d5526f3a 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml
@@ -1,3 +1,13 @@
+# WARNING (post-GitLab 13.0):
+#
+# This CI template should NOT be included in your own CI configuration files:
+# 'review_ecs' and 'production_ecs' are two temporary names given to the jobs below.
+#
+# Should this template be included in your CI configuration, the upcoming name changes could
+# then result in potentially breaking your future pipelines.
+#
+# More about including CI templates: https://docs.gitlab.com/ee/ci/yaml/#includetemplate
+
.deploy_to_ecs:
image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest'
script:
@@ -15,7 +25,9 @@ review_ecs:
when: never
- if: '$REVIEW_DISABLED'
when: never
- - if: '$CI_COMMIT_BRANCH != "master"'
+ - if: '$CI_COMMIT_BRANCH == "master"'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
production_ecs:
extends: .deploy_to_ecs
@@ -27,4 +39,6 @@ production_ecs:
when: never
- if: '$CI_KUBERNETES_ACTIVE'
when: never
- - if: '$CI_COMMIT_BRANCH == "master"'
+ - if: '$CI_COMMIT_BRANCH != "master"'
+ when: never
+ - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
index 54a29b04d39..316647b5921 100644
--- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
@@ -1,6 +1,6 @@
apply:
stage: deploy
- image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.15.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.20.0"
environment:
name: production
variables:
@@ -19,12 +19,17 @@ apply:
CROSSPLANE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/crossplane/values.yaml
FLUENTD_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/fluentd/values.yaml
KNATIVE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/knative/values.yaml
+ POSTHOG_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/posthog/values.yaml
+ FALCO_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/falco/values.yaml
+ APPARMOR_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/apparmor/values.yaml
script:
- gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml
only:
refs:
- master
artifacts:
+ reports:
+ cluster_applications: gl-cluster-applications.json
when: on_failure
paths:
- tiller.log
diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml
index a25dc38e4e7..f35470367cc 100644
--- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml
@@ -20,4 +20,4 @@ image: "rust:latest"
test:cargo:
script:
- rustc --version && cargo --version # Print version info for debugging
- - cargo test --all --verbose
+ - cargo test --workspace --verbose
diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
index 616966b4f04..fa8ccb7cf93 100644
--- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml
@@ -13,6 +13,7 @@ variables:
DS_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX"
DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python"
+ DS_EXCLUDED_PATHS: "spec, test, tests, tmp"
DS_MAJOR_VERSION: 2
DS_DISABLE_DIND: "true"
@@ -125,6 +126,7 @@ gemnasium-maven-dependency_scanning:
$DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/
exists:
- '{build.gradle,*/build.gradle,*/*/build.gradle}'
+ - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}'
- '{build.sbt,*/build.sbt,*/*/build.sbt}'
- '{pom.xml,*/pom.xml,*/*/pom.xml}'
diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
index b86014c1ebc..b0c75b0aab0 100644
--- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml
@@ -19,6 +19,7 @@ license_scanning:
entrypoint: [""]
variables:
LM_REPORT_FILE: gl-license-scanning-report.json
+ LM_REPORT_VERSION: '2.1'
SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD
allow_failure: true
script:
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index 47f68118ee0..ec7b34d17b5 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -13,6 +13,7 @@ variables:
SAST_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX"
SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec"
+ SAST_EXCLUDED_PATHS: "spec, test, tests, tmp"
SAST_ANALYZER_IMAGE_TAG: 2
SAST_DISABLE_DIND: "true"
SCAN_KUBERNETES_MANIFESTS: "false"
@@ -80,10 +81,9 @@ brakeman-sast:
- if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false'
when: never
- if: $CI_COMMIT_BRANCH &&
- $GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /brakeman/
exists:
- - '**/*.rb'
+ - 'config/routes.rb'
eslint-sast:
extends: .sast-analyzer
@@ -149,7 +149,7 @@ nodejs-scan-sast:
$GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/
exists:
- - '**/*.js'
+ - 'package.json'
phpcs-security-audit-sast:
extends: .sast-analyzer
@@ -213,8 +213,7 @@ sobelow-sast:
$GITLAB_FEATURES =~ /\bsast\b/ &&
$SAST_DEFAULT_ANALYZERS =~ /sobelow/
exists:
- - '**/*.ex'
- - '**/*.exs'
+ - 'mix.exs'
spotbugs-sast:
extends: .sast-analyzer
diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
new file mode 100644
index 00000000000..e18f89cadd7
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
@@ -0,0 +1,24 @@
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection
+#
+# Configure the scanning tool through the environment variables.
+# List of the variables: https://gitlab.com/gitlab-org/security-products/secret_detection#available-variables
+# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+
+variables:
+ SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+ SECRETS_ANALYZER_VERSION: "3"
+
+secret_detection:
+ stage: test
+ image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION"
+ services: []
+ rules:
+ - if: $SECRET_DETECTION_DISABLED
+ when: never
+ - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bsecret_detection\b/
+ when: on_success
+ artifacts:
+ reports:
+ secret_detection: gl-secret-detection-report.json
+ script:
+ - /analyzer run
diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
index a0832718214..377c72e8031 100644
--- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
@@ -40,7 +40,6 @@ plan:
- terraform plan -out=$PLAN
- "terraform show --json $PLAN | convert_report > $JSON_PLAN_FILE"
artifacts:
- name: plan
paths:
- $PLAN
reports:
diff --git a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml
new file mode 100644
index 00000000000..77a1b57d92f
--- /dev/null
+++ b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml
@@ -0,0 +1,17 @@
+rspec-rails-modified-path-specs:
+ stage: .pre
+ rules:
+ - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "merge_train"
+ changes: ["**/*.rb"]
+ script:
+ - gem install test_file_finder
+ - spec_files=$(tff $(git diff --name-only "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA..$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA"))
+ - |
+ if [ -n "$spec_files" ]
+ then
+ bundle install
+ bundle exec rspec -- $spec_files
+ else
+ echo "No relevant spec files found by tff"
+ exit 0
+ fi
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index 4e83826b249..f76aacc2d19 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -147,7 +147,7 @@ module Gitlab
raise AlreadyArchivedError, 'Could not write to the archived trace'
elsif current_path
File.open(current_path, mode)
- elsif Feature.enabled?('ci_enable_live_trace', job.project)
+ elsif Feature.enabled?(:ci_enable_live_trace, job.project)
Gitlab::Ci::Trace::ChunkedIO.new(job)
else
File.open(ensure_path, mode)
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 5816ac3bc54..6a9b7b2fc85 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -88,7 +88,7 @@ module Gitlab
end
def release(job)
- job[:release] if Feature.enabled?(:ci_release_generation, default_enabled: false)
+ job[:release] if Gitlab::Ci::Features.release_generation_enabled?
end
def stage_builds_attributes(stage)
diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb
index 57aeb6c4fb2..faf623faccf 100644
--- a/lib/gitlab/code_navigation_path.rb
+++ b/lib/gitlab/code_navigation_path.rb
@@ -13,7 +13,7 @@ module Gitlab
end
def full_json_path_for(path)
- return if Feature.disabled?(:code_navigation, project)
+ return unless Feature.enabled?(:code_navigation, project, default_enabled: true)
return unless build
raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json", file_type: :lsif)
diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb
index 4cedab1545c..e001742a7f8 100644
--- a/lib/gitlab/config/loader/yaml.rb
+++ b/lib/gitlab/config/loader/yaml.rb
@@ -21,11 +21,15 @@ module Gitlab
hash? && !too_big?
end
- def load!
+ def load_raw!
raise DataTooLargeError, 'The parsed YAML is too big' if too_big?
raise Loader::FormatError, 'Invalid configuration format' unless hash?
- @config.deep_symbolize_keys
+ @config
+ end
+
+ def load!
+ @symbolized_config ||= load_raw!.deep_symbolize_keys
end
private
diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb
index 5b0b91de5da..4e430d8937d 100644
--- a/lib/gitlab/contributions_calendar.rb
+++ b/lib/gitlab/contributions_calendar.rb
@@ -24,13 +24,13 @@ module Gitlab
# project_features for the (currently) 3 different contribution types
date_from = 1.year.ago
repo_events = event_counts(date_from, :repository)
- .having(action: Event::PUSHED)
+ .having(action: :pushed)
issue_events = event_counts(date_from, :issues)
- .having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue")
+ .having(action: [:created, :closed], target_type: "Issue")
mr_events = event_counts(date_from, :merge_requests)
- .having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest")
+ .having(action: [:merged, :created, :closed], target_type: "MergeRequest")
note_events = event_counts(date_from, :merge_requests)
- .having(action: [Event::COMMENTED])
+ .having(action: :commented)
events = Event
.from_union([repo_events, issue_events, mr_events, note_events])
diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb
index 1f426b81800..1dc9d5de966 100644
--- a/lib/gitlab/cycle_analytics/summary/commit.rb
+++ b/lib/gitlab/cycle_analytics/summary/commit.rb
@@ -5,7 +5,7 @@ module Gitlab
module Summary
class Commit < Base
def title
- n_('Commit', 'Commits', value)
+ n_('Commit', 'Commits', value.to_i)
end
def value
diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb
index 8544ea1a91e..5125c8e64ee 100644
--- a/lib/gitlab/cycle_analytics/summary/deploy.rb
+++ b/lib/gitlab/cycle_analytics/summary/deploy.rb
@@ -5,7 +5,7 @@ module Gitlab
module Summary
class Deploy < Base
def title
- n_('Deploy', 'Deploys', value)
+ n_('Deploy', 'Deploys', value.to_i)
end
def value
diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb
index ce7788590b9..462fd4c2d3d 100644
--- a/lib/gitlab/cycle_analytics/summary/issue.rb
+++ b/lib/gitlab/cycle_analytics/summary/issue.rb
@@ -12,7 +12,7 @@ module Gitlab
end
def title
- n_('New Issue', 'New Issues', value)
+ n_('New Issue', 'New Issues', value.to_i)
end
def value
diff --git a/lib/gitlab/cycle_analytics/summary/value.rb b/lib/gitlab/cycle_analytics/summary/value.rb
index ce32132e048..e443e037517 100644
--- a/lib/gitlab/cycle_analytics/summary/value.rb
+++ b/lib/gitlab/cycle_analytics/summary/value.rb
@@ -15,6 +15,14 @@ module Gitlab
end
class None < self
+ def raw_value
+ 0
+ end
+
+ def to_i
+ raw_value
+ end
+
def to_s
'-'
end
@@ -28,6 +36,10 @@ module Gitlab
def to_s
value.zero? ? '0' : value.to_s
end
+
+ def to_i
+ raw_value
+ end
end
class PrettyNumeric < Numeric
diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb
index a2867087428..e31a6ae5011 100644
--- a/lib/gitlab/danger/emoji_checker.rb
+++ b/lib/gitlab/danger/emoji_checker.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-require_relative '../json'
+require 'json'
module Gitlab
module Danger
@@ -25,8 +25,8 @@ module Gitlab
)}x.freeze
def initialize
- names = Gitlab::Json.parse(File.read(DIGESTS)).keys +
- Gitlab::Json.parse(File.read(ALIASES)).keys
+ names = JSON.parse(File.read(DIGESTS)).keys +
+ JSON.parse(File.read(ALIASES)).keys
@emoji = names.map { |name| ":#{name}:" }
end
diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb
index 0f0af5f777b..327418ad100 100644
--- a/lib/gitlab/danger/helper.rb
+++ b/lib/gitlab/danger/helper.rb
@@ -142,6 +142,7 @@ module Gitlab
%r{Dangerfile\z} => :engineering_productivity,
%r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity,
%r{\A(ee/)?scripts/} => :engineering_productivity,
+ %r{\A\.codeclimate\.yml\z} => :engineering_productivity,
%r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend,
%r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend,
@@ -151,6 +152,7 @@ module Gitlab
%r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend,
%r{\A[A-Z_]+_VERSION\z} => :backend,
%r{\A\.rubocop(_todo)?\.yml\z} => :backend,
+ %r{\Afile_hooks/} => :backend,
%r{\A(ee/)?qa/} => :qa,
diff --git a/lib/gitlab/danger/request_helper.rb b/lib/gitlab/danger/request_helper.rb
index ef51c3f2052..06da4ed9ad3 100644
--- a/lib/gitlab/danger/request_helper.rb
+++ b/lib/gitlab/danger/request_helper.rb
@@ -16,7 +16,7 @@ module Gitlab
raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}"
end
- Gitlab::Json.parse(rsp.body)
+ JSON.parse(rsp.body)
end
end
end
diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb
index dbf42912882..9f7980dc20a 100644
--- a/lib/gitlab/danger/roulette.rb
+++ b/lib/gitlab/danger/roulette.rb
@@ -6,6 +6,46 @@ module Gitlab
module Danger
module Roulette
ROULETTE_DATA_URL = 'https://about.gitlab.com/roulette.json'
+ OPTIONAL_CATEGORIES = [:qa, :test].freeze
+
+ Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role)
+
+ # Assigns GitLab team members to be reviewer and maintainer
+ # for each change category that a Merge Request contains.
+ #
+ # @return [Array<Spin>]
+ def spin(project, categories, branch_name)
+ team =
+ begin
+ project_team(project)
+ rescue => err
+ warn("Reviewer roulette failed to load team data: #{err.message}")
+ []
+ end
+
+ canonical_branch_name = canonical_branch_name(branch_name)
+
+ spin_per_category = categories.each_with_object({}) do |category, memo|
+ memo[category] = spin_for_category(team, project, category, canonical_branch_name)
+ end
+
+ spin_per_category.map do |category, spin|
+ case category
+ when :test
+ if spin.reviewer.nil?
+ # Fetch an already picked backend reviewer, or pick one otherwise
+ spin.reviewer = spin_per_category[:backend]&.reviewer || spin_for_category(team, project, :backend, canonical_branch_name).reviewer
+ end
+ when :engineering_productivity
+ if spin.maintainer.nil?
+ # Fetch an already picked backend maintainer, or pick one otherwise
+ spin.maintainer = spin_per_category[:backend]&.maintainer || spin_for_category(team, project, :backend, canonical_branch_name).maintainer
+ end
+ end
+
+ spin
+ end
+ end
# Looks up the current list of GitLab team members and parses it into a
# useful form
@@ -58,6 +98,33 @@ module Gitlab
def mr_author?(person)
person.username == gitlab.mr_author
end
+
+ def spin_role_for_category(team, role, project, category)
+ team.select do |member|
+ member.public_send("#{role}?", project, category, gitlab.mr_labels) # rubocop:disable GitlabSecurity/PublicSend
+ end
+ end
+
+ def spin_for_category(team, project, category, branch_name)
+ reviewers, traintainers, maintainers =
+ %i[reviewer traintainer maintainer].map do |role|
+ spin_role_for_category(team, role, project, category)
+ end
+
+ # TODO: take CODEOWNERS into account?
+ # https://gitlab.com/gitlab-org/gitlab/issues/26723
+
+ # Make traintainers have triple the chance to be picked as a reviewer
+ random = new_random(branch_name)
+ reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random)
+ maintainer = spin_for_person(maintainers, random: random)
+
+ Spin.new(category, reviewer, maintainer).tap do |spin|
+ if OPTIONAL_CATEGORIES.include?(category)
+ spin.optional_role = :maintainer
+ end
+ end
+ end
end
end
end
diff --git a/lib/gitlab/data_builder/alert.rb b/lib/gitlab/data_builder/alert.rb
new file mode 100644
index 00000000000..e34bdeea799
--- /dev/null
+++ b/lib/gitlab/data_builder/alert.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module DataBuilder
+ module Alert
+ extend self
+
+ def build(alert)
+ {
+ object_kind: 'alert',
+ object_attributes: hook_attrs(alert)
+ }
+ end
+
+ def hook_attrs(alert)
+ {
+ title: alert.title,
+ url: Gitlab::Routing.url_helpers.details_project_alert_management_url(alert.project, alert.iid),
+ severity: alert.severity,
+ events: alert.events,
+ status: alert.status_name,
+ started_at: alert.started_at
+ }
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb
index 2c4ef73a688..73518d36d43 100644
--- a/lib/gitlab/data_builder/note.rb
+++ b/lib/gitlab/data_builder/note.rb
@@ -55,7 +55,7 @@ module Gitlab
end
def build_base_data(project, user, note)
- event_type = note.confidential? ? 'confidential_note' : 'note'
+ event_type = note.confidential?(include_noteable: true) ? 'confidential_note' : 'note'
base_data = {
object_kind: "note",
diff --git a/lib/gitlab/database/custom_structure.rb b/lib/gitlab/database/custom_structure.rb
new file mode 100644
index 00000000000..c5a76c5a787
--- /dev/null
+++ b/lib/gitlab/database/custom_structure.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class CustomStructure
+ CUSTOM_DUMP_FILE = 'db/gitlab_structure.sql'
+
+ def dump
+ File.open(self.class.custom_dump_filepath, 'wb') do |io|
+ io << "-- this file tracks custom GitLab data, such as foreign keys referencing partitioned tables\n"
+ io << "-- more details can be found in the issue: https://gitlab.com/gitlab-org/gitlab/-/issues/201872\n"
+ io << "SET search_path=public;\n\n"
+
+ dump_partitioned_foreign_keys(io) if partitioned_foreign_keys_exist?
+ end
+ end
+
+ def self.custom_dump_filepath
+ Rails.root.join(CUSTOM_DUMP_FILE)
+ end
+
+ private
+
+ def dump_partitioned_foreign_keys(io)
+ io << "COPY partitioned_foreign_keys (#{partitioned_fk_columns.join(", ")}) FROM STDIN;\n"
+
+ PartitioningMigrationHelpers::PartitionedForeignKey.find_each do |fk|
+ io << fk.attributes.values_at(*partitioned_fk_columns).join("\t") << "\n"
+ end
+ io << "\\.\n"
+ end
+
+ def partitioned_foreign_keys_exist?
+ return false unless PartitioningMigrationHelpers::PartitionedForeignKey.table_exists?
+
+ PartitioningMigrationHelpers::PartitionedForeignKey.exists?
+ end
+
+ def partitioned_fk_columns
+ @partitioned_fk_columns ||= PartitioningMigrationHelpers::PartitionedForeignKey.column_names
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index 96be057f77e..fd09c31e994 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -3,6 +3,8 @@
module Gitlab
module Database
module MigrationHelpers
+ # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
+ MAX_IDENTIFIER_NAME_LENGTH = 63
BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job
BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time
@@ -1209,6 +1211,8 @@ into similar problems in the future (e.g. when new tables are created).
#
# rubocop:disable Gitlab/RailsLogger
def add_check_constraint(table, check, constraint_name, validate: true)
+ validate_check_constraint_name!(constraint_name)
+
# Transactions would result in ALTER TABLE locks being held for the
# duration of the transaction, defeating the purpose of this method.
if transaction_open?
@@ -1244,6 +1248,8 @@ into similar problems in the future (e.g. when new tables are created).
end
def validate_check_constraint(table, constraint_name)
+ validate_check_constraint_name!(constraint_name)
+
unless check_constraint_exists?(table, constraint_name)
raise missing_schema_object_message(table, "check constraint", constraint_name)
end
@@ -1256,6 +1262,8 @@ into similar problems in the future (e.g. when new tables are created).
end
def remove_check_constraint(table, constraint_name)
+ validate_check_constraint_name!(constraint_name)
+
# DROP CONSTRAINT requires an EXCLUSIVE lock
# Use with_lock_retries to make sure that this will not timeout
with_lock_retries do
@@ -1330,6 +1338,12 @@ into similar problems in the future (e.g. when new tables are created).
private
+ def validate_check_constraint_name!(constraint_name)
+ if constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH
+ raise "The maximum allowed constraint name is #{MAX_IDENTIFIER_NAME_LENGTH} characters"
+ end
+ end
+
def statement_timeout_disabled?
# This is a string of the form "100ms" or "0" when disabled
connection.select_value('SHOW statement_timeout') == "0"
diff --git a/lib/gitlab/database/partitioning_migration_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers.rb
index 55649ebbf8a..881177a195e 100644
--- a/lib/gitlab/database/partitioning_migration_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers.rb
@@ -3,120 +3,8 @@
module Gitlab
module Database
module PartitioningMigrationHelpers
- include SchemaHelpers
-
- def add_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id, on_delete: :cascade)
- cascade_delete = extract_cascade_option(on_delete)
-
- update_foreign_keys(from_table, to_table, column, primary_key, cascade_delete) do |current_keys, existing_key, specified_key|
- if existing_key.nil?
- unless specified_key.save
- raise "failed to create foreign key: #{specified_key.errors.full_messages.to_sentence}"
- end
-
- current_keys << specified_key
- else
- Rails.logger.warn "foreign key not added because it already exists: #{specified_key}" # rubocop:disable Gitlab/RailsLogger
- current_keys
- end
- end
- end
-
- def remove_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id)
- update_foreign_keys(from_table, to_table, column, primary_key) do |current_keys, existing_key, specified_key|
- if existing_key
- existing_key.destroy!
- current_keys.delete(existing_key)
- else
- Rails.logger.warn "foreign key not removed because it doesn't exist: #{specified_key}" # rubocop:disable Gitlab/RailsLogger
- end
-
- current_keys
- end
- end
-
- def fk_function_name(table)
- object_name(table, 'fk_cascade_function')
- end
-
- def fk_trigger_name(table)
- object_name(table, 'fk_cascade_trigger')
- end
-
- private
-
- def fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete)
- PartitionedForeignKey.new(from_table: from_table.to_s, to_table: to_table.to_s, from_column: from_column.to_s,
- to_column: to_column.to_s, cascade_delete: cascade_delete)
- end
-
- def update_foreign_keys(from_table, to_table, from_column, to_column, cascade_delete = nil)
- if transaction_open?
- raise 'partitioned foreign key operations can not be run inside a transaction block, ' \
- 'you can disable transaction blocks by calling disable_ddl_transaction! ' \
- 'in the body of your migration class'
- end
-
- from_column ||= "#{to_table.to_s.singularize}_id"
- specified_key = fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete)
-
- current_keys = PartitionedForeignKey.by_referenced_table(to_table).to_a
- existing_key = find_existing_key(current_keys, specified_key)
-
- final_keys = yield current_keys, existing_key, specified_key
-
- fn_name = fk_function_name(to_table)
- trigger_name = fk_trigger_name(to_table)
-
- with_lock_retries do
- drop_trigger(to_table, trigger_name, if_exists: true)
-
- if final_keys.empty?
- drop_function(fn_name, if_exists: true)
- else
- create_or_replace_fk_function(fn_name, final_keys)
- create_function_trigger(trigger_name, fn_name, fires: "AFTER DELETE ON #{to_table}")
- end
- end
- end
-
- def extract_cascade_option(on_delete)
- case on_delete
- when :cascade then true
- when :nullify then false
- else raise ArgumentError, "invalid option #{on_delete} for :on_delete"
- end
- end
-
- def with_lock_retries(&block)
- Gitlab::Database::WithLockRetries.new({
- klass: self.class,
- logger: Gitlab::BackgroundMigration::Logger
- }).run(&block)
- end
-
- def find_existing_key(keys, key)
- keys.find { |k| k.from_table == key.from_table && k.from_column == key.from_column }
- end
-
- def create_or_replace_fk_function(fn_name, fk_specs)
- create_trigger_function(fn_name, replace: true) do
- cascade_statements = build_cascade_statements(fk_specs)
- cascade_statements << 'RETURN OLD;'
-
- cascade_statements.join("\n")
- end
- end
-
- def build_cascade_statements(foreign_keys)
- foreign_keys.map do |fks|
- if fks.cascade_delete?
- "DELETE FROM #{fks.from_table} WHERE #{fks.from_column} = OLD.#{fks.to_column};"
- else
- "UPDATE #{fks.from_table} SET #{fks.from_column} = NULL WHERE #{fks.from_column} = OLD.#{fks.to_column};"
- end
- end
- end
+ include ForeignKeyHelpers
+ include TableManagementHelpers
end
end
end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
new file mode 100644
index 00000000000..9e687009cd7
--- /dev/null
+++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module PartitioningMigrationHelpers
+ module ForeignKeyHelpers
+ include ::Gitlab::Database::SchemaHelpers
+
+ # Creates a "foreign key" that references a partitioned table. Because foreign keys referencing partitioned
+ # tables are not supported in PG11, this does not create a true database foreign key, but instead implements the
+ # same functionality at the database level by using triggers.
+ #
+ # Example:
+ #
+ # add_partitioned_foreign_key :issues, :projects
+ #
+ # Available options:
+ #
+ # :column - name of the referencing column (otherwise inferred from the referenced table name)
+ # :primary_key - name of the primary key in the referenced table (defaults to id)
+ # :on_delete - supports either :cascade for ON DELETE CASCADE or :nullify for ON DELETE SET NULL
+ #
+ def add_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id, on_delete: :cascade)
+ cascade_delete = extract_cascade_option(on_delete)
+
+ update_foreign_keys(from_table, to_table, column, primary_key, cascade_delete) do |current_keys, existing_key, specified_key|
+ if existing_key.nil?
+ unless specified_key.save
+ raise "failed to create foreign key: #{specified_key.errors.full_messages.to_sentence}"
+ end
+
+ current_keys << specified_key
+ else
+ Rails.logger.warn "foreign key not added because it already exists: #{specified_key}" # rubocop:disable Gitlab/RailsLogger
+ current_keys
+ end
+ end
+ end
+
+ # Drops a "foreign key" that references a partitioned table. This method ONLY applies to foreign keys previously
+ # created through the `add_partitioned_foreign_key` method. Standard database foreign keys should be managed
+ # through the familiar Rails helpers.
+ #
+ # Example:
+ #
+ # remove_partitioned_foreign_key :issues, :projects
+ #
+ # Available options:
+ #
+ # :column - name of the referencing column (otherwise inferred from the referenced table name)
+ # :primary_key - name of the primary key in the referenced table (defaults to id)
+ #
+ def remove_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id)
+ update_foreign_keys(from_table, to_table, column, primary_key) do |current_keys, existing_key, specified_key|
+ if existing_key
+ existing_key.delete
+ current_keys.delete(existing_key)
+ else
+ Rails.logger.warn "foreign key not removed because it doesn't exist: #{specified_key}" # rubocop:disable Gitlab/RailsLogger
+ end
+
+ current_keys
+ end
+ end
+
+ private
+
+ def fk_function_name(table)
+ object_name(table, 'fk_cascade_function')
+ end
+
+ def fk_trigger_name(table)
+ object_name(table, 'fk_cascade_trigger')
+ end
+
+ def fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete)
+ PartitionedForeignKey.new(from_table: from_table.to_s, to_table: to_table.to_s, from_column: from_column.to_s,
+ to_column: to_column.to_s, cascade_delete: cascade_delete)
+ end
+
+ def update_foreign_keys(from_table, to_table, from_column, to_column, cascade_delete = nil)
+ assert_not_in_transaction_block(scope: 'partitioned foreign key')
+
+ from_column ||= "#{to_table.to_s.singularize}_id"
+ specified_key = fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete)
+
+ current_keys = PartitionedForeignKey.by_referenced_table(to_table).to_a
+ existing_key = find_existing_key(current_keys, specified_key)
+
+ final_keys = yield current_keys, existing_key, specified_key
+
+ fn_name = fk_function_name(to_table)
+ trigger_name = fk_trigger_name(to_table)
+
+ with_lock_retries do
+ drop_trigger(to_table, trigger_name, if_exists: true)
+
+ if final_keys.empty?
+ drop_function(fn_name, if_exists: true)
+ else
+ create_or_replace_fk_function(fn_name, final_keys)
+ create_trigger(trigger_name, fn_name, fires: "AFTER DELETE ON #{to_table}")
+ end
+ end
+ end
+
+ def extract_cascade_option(on_delete)
+ case on_delete
+ when :cascade then true
+ when :nullify then false
+ else raise ArgumentError, "invalid option #{on_delete} for :on_delete"
+ end
+ end
+
+ def find_existing_key(keys, key)
+ keys.find { |k| k.from_table == key.from_table && k.from_column == key.from_column }
+ end
+
+ def create_or_replace_fk_function(fn_name, fk_specs)
+ create_trigger_function(fn_name, replace: true) do
+ cascade_statements = build_cascade_statements(fk_specs)
+ cascade_statements << 'RETURN OLD;'
+
+ cascade_statements.join("\n")
+ end
+ end
+
+ def build_cascade_statements(foreign_keys)
+ foreign_keys.map do |fks|
+ if fks.cascade_delete?
+ "DELETE FROM #{fks.from_table} WHERE #{fks.from_column} = OLD.#{fks.to_column};"
+ else
+ "UPDATE #{fks.from_table} SET #{fks.from_column} = NULL WHERE #{fks.from_column} = OLD.#{fks.to_column};"
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
new file mode 100644
index 00000000000..f77fbe98df1
--- /dev/null
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module PartitioningMigrationHelpers
+ module TableManagementHelpers
+ include ::Gitlab::Database::SchemaHelpers
+
+ WHITELISTED_TABLES = %w[audit_events].freeze
+ ERROR_SCOPE = 'table partitioning'
+
+ # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column.
+ # One partition is created per month between the given `min_date` and `max_date`.
+ #
+ # A copy of the original table is required as PG currently does not support partitioning existing tables.
+ #
+ # Example:
+ #
+ # partition_table_by_date :audit_events, :created_at, min_date: Date.new(2020, 1), max_date: Date.new(2020, 6)
+ #
+ # Required options are:
+ # :min_date - a date specifying the lower bounds of the partition range
+ # :max_date - a date specifying the upper bounds of the partitioning range
+ #
+ def partition_table_by_date(table_name, column_name, min_date:, max_date:)
+ assert_table_is_whitelisted(table_name)
+ assert_not_in_transaction_block(scope: ERROR_SCOPE)
+
+ raise "max_date #{max_date} must be greater than min_date #{min_date}" if min_date >= max_date
+
+ primary_key = connection.primary_key(table_name)
+ raise "primary key not defined for #{table_name}" if primary_key.nil?
+
+ partition_column = find_column_definition(table_name, column_name)
+ raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil?
+
+ new_table_name = partitioned_table_name(table_name)
+ create_range_partitioned_copy(new_table_name, table_name, partition_column, primary_key)
+ create_daterange_partitions(new_table_name, partition_column.name, min_date, max_date)
+ create_sync_trigger(table_name, new_table_name, primary_key)
+ end
+
+ # Clean up a partitioned copy of an existing table. This deletes the partitioned table and all partitions.
+ #
+ # Example:
+ #
+ # drop_partitioned_table_for :audit_events
+ #
+ def drop_partitioned_table_for(table_name)
+ assert_table_is_whitelisted(table_name)
+ assert_not_in_transaction_block(scope: ERROR_SCOPE)
+
+ with_lock_retries do
+ trigger_name = sync_trigger_name(table_name)
+ drop_trigger(table_name, trigger_name)
+ end
+
+ function_name = sync_function_name(table_name)
+ drop_function(function_name)
+
+ part_table_name = partitioned_table_name(table_name)
+ drop_table(part_table_name)
+ end
+
+ private
+
+ def assert_table_is_whitelisted(table_name)
+ return if WHITELISTED_TABLES.include?(table_name.to_s)
+
+ raise "partitioning helpers are in active development, and #{table_name} is not whitelisted for use, " \
+ "for more information please contact the database team"
+ end
+
+ def partitioned_table_name(table)
+ tmp_table_name("#{table}_part")
+ end
+
+ def sync_function_name(table)
+ object_name(table, 'table_sync_function')
+ end
+
+ def sync_trigger_name(table)
+ object_name(table, 'table_sync_trigger')
+ end
+
+ def find_column_definition(table, column)
+ connection.columns(table).find { |c| c.name == column.to_s }
+ end
+
+ def create_range_partitioned_copy(table_name, template_table_name, partition_column, primary_key)
+ if table_exists?(table_name)
+ # rubocop:disable Gitlab/RailsLogger
+ Rails.logger.warn "Partitioned table not created because it already exists" \
+ " (this may be due to an aborted migration or similar): table_name: #{table_name} "
+ # rubocop:enable Gitlab/RailsLogger
+ return
+ end
+
+ tmp_column_name = object_name(partition_column.name, 'partition_key')
+ transaction do
+ execute(<<~SQL)
+ CREATE TABLE #{table_name} (
+ LIKE #{template_table_name} INCLUDING ALL EXCLUDING INDEXES,
+ #{tmp_column_name} #{partition_column.sql_type} NOT NULL,
+ PRIMARY KEY (#{[primary_key, tmp_column_name].join(", ")})
+ ) PARTITION BY RANGE (#{tmp_column_name})
+ SQL
+
+ remove_column(table_name, partition_column.name)
+ rename_column(table_name, tmp_column_name, partition_column.name)
+ change_column_default(table_name, primary_key, nil)
+
+ if column_of_type?(table_name, primary_key, :integer)
+ # Default to int8 primary keys to prevent overflow
+ change_column(table_name, primary_key, :bigint)
+ end
+ end
+ end
+
+ def column_of_type?(table_name, column, type)
+ find_column_definition(table_name, column).type == type
+ end
+
+ def create_daterange_partitions(table_name, column_name, min_date, max_date)
+ min_date = min_date.beginning_of_month.to_date
+ max_date = max_date.next_month.beginning_of_month.to_date
+
+ create_range_partition_safely("#{table_name}_000000", table_name, 'MINVALUE', to_sql_date_literal(min_date))
+
+ while min_date < max_date
+ partition_name = "#{table_name}_#{min_date.strftime('%Y%m')}"
+ next_date = min_date.next_month
+ lower_bound = to_sql_date_literal(min_date)
+ upper_bound = to_sql_date_literal(next_date)
+
+ create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound)
+ min_date = next_date
+ end
+ end
+
+ def to_sql_date_literal(date)
+ connection.quote(date.strftime('%Y-%m-%d'))
+ end
+
+ def create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound)
+ if table_exists?(partition_name)
+ # rubocop:disable Gitlab/RailsLogger
+ Rails.logger.warn "Partition not created because it already exists" \
+ " (this may be due to an aborted migration or similar): partition_name: #{partition_name}"
+ # rubocop:enable Gitlab/RailsLogger
+ return
+ end
+
+ create_range_partition(partition_name, table_name, lower_bound, upper_bound)
+ end
+
+ def create_sync_trigger(source_table, target_table, unique_key)
+ function_name = sync_function_name(source_table)
+ trigger_name = sync_trigger_name(source_table)
+
+ with_lock_retries do
+ create_sync_function(function_name, target_table, unique_key)
+ create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table} table")
+
+ create_trigger(trigger_name, function_name, fires: "AFTER INSERT OR UPDATE OR DELETE ON #{source_table}")
+ end
+ end
+
+ def create_sync_function(name, target_table, unique_key)
+ delimiter = ",\n "
+ column_names = connection.columns(target_table).map(&:name)
+ set_statements = build_set_statements(column_names, unique_key)
+ insert_values = column_names.map { |name| "NEW.#{name}" }
+
+ create_trigger_function(name, replace: false) do
+ <<~SQL
+ IF (TG_OP = 'DELETE') THEN
+ DELETE FROM #{target_table} where #{unique_key} = OLD.#{unique_key};
+ ELSIF (TG_OP = 'UPDATE') THEN
+ UPDATE #{target_table}
+ SET #{set_statements.join(delimiter)}
+ WHERE #{target_table}.#{unique_key} = NEW.#{unique_key};
+ ELSIF (TG_OP = 'INSERT') THEN
+ INSERT INTO #{target_table} (#{column_names.join(delimiter)})
+ VALUES (#{insert_values.join(delimiter)});
+ END IF;
+ RETURN NULL;
+ SQL
+ end
+ end
+
+ def build_set_statements(column_names, unique_key)
+ column_names.reject { |name| name == unique_key }.map { |column_name| "#{column_name} = NEW.#{column_name}" }
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb
index c1436d3e7ca..ae9d77e635e 100644
--- a/lib/gitlab/database/schema_cleaner.rb
+++ b/lib/gitlab/database/schema_cleaner.rb
@@ -12,27 +12,15 @@ module Gitlab
def clean(io)
structure = original_schema.dup
- # Postgres compat fix for PG 9.6 (which doesn't support (AS datatype) syntax for sequences)
- structure.gsub!(/CREATE SEQUENCE [^.]+\.\S+\n(\s+AS integer\n)/) { |m| m.gsub(Regexp.last_match[1], '') }
-
- # Also a PG 9.6 compatibility fix, see below.
- structure.gsub!(/^CREATE EXTENSION IF NOT EXISTS plpgsql.*/, '')
- structure.gsub!(/^COMMENT ON EXTENSION.*/, '')
-
# Remove noise
+ structure.gsub!(/^COMMENT ON EXTENSION.*/, '')
structure.gsub!(/^SET.+/, '')
structure.gsub!(/^SELECT pg_catalog\.set_config\('search_path'.+/, '')
structure.gsub!(/^--.*/, "\n")
- structure.gsub!(/\n{3,}/, "\n\n")
- io << "SET search_path=public;\n\n"
+ structure = "SET search_path=public;\n" + structure
- # Adding plpgsql explicitly is again a compatibility fix for PG 9.6
- # In more recent versions of pg_dump, the extension isn't explicitly dumped anymore.
- # We use PG 9.6 still on CI and for schema checks - here this is still the case.
- io << <<~SQL.strip
- CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
- SQL
+ structure.gsub!(/\n{3,}/, "\n\n")
io << structure
diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb
index f8d01c78ae8..8e544307d81 100644
--- a/lib/gitlab/database/schema_helpers.rb
+++ b/lib/gitlab/database/schema_helpers.rb
@@ -16,12 +16,12 @@ module Gitlab
SQL
end
- def create_function_trigger(name, fn_name, fires: nil)
+ def create_trigger(name, function_name, fires: nil)
execute(<<~SQL)
CREATE TRIGGER #{name}
#{fires}
FOR EACH ROW
- EXECUTE PROCEDURE #{fn_name}()
+ EXECUTE PROCEDURE #{function_name}()
SQL
end
@@ -35,6 +35,16 @@ module Gitlab
execute("DROP TRIGGER #{exists_clause} #{name} ON #{table_name}")
end
+ def create_comment(type, name, text)
+ execute("COMMENT ON #{type} #{name} IS '#{text}'")
+ end
+
+ def tmp_table_name(base)
+ hashed_base = Digest::SHA256.hexdigest(base).first(10)
+
+ "#{base}_#{hashed_base}"
+ end
+
def object_name(table, type)
identifier = "#{table}_#{type}"
hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10)
@@ -42,8 +52,30 @@ module Gitlab
"#{type}_#{hashed_identifier}"
end
+ def with_lock_retries(&block)
+ Gitlab::Database::WithLockRetries.new({
+ klass: self.class,
+ logger: Gitlab::BackgroundMigration::Logger
+ }).run(&block)
+ end
+
+ def assert_not_in_transaction_block(scope:)
+ return unless transaction_open?
+
+ raise "#{scope} operations can not be run inside a transaction block, " \
+ "you can disable transaction blocks by calling disable_ddl_transaction! " \
+ "in the body of your migration class"
+ end
+
private
+ def create_range_partition(partition_name, table_name, lower_bound, upper_bound)
+ execute(<<~SQL)
+ CREATE TABLE #{partition_name} PARTITION OF #{table_name}
+ FOR VALUES FROM (#{lower_bound}) TO (#{upper_bound})
+ SQL
+ end
+
def optional_clause(flag, clause)
flag ? clause : ""
end
diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb
index 7af380689d5..db60128b979 100644
--- a/lib/gitlab/dependency_linker.rb
+++ b/lib/gitlab/dependency_linker.rb
@@ -13,7 +13,9 @@ module Gitlab
CartfileLinker,
GodepsJsonLinker,
RequirementsTxtLinker,
- CargoTomlLinker
+ CargoTomlLinker,
+ GoModLinker,
+ GoSumLinker
].freeze
def self.linker(blob_name)
diff --git a/lib/gitlab/dependency_linker/go_mod_linker.rb b/lib/gitlab/dependency_linker/go_mod_linker.rb
new file mode 100644
index 00000000000..4d6fe366333
--- /dev/null
+++ b/lib/gitlab/dependency_linker/go_mod_linker.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module DependencyLinker
+ class GoModLinker < BaseLinker
+ include Gitlab::Golang
+
+ self.file_type = :go_mod
+
+ private
+
+ SEMVER = Gitlab::Regex.unbounded_semver_regex
+ NAME = Gitlab::Regex.go_package_regex
+ REGEX = Regexp.new("(?<name>#{NAME.source})(?:\\s+(?<version>v#{SEMVER.source}))?", SEMVER.options | NAME.options).freeze
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def link_dependencies
+ highlighted_lines.map!.with_index do |rich_line, i|
+ plain_line = plain_lines[i].chomp
+ match = REGEX.match(plain_line)
+ next rich_line unless match
+
+ i, j = match.offset(:name)
+ marker = StringRangeMarker.new(plain_line, rich_line.html_safe)
+ marker.mark([i..(j - 1)]) do |text, left:, right:|
+ url = package_url(text, match[:version])
+ url ? link_tag(text, url) : text
+ end
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+ end
+ end
+end
diff --git a/lib/gitlab/dependency_linker/go_sum_linker.rb b/lib/gitlab/dependency_linker/go_sum_linker.rb
new file mode 100644
index 00000000000..20dc82ede9f
--- /dev/null
+++ b/lib/gitlab/dependency_linker/go_sum_linker.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module DependencyLinker
+ class GoSumLinker < GoModLinker
+ self.file_type = :go_sum
+
+ private
+
+ BASE64 = Gitlab::Regex.base64_regex
+ REGEX = Regexp.new("^\\s*(?<name>#{NAME.source})\\s+(?<version>v#{SEMVER.source})(\/go.mod)?\\s+h1:(?<checksum>#{BASE64.source})\\s*$", NAME.options).freeze
+
+ # rubocop: disable CodeReuse/ActiveRecord
+ def link_dependencies
+ highlighted_lines.map!.with_index do |rich_line, i|
+ plain_line = plain_lines[i].chomp
+ match = REGEX.match(plain_line)
+ next rich_line unless match
+
+ i0, j0 = match.offset(:name)
+ i2, j2 = match.offset(:checksum)
+
+ marker = StringRangeMarker.new(plain_line, rich_line.html_safe)
+ marker.mark([i0..(j0 - 1), i2..(j2 - 1)]) do |text, left:, right:|
+ if left
+ url = package_url(text, match[:version])
+ url ? link_tag(text, url) : text
+
+ elsif right
+ link_tag(text, "https://sum.golang.org/lookup/#{match[:name]}@#{match[:version]}")
+ end
+ end
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+ end
+ end
+end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index d1398ddb642..72dcc4fde71 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -225,6 +225,10 @@ module Gitlab
new_path.presence || old_path
end
+ def file_hash
+ Digest::SHA1.hexdigest(file_path)
+ end
+
def added_lines
@stats&.additions || diff_lines.count(&:added?)
end
@@ -237,6 +241,10 @@ module Gitlab
"#{file_path}-#{new_file?}-#{deleted_file?}-#{renamed_file?}"
end
+ def file_identifier_hash
+ Digest::SHA1.hexdigest(file_identifier)
+ end
+
def diffable?
repository.attributes(file_path).fetch('diff') { true }
end
diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb
index 9704aed82c1..31eeadc45f7 100644
--- a/lib/gitlab/diff/formatters/base_formatter.rb
+++ b/lib/gitlab/diff/formatters/base_formatter.rb
@@ -6,6 +6,7 @@ module Gitlab
class BaseFormatter
attr_reader :old_path
attr_reader :new_path
+ attr_reader :file_identifier_hash
attr_reader :base_sha
attr_reader :start_sha
attr_reader :head_sha
@@ -16,6 +17,7 @@ module Gitlab
attrs[:diff_refs] = diff_file.diff_refs
attrs[:old_path] = diff_file.old_path
attrs[:new_path] = diff_file.new_path
+ attrs[:file_identifier_hash] = diff_file.file_identifier_hash
end
if diff_refs = attrs[:diff_refs]
@@ -26,6 +28,7 @@ module Gitlab
@old_path = attrs[:old_path]
@new_path = attrs[:new_path]
+ @file_identifier_hash = attrs[:file_identifier_hash]
@base_sha = attrs[:base_sha]
@start_sha = attrs[:start_sha]
@head_sha = attrs[:head_sha]
@@ -36,7 +39,7 @@ module Gitlab
end
def to_h
- {
+ out = {
base_sha: base_sha,
start_sha: start_sha,
head_sha: head_sha,
@@ -44,6 +47,12 @@ module Gitlab
new_path: new_path,
position_type: position_type
}
+
+ if Feature.enabled?(:file_identifier_hash)
+ out[:file_identifier_hash] = file_identifier_hash
+ end
+
+ out
end
def position_type
diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb
index 10ad23b7774..e43f301c280 100644
--- a/lib/gitlab/diff/position.rb
+++ b/lib/gitlab/diff/position.rb
@@ -9,6 +9,7 @@ module Gitlab
delegate :old_path,
:new_path,
+ :file_identifier_hash,
:base_sha,
:start_sha,
:head_sha,
@@ -156,13 +157,23 @@ module Gitlab
position_type == 'text'
end
+ def find_diff_file_from(diffable)
+ diff_files = diffable.diffs(diff_options).diff_files
+
+ if Feature.enabled?(:file_identifier_hash) && file_identifier_hash.present?
+ diff_files.find { |df| df.file_identifier_hash == file_identifier_hash }
+ else
+ diff_files.first
+ end
+ end
+
private
def find_diff_file(repository)
return unless diff_refs.complete?
return unless comparison = diff_refs.compare_in(repository.project)
- comparison.diffs(diff_options).diff_files.first
+ find_diff_file_from(comparison)
end
def get_formatter_class(type)
diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb
new file mode 100644
index 00000000000..31c5dded3ff
--- /dev/null
+++ b/lib/gitlab/doctor/secrets.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Doctor
+ class Secrets
+ attr_reader :logger
+
+ def initialize(logger)
+ @logger = logger
+ end
+
+ def run!
+ logger.info "Checking encrypted values in the database"
+ Rails.application.eager_load! unless Rails.application.config.eager_load
+
+ models_with_attributes = Hash.new { |h, k| h[k] = [] }
+
+ models_with_encrypted_attributes.each do |model|
+ models_with_attributes[model] += model.encrypted_attributes.keys
+ end
+
+ models_with_encrypted_tokens.each do |model|
+ models_with_attributes[model] += model.encrypted_token_authenticatable_fields
+ end
+
+ check_model_attributes(models_with_attributes)
+
+ logger.info "Done!"
+ end
+
+ private
+
+ def check_model_attributes(models_with_attributes)
+ running_failures = 0
+
+ models_with_attributes.each do |model, attributes|
+ failures_per_row = Hash.new { |h, k| h[k] = [] }
+ model.find_each do |data|
+ attributes.each do |att|
+ failures_per_row[data.id] << att unless valid_attribute?(data, att)
+ end
+ end
+
+ running_failures += failures_per_row.keys.count
+ output_failures_for_model(model, failures_per_row)
+ end
+
+ logger.info "Total: #{running_failures} row(s) affected".color(:blue)
+ end
+
+ def output_failures_for_model(model, failures)
+ status_color = failures.empty? ? :green : :red
+
+ logger.info "- #{model} failures: #{failures.count}".color(status_color)
+ failures.each do |row_id, attributes|
+ logger.debug " - #{model}[#{row_id}]: #{attributes.join(", ")}".color(:red)
+ end
+ end
+
+ def models_with_encrypted_attributes
+ all_models.select { |d| d.encrypted_attributes.present? }
+ end
+
+ def models_with_encrypted_tokens
+ all_models.select do |d|
+ d.include?(TokenAuthenticatable) && d.encrypted_token_authenticatable_fields.present?
+ end
+ end
+
+ def all_models
+ @all_models ||= ApplicationRecord.descendants
+ end
+
+ def valid_attribute?(data, attr)
+ data.public_send(attr) # rubocop:disable GitlabSecurity/PublicSend
+
+ true
+ rescue OpenSSL::Cipher::CipherError, TypeError
+ false
+ rescue => e
+ logger.debug "> Something went wrong for #{data.class.name}[#{data.id}].#{attr}: #{e}".color(:red)
+
+ false
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb
index b893d625f8d..a19ce22e53f 100644
--- a/lib/gitlab/error_tracking.rb
+++ b/lib/gitlab/error_tracking.rb
@@ -26,10 +26,13 @@ module Gitlab
# Sanitize fields based on those sanitized from Rails.
config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s)
+ config.processors << ::Gitlab::ErrorTracking::Processor::SidekiqProcessor
# Sanitize authentication headers
config.sanitize_http_headers = %w[Authorization Private-Token]
config.tags = { program: Gitlab.process_name }
config.before_send = method(:before_send)
+
+ yield config if block_given?
end
end
diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb
new file mode 100644
index 00000000000..272cb689ad5
--- /dev/null
+++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Gitlab
+ module ErrorTracking
+ module Processor
+ class SidekiqProcessor < ::Raven::Processor
+ FILTERED_STRING = '[FILTERED]'
+
+ def self.filter_arguments(args, klass)
+ args.lazy.with_index.map do |arg, i|
+ case arg
+ when Numeric
+ arg
+ else
+ if permitted_arguments_for_worker(klass).include?(i)
+ arg
+ else
+ FILTERED_STRING
+ end
+ end
+ end
+ end
+
+ def self.permitted_arguments_for_worker(klass)
+ @permitted_arguments_for_worker ||= {}
+ @permitted_arguments_for_worker[klass] ||=
+ begin
+ klass.constantize&.loggable_arguments&.to_set
+ rescue
+ Set.new
+ end
+ end
+
+ def self.loggable_arguments(args, klass)
+ Gitlab::Utils::LogLimitedArray
+ .log_limited_array(filter_arguments(args, klass))
+ .map(&:to_s)
+ .to_a
+ end
+
+ def process(value, key = nil)
+ sidekiq = value.dig(:extra, :sidekiq)
+
+ return value unless sidekiq
+
+ sidekiq = sidekiq.deep_dup
+ sidekiq.delete(:jobstr)
+
+ # 'args' in this hash => from Gitlab::ErrorTracking.track_*
+ # 'args' in :job => from default error handler
+ job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job]
+
+ if job_holder['args']
+ job_holder['args'] = self.class.filter_arguments(job_holder['args'], job_holder['class']).to_a
+ end
+
+ value[:extra][:sidekiq] = sidekiq
+
+ value
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb
index 92d55213cc2..2da1b8915e4 100644
--- a/lib/gitlab/exception_log_formatter.rb
+++ b/lib/gitlab/exception_log_formatter.rb
@@ -12,6 +12,16 @@ module Gitlab
'exception.message' => exception.message
)
+ payload.delete('extra.server')
+
+ payload['extra.sidekiq'].tap do |value|
+ if value.is_a?(Hash) && value.key?('args')
+ value = value.dup
+ payload['extra.sidekiq']['args'] = Gitlab::ErrorTracking::Processor::SidekiqProcessor
+ .loggable_arguments(value['args'], value['class'])
+ end
+ end
+
if exception.backtrace
payload['exception.backtrace'] = Gitlab::BacktraceCleaner.clean_backtrace(exception.backtrace)
end
diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb
index 3495b4a0b72..d3df9be0d63 100644
--- a/lib/gitlab/experimentation.rb
+++ b/lib/gitlab/experimentation.rb
@@ -13,18 +13,20 @@
# To enable the experiment for 10% of the users:
#
# chatops: `/chatops run feature set experiment_key_experiment_percentage 10`
-# console: `Feature.get(:experiment_key_experiment_percentage).enable_percentage_of_time(10)`
+# console: `Feature.enable_percentage_of_time(:experiment_key_experiment_percentage, 10)`
#
# To disable the experiment:
#
# chatops: `/chatops run feature delete experiment_key_experiment_percentage`
-# console: `Feature.get(:experiment_key_experiment_percentage).remove`
+# console: `Feature.remove(:experiment_key_experiment_percentage)`
#
# To check the current rollout percentage:
#
# chatops: `/chatops run feature get experiment_key_experiment_percentage`
# console: `Feature.get(:experiment_key_experiment_percentage).percentage_of_time_value`
#
+
+# TODO: see https://gitlab.com/gitlab-org/gitlab/-/issues/217490
module Gitlab
module Experimentation
EXPERIMENTS = {
@@ -45,6 +47,12 @@ module Gitlab
},
upgrade_link_in_user_menu_a: {
tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA'
+ },
+ invite_members_version_a: {
+ tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA'
+ },
+ new_create_project_ui: {
+ tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi'
}
}.freeze
@@ -66,7 +74,6 @@ module Gitlab
cookies.permanent.signed[:experimentation_subject_id] = {
value: SecureRandom.uuid,
- domain: :all,
secure: ::Gitlab.config.gitlab.https,
httponly: true
}
@@ -179,7 +186,7 @@ module Gitlab
# When a feature does not exist, the `percentage_of_time_value` method will return 0
def experiment_percentage
- @experiment_percentage ||= Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value
+ @experiment_percentage ||= Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet
end
end
end
diff --git a/lib/gitlab/export/logger.rb b/lib/gitlab/export/logger.rb
new file mode 100644
index 00000000000..b3c05651cd4
--- /dev/null
+++ b/lib/gitlab/export/logger.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Export
+ class Logger < ::Gitlab::JsonLogger
+ def self.file_name_noext
+ 'exporter'
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb
index a9e9261cd3c..13959f6aa68 100644
--- a/lib/gitlab/file_detector.rb
+++ b/lib/gitlab/file_detector.rb
@@ -32,6 +32,8 @@ module Gitlab
gemfile_lock: 'Gemfile.lock',
gemspec: %r{\A[^/]*\.gemspec\z},
godeps_json: 'Godeps.json',
+ go_mod: 'go.mod',
+ go_sum: 'go.sum',
package_json: 'package.json',
podfile: 'Podfile',
podspec_json: %r{\A[^/]*\.podspec\.json\z},
diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb
index 23af0a9bb18..08321d5fda6 100644
--- a/lib/gitlab/gfm/uploads_rewriter.rb
+++ b/lib/gitlab/gfm/uploads_rewriter.rb
@@ -22,9 +22,10 @@ module Gitlab
return @text unless needs_rewrite?
@text.gsub(@pattern) do |markdown|
- Gitlab::Utils.check_path_traversal!($~[:file])
+ file = find_file($~[:secret], $~[:file])
+ # No file will be returned for a path traversal
+ next if file.nil?
- file = find_file(@source_project, $~[:secret], $~[:file])
break markdown unless file.try(:exists?)
klass = target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader
@@ -47,7 +48,7 @@ module Gitlab
def files
referenced_files = @text.scan(@pattern).map do
- find_file(@source_project, $~[:secret], $~[:file])
+ find_file($~[:secret], $~[:file])
end
referenced_files.compact.select(&:exists?)
@@ -57,12 +58,8 @@ module Gitlab
markdown.starts_with?("!")
end
- private
-
- def find_file(project, secret, file)
- uploader = FileUploader.new(project, secret: secret)
- uploader.retrieve_from_store!(file)
- uploader
+ def find_file(secret, file_name)
+ UploaderFinder.new(@source_project, secret, file_name).execute
end
end
end
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index a554dc0b667..17d0a62ba8c 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -7,6 +7,7 @@ module Gitlab
include Gitlab::EncodingHelper
prepend Gitlab::Git::RuggedImpl::Commit
extend Gitlab::Git::WrapsGitalyErrors
+ include Gitlab::Utils::StrongMemoize
attr_accessor :raw_commit, :head
@@ -231,6 +232,18 @@ module Gitlab
parent_ids.first
end
+ def committed_date
+ strong_memoize(:committed_date) do
+ init_date_from_gitaly(raw_commit.committer) if raw_commit
+ end
+ end
+
+ def authored_date
+ strong_memoize(:authored_date) do
+ init_date_from_gitaly(raw_commit.author) if raw_commit
+ end
+ end
+
# Returns a diff object for the changes from this commit's first parent.
# If there is no parent, then the diff is between this commit and an
# empty repo. See Repository#diff for keys allowed in the +options+
@@ -369,11 +382,9 @@ module Gitlab
# subject from the message to make it clearer when there's one
# available but not the other.
@message = message_from_gitaly_body
- @authored_date = init_date_from_gitaly(commit.author)
@author_name = commit.author.name.dup
@author_email = commit.author.email.dup
- @committed_date = init_date_from_gitaly(commit.committer)
@committer_name = commit.committer.name.dup
@committer_email = commit.committer.email.dup
@parent_ids = Array(commit.parent_ids)
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index 74a4633424f..bb845f11181 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -225,7 +225,7 @@ module Gitlab
end
def init_from_gitaly(diff)
- @diff = encode!(diff.patch) if diff.respond_to?(:patch)
+ @diff = diff.respond_to?(:patch) ? encode!(diff.patch) : ''
@new_path = encode!(diff.to_path.dup)
@old_path = encode!(diff.from_path.dup)
@a_mode = diff.old_mode.to_s(8)
diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb
index f9573bedba7..dae208e6955 100644
--- a/lib/gitlab/git/rugged_impl/use_rugged.rb
+++ b/lib/gitlab/git/rugged_impl/use_rugged.rb
@@ -5,8 +5,7 @@ module Gitlab
module RuggedImpl
module UseRugged
def use_rugged?(repo, feature_key)
- feature = Feature.get(feature_key)
- return feature.enabled? if Feature.persisted?(feature)
+ return Feature.enabled?(feature_key) if Feature.persisted_name?(feature_key)
# Disable Rugged auto-detect(can_use_disk?) when Puma threads>1
# https://gitlab.com/gitlab-org/gitlab/issues/119326
@@ -25,7 +24,7 @@ module Gitlab
if Gitlab::RuggedInstrumentation.active?
Gitlab::RuggedInstrumentation.increment_query_count
- Gitlab::RuggedInstrumentation.query_time += duration
+ Gitlab::RuggedInstrumentation.add_query_time(duration)
Gitlab::RuggedInstrumentation.add_call_details(
feature: method_name,
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index f1ca8900c30..37e3da984d6 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -8,7 +8,6 @@ module Gitlab
ForbiddenError = Class.new(StandardError)
NotFoundError = Class.new(StandardError)
- ProjectCreationError = Class.new(StandardError)
TimeoutError = Class.new(StandardError)
ProjectMovedError = Class.new(NotFoundError)
@@ -24,6 +23,7 @@ module Gitlab
deploy_key_upload: 'This deploy key does not have write access to this project.',
no_repo: 'A repository for this project does not exist yet.',
project_not_found: 'The project you were looking for could not be found.',
+ namespace_not_found: 'The namespace you were looking for could not be found.',
command_not_allowed: "The command you're trying to execute is not allowed.",
upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.',
receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.',
@@ -73,7 +73,8 @@ module Gitlab
return custom_action if custom_action
check_db_accessibility!(cmd)
- check_project!(changes, cmd)
+ check_namespace!
+ check_project!(cmd)
check_repository_existence!
case cmd
@@ -110,9 +111,7 @@ module Gitlab
private
- def check_project!(changes, cmd)
- check_namespace!
- ensure_project_on_push!(cmd, changes)
+ def check_project!(_cmd)
check_project_accessibility!
add_project_moved_message!
end
@@ -156,7 +155,7 @@ module Gitlab
def check_namespace!
return if namespace_path.present?
- raise NotFoundError, ERROR_MESSAGES[:project_not_found]
+ raise NotFoundError, ERROR_MESSAGES[:namespace_not_found]
end
def check_active_user!
@@ -229,32 +228,6 @@ module Gitlab
end
end
- def ensure_project_on_push!(cmd, changes)
- return if project || deploy_key?
- return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code)
-
- namespace = Namespace.find_by_full_path(namespace_path)
-
- return unless user&.can?(:create_projects, namespace)
-
- project_params = {
- path: repository_path,
- namespace_id: namespace.id,
- visibility_level: Gitlab::VisibilityLevel::PRIVATE
- }
-
- project = Projects::CreateService.new(user, project_params).execute
-
- unless project.saved?
- raise ProjectCreationError, "Could not create project: #{project.errors.full_messages.join(', ')}"
- end
-
- @project = project
- user_access.project = @project
-
- Checks::ProjectCreated.new(repository, user, protocol).add_message
- end
-
def check_repository_existence!
unless repository.exists?
raise NotFoundError, ERROR_MESSAGES[:no_repo]
diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb
new file mode 100644
index 00000000000..c79a61c263e
--- /dev/null
+++ b/lib/gitlab/git_access_project.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+ class GitAccessProject < GitAccess
+ extend ::Gitlab::Utils::Override
+
+ CreationError = Class.new(StandardError)
+
+ private
+
+ override :check_project!
+ def check_project!(cmd)
+ ensure_project_on_push!(cmd)
+
+ super
+ end
+
+ def ensure_project_on_push!(cmd)
+ return if project || deploy_key?
+ return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code)
+
+ namespace = Namespace.find_by_full_path(namespace_path)
+
+ return unless user&.can?(:create_projects, namespace)
+
+ project_params = {
+ path: repository_path,
+ namespace_id: namespace.id,
+ visibility_level: Gitlab::VisibilityLevel::PRIVATE
+ }
+
+ project = Projects::CreateService.new(user, project_params).execute
+
+ unless project.saved?
+ raise CreationError, "Could not create project: #{project.errors.full_messages.join(', ')}"
+ end
+
+ @project = project
+ user_access.project = @project
+
+ Checks::ProjectCreated.new(repository, user, protocol).add_message
+ end
+ end
+end
diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb
index 70db4271469..3de6c9ee30a 100644
--- a/lib/gitlab/git_access_snippet.rb
+++ b/lib/gitlab/git_access_snippet.rb
@@ -39,13 +39,18 @@ module Gitlab
private
+ override :check_namespace!
+ def check_namespace!
+ return unless snippet.is_a?(ProjectSnippet)
+
+ super
+ end
+
override :check_project!
- def check_project!(cmd, changes)
+ def check_project!(cmd)
return unless snippet.is_a?(ProjectSnippet)
- check_namespace!
- check_project_accessibility!
- add_project_moved_message!
+ super
end
override :check_push_access!
diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb
index 3c0dbba64bf..aad46937c32 100644
--- a/lib/gitlab/git_access_wiki.rb
+++ b/lib/gitlab/git_access_wiki.rb
@@ -2,6 +2,8 @@
module Gitlab
class GitAccessWiki < GitAccess
+ prepend_if_ee('EE::Gitlab::GitAccessWiki') # rubocop: disable Cop/InjectEnterpriseEditionModule
+
ERROR_MESSAGES = {
read_only: "You can't push code to a read-only GitLab instance.",
write_to_wiki: "You are not allowed to write to this project's wiki."
@@ -31,8 +33,10 @@ module Gitlab
ERROR_MESSAGES[:read_only]
end
- def container
- project.wiki
+ private
+
+ def repository
+ project.wiki.repository
end
end
end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 3aaed0edb87..bed99ef0ed4 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -201,7 +201,8 @@ module Gitlab
request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {}
# Keep track, separately, for the performance bar
- self.query_time += duration
+ self.add_query_time(duration)
+
if Gitlab::PerformanceBar.enabled_for_request?
add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc,
backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller))
@@ -209,12 +210,15 @@ module Gitlab
end
def self.query_time
- query_time = SafeRequestStore[:gitaly_query_time] ||= 0
+ query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0
query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
end
- def self.query_time=(duration)
- SafeRequestStore[:gitaly_query_time] = duration
+ def self.add_query_time(duration)
+ return unless Gitlab::SafeRequestStore.active?
+
+ Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0
+ Gitlab::SafeRequestStore[:gitaly_query_time] += duration
end
def self.current_transaction_labels
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 1f914dc95d1..aed132aaca0 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -212,8 +212,9 @@ module Gitlab
right_commit_id: right_commit_sha
)
- response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout)
- response.flat_map(&:stats)
+ GitalyClient.streaming_call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) do |response|
+ response.flat_map(&:stats)
+ end
end
def find_all_commits(opts = {})
@@ -246,8 +247,8 @@ module Gitlab
request = Gitaly::CommitsByMessageRequest.new(
repository: @gitaly_repo,
query: query,
- revision: revision.to_s.force_encoding(Encoding::ASCII_8BIT),
- path: path.to_s.force_encoding(Encoding::ASCII_8BIT),
+ revision: encode_binary(revision),
+ path: encode_binary(path),
limit: limit.to_i,
offset: offset.to_i
)
diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb
index 147597289cf..0d448b55104 100644
--- a/lib/gitlab/github_import/bulk_importing.rb
+++ b/lib/gitlab/github_import/bulk_importing.rb
@@ -17,7 +17,7 @@ module Gitlab
# Bulk inserts the given rows into the database.
def bulk_insert(model, rows, batch_size: 100)
rows.each_slice(batch_size) do |slice|
- Gitlab::Database.bulk_insert(model.table_name, slice)
+ Gitlab::Database.bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert
end
end
end
diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb
index d562958e955..53b17f77ccd 100644
--- a/lib/gitlab/github_import/importer/diff_note_importer.rb
+++ b/lib/gitlab/github_import/importer/diff_note_importer.rb
@@ -47,7 +47,7 @@ module Gitlab
# To work around this we're using bulk_insert with a single row. This
# allows us to efficiently insert data (even if it's just 1 row)
# without having to use all sorts of hacks to disable callbacks.
- Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes])
+ Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
rescue ActiveRecord::InvalidForeignKey
# It's possible the project and the issue have been deleted since
# scheduling this job. In this case we'll just skip creating the note.
diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb
index 8648cbaec9d..13061d2c9df 100644
--- a/lib/gitlab/github_import/importer/issue_importer.rb
+++ b/lib/gitlab/github_import/importer/issue_importer.rb
@@ -75,7 +75,7 @@ module Gitlab
end
end
- Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees)
+ Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert
end
end
end
diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb
index 2001b7e3482..77eb4542195 100644
--- a/lib/gitlab/github_import/importer/label_links_importer.rb
+++ b/lib/gitlab/github_import/importer/label_links_importer.rb
@@ -40,7 +40,7 @@ module Gitlab
}
end
- Gitlab::Database.bulk_insert(LabelLink.table_name, rows)
+ Gitlab::Database.bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert
end
def find_target_id
diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb
index 30763492235..5980b3c2179 100644
--- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb
+++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb
@@ -29,7 +29,10 @@ module Gitlab
yield object
end
rescue StandardError => e
- Rails.logger.error("The Lfs import process failed. #{e.message}") # rubocop:disable Gitlab/RailsLogger
+ Gitlab::Import::Logger.error(
+ message: 'The Lfs import process failed',
+ error: e.message
+ )
end
end
end
diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb
index 2b06d1b3baf..41f179d275b 100644
--- a/lib/gitlab/github_import/importer/note_importer.rb
+++ b/lib/gitlab/github_import/importer/note_importer.rb
@@ -38,7 +38,7 @@ module Gitlab
# We're using bulk_insert here so we can bypass any validations and
# callbacks. Running these would result in a lot of unnecessary SQL
# queries being executed when importing large projects.
- Gitlab::Database.bulk_insert(Note.table_name, [attributes])
+ Gitlab::Database.bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert
rescue ActiveRecord::InvalidForeignKey
# It's possible the project and the issue have been deleted since
# scheduling this job. In this case we'll just skip creating the note.
diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb
index 929fceaacf2..dcae8ca01fa 100644
--- a/lib/gitlab/github_import/importer/pull_requests_importer.rb
+++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb
@@ -40,8 +40,10 @@ module Gitlab
pname = project.path_with_namespace
- Rails.logger # rubocop:disable Gitlab/RailsLogger
- .info("GitHub importer finished updating repository for #{pname}")
+ Gitlab::Import::Logger.info(
+ message: 'GitHub importer finished updating repository',
+ project_name: pname
+ )
repository_updates_counter.increment
end
diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb
index 8abefad1ef3..abd4e847a50 100644
--- a/lib/gitlab/gl_repository.rb
+++ b/lib/gitlab/gl_repository.rb
@@ -6,20 +6,21 @@ module Gitlab
PROJECT = RepoType.new(
name: :project,
- access_checker_class: Gitlab::GitAccess,
+ access_checker_class: Gitlab::GitAccessProject,
repository_resolver: -> (project) { project&.repository }
).freeze
WIKI = RepoType.new(
name: :wiki,
access_checker_class: Gitlab::GitAccessWiki,
- repository_resolver: -> (project) { project&.wiki&.repository },
+ repository_resolver: -> (container) { container&.wiki&.repository },
+ project_resolver: -> (container) { container.is_a?(Project) ? container : nil },
suffix: :wiki
).freeze
SNIPPET = RepoType.new(
name: :snippet,
access_checker_class: Gitlab::GitAccessSnippet,
repository_resolver: -> (snippet) { snippet&.repository },
- container_resolver: -> (id) { Snippet.find_by_id(id) },
+ container_class: Snippet,
project_resolver: -> (snippet) { snippet&.project },
guest_read_ability: :read_snippet
).freeze
@@ -42,16 +43,12 @@ module Gitlab
end
def self.parse(gl_repository)
- type_name, _id = gl_repository.split('-').first
- type = types[type_name]
+ result = ::Gitlab::GlRepository::Identifier.new(gl_repository)
- unless type
- raise ArgumentError, "Invalid GL Repository \"#{gl_repository}\""
- end
+ repo_type = result.repo_type
+ container = result.fetch_container!
- container = type.fetch_container!(gl_repository)
-
- [container, type.project_for(container), type]
+ [container, repo_type.project_for(container), repo_type]
end
def self.default_type
diff --git a/lib/gitlab/gl_repository/identifier.rb b/lib/gitlab/gl_repository/identifier.rb
new file mode 100644
index 00000000000..dc3e7931696
--- /dev/null
+++ b/lib/gitlab/gl_repository/identifier.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+ class GlRepository
+ class Identifier
+ attr_reader :gl_repository, :repo_type
+
+ def initialize(gl_repository)
+ @gl_repository = gl_repository
+ @segments = gl_repository.split('-')
+
+ raise_error if segments.size > 3
+
+ @repo_type = find_repo_type
+ @container_id = find_container_id
+ @container_class = find_container_class
+ end
+
+ def fetch_container!
+ container_class.find_by_id(container_id)
+ end
+
+ private
+
+ attr_reader :segments, :container_class, :container_id
+
+ def find_repo_type
+ type_name = three_segments_format? ? segments.last : segments.first
+ type = Gitlab::GlRepository.types[type_name]
+
+ raise_error unless type
+
+ type
+ end
+
+ def find_container_class
+ if three_segments_format?
+ case segments[0]
+ when 'project'
+ Project
+ when 'group'
+ Group
+ else
+ raise_error
+ end
+ else
+ repo_type.container_class
+ end
+ end
+
+ def find_container_id
+ id = Integer(segments[1], 10, exception: false)
+
+ raise_error unless id
+
+ id
+ end
+
+ # gl_repository can either have 2 or 3 segments:
+ # "wiki-1" is the older 2-segment format, where container is implied.
+ # "group-1-wiki" is the newer 3-segment format, including container information.
+ #
+ # TODO: convert all 2-segment format to 3-segment:
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/219192
+ def three_segments_format?
+ segments.size == 3
+ end
+
+ def raise_error
+ raise ArgumentError, "Invalid GL Repository \"#{gl_repository}\""
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb
index 64c329b15ae..2b482ee3d2d 100644
--- a/lib/gitlab/gl_repository/repo_type.rb
+++ b/lib/gitlab/gl_repository/repo_type.rb
@@ -6,7 +6,7 @@ module Gitlab
attr_reader :name,
:access_checker_class,
:repository_resolver,
- :container_resolver,
+ :container_class,
:project_resolver,
:guest_read_ability,
:suffix
@@ -15,34 +15,25 @@ module Gitlab
name:,
access_checker_class:,
repository_resolver:,
- container_resolver: default_container_resolver,
+ container_class: default_container_class,
project_resolver: nil,
guest_read_ability: :download_code,
suffix: nil)
@name = name
@access_checker_class = access_checker_class
@repository_resolver = repository_resolver
- @container_resolver = container_resolver
+ @container_class = container_class
@project_resolver = project_resolver
@guest_read_ability = guest_read_ability
@suffix = suffix
end
def identifier_for_container(container)
- "#{name}-#{container.id}"
- end
-
- def fetch_id(identifier)
- match = /\A#{name}-(?<id>\d+)\z/.match(identifier)
- match[:id] if match
- end
+ if container.is_a?(Group)
+ return "#{container.class.name.underscore}-#{container.id}-#{name}"
+ end
- def fetch_container!(identifier)
- id = fetch_id(identifier)
-
- raise ArgumentError, "Invalid GL Repository \"#{identifier}\"" unless id
-
- container_resolver.call(id)
+ "#{name}-#{container.id}"
end
def wiki?
@@ -85,8 +76,8 @@ module Gitlab
private
- def default_container_resolver
- -> (id) { Project.find_by_id(id) }
+ def default_container_class
+ Project
end
end
end
diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb
new file mode 100644
index 00000000000..f2dc668c482
--- /dev/null
+++ b/lib/gitlab/golang.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Golang
+ extend self
+
+ def local_module_prefix
+ @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/".freeze
+ end
+
+ def semver_tag?(tag)
+ return false if tag.dereferenced_target.nil?
+
+ Packages::SemVer.match?(tag.name, prefixed: true)
+ end
+
+ def pseudo_version?(version)
+ return false unless version
+
+ if version.is_a? String
+ version = parse_semver version
+ return false unless version
+ end
+
+ pre = version.prerelease
+
+ # Valid pseudo-versions are:
+ # vX.0.0-yyyymmddhhmmss-sha1337beef0, when no earlier tagged commit exists for X
+ # vX.Y.Z-pre.0.yyyymmddhhmmss-sha1337beef0, when most recent prior tag is vX.Y.Z-pre
+ # vX.Y.(Z+1)-0.yyyymmddhhmmss-sha1337beef0, when most recent prior tag is vX.Y.Z
+
+ if version.minor != 0 || version.patch != 0
+ m = /\A(.*\.)?0\./.freeze.match pre
+ return false unless m
+
+ pre = pre[m[0].length..]
+ end
+
+ # This pattern is intentionally more forgiving than the patterns
+ # above. Correctness is verified by #pseudo_version_commit.
+ /\A\d{14}-\h+\z/.freeze.match? pre
+ end
+
+ def pseudo_version_commit(project, semver)
+ # Per Go's implementation of pseudo-versions, a tag should be
+ # considered a pseudo-version if it matches one of the patterns
+ # listed in #pseudo_version?, regardless of the content of the
+ # timestamp or the length of the SHA fragment. However, an error
+ # should be returned if the timestamp is not correct or if the SHA
+ # fragment is not exactly 12 characters long. See also Go's
+ # implementation of:
+ #
+ # - [*codeRepo.validatePseudoVersion](https://github.com/golang/go/blob/daf70d6c1688a1ba1699c933b3c3f04d6f2f73d9/src/cmd/go/internal/modfetch/coderepo.go#L530)
+ # - [Pseudo-version parsing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/pseudo.go)
+ # - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go)
+
+ # Go ignores anything before '.' or after the second '-', so we will do the same
+ timestamp, sha = semver.prerelease.split('-').last 2
+ timestamp = timestamp.split('.').last
+ commit = project.repository.commit_by(oid: sha)
+
+ # Error messages are based on the responses of proxy.golang.org
+
+ # Verify that the SHA fragment references a commit
+ raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit
+
+ # Require the SHA fragment to be 12 characters long
+ raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12
+
+ # Require the timestamp to match that of the commit
+ raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp
+
+ commit
+ end
+
+ def parse_semver(str)
+ Packages::SemVer.parse(str, prefixed: true)
+ end
+
+ def pkg_go_dev_url(name, version = nil)
+ if version
+ "https://pkg.go.dev/#{name}@#{version}"
+ else
+ "https://pkg.go.dev/#{name}"
+ end
+ end
+
+ def package_url(name, version = nil)
+ return unless UrlSanitizer.valid?("https://#{name}")
+
+ return pkg_go_dev_url(name, version) unless name.starts_with?(local_module_prefix)
+
+ # This will not work if `name` refers to a subdirectory of a project. This
+ # could be expanded with logic similar to Gitlab::Middleware::Go to locate
+ # the project, check for permissions, and return a smarter result.
+ "#{Gitlab.config.gitlab.protocol}://#{name}/"
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb
index 61668b634fd..cbf3e7b8429 100644
--- a/lib/gitlab/graphql/authorize/authorize_field_service.rb
+++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb
@@ -84,13 +84,25 @@ module Gitlab
elsif resolved_type.is_a? Array
# A simple list of rendered types each object being an object to authorize
resolved_type.select do |single_object_type|
- allowed_access?(current_user, single_object_type.object)
+ allowed_access?(current_user, realized(single_object_type).object)
end
else
raise "Can't authorize #{@field}"
end
end
+ # Ensure that we are dealing with realized objects, not delayed promises
+ def realized(thing)
+ case thing
+ when BatchLoader::GraphQL
+ thing.sync
+ when GraphQL::Execution::Lazy
+ thing.value # part of the private api, but we need to unwrap it here.
+ else
+ thing
+ end
+ end
+
def allowed_access?(current_user, object)
object = object.sync if object.respond_to?(:sync)
diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb
index f9ff2b30eae..15ecc3b04f0 100644
--- a/lib/gitlab/graphql/authorize/instrumentation.rb
+++ b/lib/gitlab/graphql/authorize/instrumentation.rb
@@ -9,16 +9,12 @@ module Gitlab
def instrument(_type, field)
service = AuthorizeFieldService.new(field)
- if service.authorizations? && !resolver_skips_authorizations?(field)
+ if service.authorizations?
field.redefine { resolve(service.authorized_resolve) }
else
field
end
end
-
- def resolver_skips_authorizations?(field)
- field.metadata[:resolver].try(:skip_authorizations?)
- end
end
end
end
diff --git a/lib/gitlab/graphql/filterable_array.rb b/lib/gitlab/graphql/filterable_array.rb
deleted file mode 100644
index 4909d291fd6..00000000000
--- a/lib/gitlab/graphql/filterable_array.rb
+++ /dev/null
@@ -1,14 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Graphql
- class FilterableArray < Array
- attr_reader :filter_callback
-
- def initialize(filter_callback, *args)
- super(args)
- @filter_callback = filter_callback
- end
- end
- end
-end
diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb
new file mode 100644
index 00000000000..0aa237c78de
--- /dev/null
+++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Graphql
+ module Loaders
+ # Suitable for use to find resources that expose `where_full_path_in`,
+ # such as Project, Group, Namespace
+ class FullPathModelLoader
+ attr_reader :model_class, :full_path
+
+ def initialize(model_class, full_path)
+ @model_class, @full_path = model_class, full_path
+ end
+
+ def find
+ BatchLoader::GraphQL.for(full_path).batch(key: model_class) do |full_paths, loader, args|
+ # `with_route` avoids an N+1 calculating full_path
+ args[:key].where_full_path_in(full_paths).with_route.each do |model_instance|
+ loader.call(model_instance.full_path, model_instance)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb
index febdc938317..8f37fa3f474 100644
--- a/lib/gitlab/graphql/pagination/connections.rb
+++ b/lib/gitlab/graphql/pagination/connections.rb
@@ -10,10 +10,6 @@ module Gitlab
Gitlab::Graphql::Pagination::Keyset::Connection)
schema.connections.add(
- Gitlab::Graphql::FilterableArray,
- Gitlab::Graphql::Pagination::FilterableArrayConnection)
-
- schema.connections.add(
Gitlab::Graphql::ExternallyPaginatedArray,
Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection)
end
diff --git a/lib/gitlab/graphql/pagination/filterable_array_connection.rb b/lib/gitlab/graphql/pagination/filterable_array_connection.rb
deleted file mode 100644
index 4a76cd5fb00..00000000000
--- a/lib/gitlab/graphql/pagination/filterable_array_connection.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Graphql
- module Pagination
- # FilterableArrayConnection is useful especially for lazy-loaded values.
- # It allows us to call a callback only on the slice of array being
- # rendered in the "after loaded" phase. For example we can check
- # permissions only on a small subset of items.
- class FilterableArrayConnection < GraphQL::Pagination::ArrayConnection
- def nodes
- @nodes ||= items.filter_callback.call(super)
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb
index 1a32ab468b1..17cd22d38ad 100644
--- a/lib/gitlab/graphql/pagination/keyset/connection.rb
+++ b/lib/gitlab/graphql/pagination/keyset/connection.rb
@@ -32,6 +32,49 @@ module Gitlab
class Connection < GraphQL::Pagination::ActiveRecordRelationConnection
include Gitlab::Utils::StrongMemoize
+ # rubocop: disable Naming/PredicateName
+ # https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo.Fields
+ def has_previous_page
+ strong_memoize(:has_previous_page) do
+ if after
+ # If `after` is specified, that points to a specific record,
+ # even if it's the first one. Since we're asking for `after`,
+ # then the specific record we're pointing to is in the
+ # previous page
+ true
+ elsif last
+ limited_nodes
+ !!@has_previous_page
+ else
+ # Key thing to remember. When `before` is specified (and no `last`),
+ # the spec says return _all_ edges minus anything after the `before`.
+ # Which means the returned list starts at the very first record.
+ # Then the max_page kicks in, and returns the first max_page items.
+ # Because of this, `has_previous_page` will be false
+ false
+ end
+ end
+ end
+
+ def has_next_page
+ strong_memoize(:has_next_page) do
+ if before
+ # If `before` is specified, that points to a specific record,
+ # even if it's the last one. Since we're asking for `before`,
+ # then the specific record we're pointing to is in the
+ # next page
+ true
+ elsif first
+ # If we count the number of requested items plus one (`limit_value + 1`),
+ # then if we get `limit_value + 1` then we know there is a next page
+ relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1
+ else
+ false
+ end
+ end
+ end
+ # rubocop: enable Naming/PredicateName
+
def cursor_for(node)
encoded_json_from_ordering(node)
end
@@ -39,7 +82,7 @@ module Gitlab
def sliced_nodes
@sliced_nodes ||=
begin
- OrderInfo.validate_ordering(ordered_items, order_list)
+ OrderInfo.validate_ordering(ordered_items, order_list) unless loaded?(ordered_items)
sliced = ordered_items
sliced = slice_nodes(sliced, before, :before) if before.present?
@@ -54,20 +97,30 @@ module Gitlab
# So we're ok loading them into memory here as that's bound to happen
# anyway. Having them ready means we can modify the result while
# rendering the fields.
- @nodes ||= load_paged_nodes.to_a
+ @nodes ||= limited_nodes.to_a
end
private
- def load_paged_nodes
- if first && last
- raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both")
- end
+ # Apply `first` and `last` to `sliced_nodes`
+ def limited_nodes
+ strong_memoize(:limited_nodes) do
+ if first && last
+ raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both")
+ end
- if last
- sliced_nodes.last(limit_value)
- else
- sliced_nodes.limit(limit_value) # rubocop: disable CodeReuse/ActiveRecord
+ if last
+ # grab one more than we need
+ paginated_nodes = sliced_nodes.last(limit_value + 1)
+
+ # there is an extra node, so there is a previous page
+ @has_previous_page = paginated_nodes.count > limit_value
+ @has_previous_page ? paginated_nodes.last(limit_value) : paginated_nodes
+ elsif loaded?(sliced_nodes)
+ sliced_nodes.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord
+ else
+ sliced_nodes.limit(limit_value) # rubocop: disable CodeReuse/ActiveRecord
+ end
end
end
@@ -82,9 +135,19 @@ module Gitlab
# rubocop: enable CodeReuse/ActiveRecord
def limit_value
+ # note: only first _or_ last can be specified, not both
@limit_value ||= [first, last, max_page_size].compact.min
end
+ def loaded?(items)
+ case items
+ when Array
+ true
+ else
+ items.loaded?
+ end
+ end
+
def ordered_items
strong_memoize(:ordered_items) do
unless items.primary_key.present?
@@ -93,6 +156,16 @@ module Gitlab
list = OrderInfo.build_order_list(items)
+ if loaded?(items)
+ @order_list = list.presence || [items.primary_key]
+
+ # already sorted, or trivially sorted
+ next items if list.present? || items.size <= 1
+
+ pkey = items.primary_key.to_sym
+ next items.sort_by { |item| item[pkey] }.reverse
+ end
+
# ensure there is a primary key ordering
if list&.last&.attribute_name != items.primary_key
items.order(arel_table[items.primary_key].desc) # rubocop: disable CodeReuse/ActiveRecord
@@ -121,7 +194,12 @@ module Gitlab
order_list.each do |field|
field_name = field.attribute_name
- ordering[field_name] = node[field_name].to_s
+ field_value = node[field_name]
+ ordering[field_name] = if field_value.is_a?(Time)
+ field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z')
+ else
+ field_value.to_s
+ end
end
encode(ordering.to_json)
diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb
index aaade39dd62..f8ea7a7adcd 100644
--- a/lib/gitlab/import/database_helpers.rb
+++ b/lib/gitlab/import/database_helpers.rb
@@ -11,7 +11,7 @@ module Gitlab
# We use bulk_insert here so we can bypass any queries executed by
# callbacks or validation rules, as doing this wouldn't scale when
# importing very large projects.
- result = Gitlab::Database
+ result = Gitlab::Database # rubocop:disable Gitlab/BulkInsert
.bulk_insert(relation.table_name, [attributes], return_ids: true)
result.first
diff --git a/lib/gitlab/import/set_async_jid.rb b/lib/gitlab/import/set_async_jid.rb
index 3b11c92fb56..527d84477fe 100644
--- a/lib/gitlab/import/set_async_jid.rb
+++ b/lib/gitlab/import/set_async_jid.rb
@@ -2,8 +2,8 @@
# The original import JID is the JID of the RepositoryImportWorker job,
# which will be removed once that job completes. Reusing that JID could
-# result in StuckImportJobsWorker marking the job as stuck before we get
-# to running Stage::ImportRepositoryWorker.
+# result in Gitlab::Import::StuckProjectImportJobsWorker marking the job
+# as stuck before we get to running Stage::ImportRepositoryWorker.
#
# We work around this by setting the JID to a custom generated one, then
# refreshing it in the various stages whenever necessary.
@@ -13,8 +13,7 @@ module Gitlab
def self.set_jid(import_state)
jid = generate_jid(import_state)
- Gitlab::SidekiqStatus
- .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION)
+ Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION)
import_state.update_column(:jid, jid)
end
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb
index bab473741b1..1e98595bb07 100644
--- a/lib/gitlab/import_export/attributes_finder.rb
+++ b/lib/gitlab/import_export/attributes_finder.rb
@@ -3,6 +3,8 @@
module Gitlab
module ImportExport
class AttributesFinder
+ attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads
+
def initialize(config:)
@tree = config[:tree] || {}
@included_attributes = config[:included_attributes] || {}
diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb
new file mode 100644
index 00000000000..86f51add504
--- /dev/null
+++ b/lib/gitlab/import_export/attributes_permitter.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# AttributesPermitter builds a hash of permitted attributes for
+# every model defined in import_export.yml that is used to validate and
+# filter out any attributes that are not permitted when doing Project/Group Import
+#
+# Each model's list includes:
+# - attributes defined under included_attributes section
+# - associations defined under project/group tree
+# - methods defined under methods section
+#
+# Given the following import_export.yml example:
+# ```
+# tree:
+# project:
+# - labels:
+# - :priorities
+# included_attributes:
+# labels:
+# - :title
+# - :description
+# methods:
+# labels:
+# - :type
+# ```
+#
+# Produces a list of permitted attributes:
+# ```
+# Gitlab::ImportExport::AttributesPermitter.new.permitted_attributes
+#
+# => { labels: [:priorities, :title, :description, :type] }
+# ```
+#
+# Filters out any other attributes from specific relation hash:
+# ```
+# Gitlab::ImportExport::AttributesPermitter.new.permit(:labels, {id: 5, type: 'opened', description: 'test', sensitive_attribute: 'my_sensitive_attribute'})
+#
+# => {:type=>"opened", :description=>"test"}
+# ```
+module Gitlab
+ module ImportExport
+ class AttributesPermitter
+ attr_reader :permitted_attributes
+
+ def initialize(config: ImportExport::Config.new.to_h)
+ @config = config
+ @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config)
+ @permitted_attributes = {}
+
+ build_permitted_attributes
+ end
+
+ def permit(relation_name, relation_hash)
+ permitted_attributes = permitted_attributes_for(relation_name)
+
+ relation_hash.select do |key, _|
+ permitted_attributes.include?(key)
+ end
+ end
+
+ def permitted_attributes_for(relation_name)
+ @permitted_attributes[relation_name] || []
+ end
+
+ private
+
+ def build_permitted_attributes
+ build_associations
+ build_attributes
+ build_methods
+ end
+
+ # Deep traverse relations tree to build a list of allowed model relations
+ def build_associations
+ stack = @attributes_finder.tree.to_a
+
+ while stack.any?
+ model_name, relations = stack.pop
+
+ if relations.is_a?(Hash)
+ add_permitted_attributes(model_name, relations.keys)
+
+ stack.concat(relations.to_a)
+ end
+ end
+
+ @permitted_attributes
+ end
+
+ def build_attributes
+ @attributes_finder.included_attributes.each(&method(:add_permitted_attributes))
+ end
+
+ def build_methods
+ @attributes_finder.methods.each(&method(:add_permitted_attributes))
+ end
+
+ def add_permitted_attributes(model_name, attributes)
+ @permitted_attributes[model_name] ||= []
+
+ @permitted_attributes[model_name].concat(attributes) if attributes.any?
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index b1219384732..7b8689069d8 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -24,8 +24,14 @@ module Gitlab
raise Projects::ImportService::Error.new(shared.errors.to_sentence)
end
rescue => e
+ # If some exception was raised could mean that the SnippetsRepoRestorer
+ # was not called. This would leave us with snippets without a repository.
+ # This is a state we don't want them to be, so we better delete them.
+ remove_non_migrated_snippets
+
raise Projects::ImportService::Error.new(e.message)
ensure
+ remove_base_tmp_dir
remove_import_file
end
@@ -148,6 +154,18 @@ module Gitlab
::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
end
end
+
+ def remove_base_tmp_dir
+ FileUtils.rm_rf(@shared.base_path)
+ end
+
+ def remove_non_migrated_snippets
+ project
+ .snippets
+ .left_joins(:snippet_repository)
+ .where(snippet_repositories: { snippet_id: nil })
+ .delete_all
+ end
end
end
end
diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb
index 7f55a0a3821..20f9c668b9c 100644
--- a/lib/gitlab/import_export/json/streaming_serializer.rb
+++ b/lib/gitlab/import_export/json/streaming_serializer.rb
@@ -7,6 +7,15 @@ module Gitlab
include Gitlab::ImportExport::CommandLineUtil
BATCH_SIZE = 100
+ SMALLER_BATCH_SIZE = 20
+
+ def self.batch_size(exportable)
+ if Feature.enabled?(:export_reduce_relation_batch_size, exportable)
+ SMALLER_BATCH_SIZE
+ else
+ BATCH_SIZE
+ end
+ end
class Raw < String
def to_json(*_args)
@@ -60,7 +69,7 @@ module Gitlab
key_preloads = preloads&.dig(key)
records = records.preload(key_preloads) if key_preloads
- records.find_each(batch_size: BATCH_SIZE) do |record|
+ records.find_each(batch_size: batch_size) do |record|
items << Raw.new(record.to_json(options))
end
end
@@ -91,6 +100,10 @@ module Gitlab
def preloads
relations_schema[:preload]
end
+
+ def batch_size
+ @batch_size ||= self.class.batch_size(@exportable)
+ end
end
end
end
diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
index cf75a2c7fa8..f8b8b74ffd7 100644
--- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb
+++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
@@ -7,7 +7,7 @@ module Gitlab
def serialize(exportable, relations_tree)
Gitlab::ImportExport::FastHashSerializer
- .new(exportable, relations_tree)
+ .new(exportable, relations_tree, batch_size: batch_size(exportable))
.execute
end
@@ -18,6 +18,12 @@ module Gitlab
File.write(File.join(dir_path, filename), tree_json)
end
+
+ private
+
+ def batch_size(exportable)
+ Gitlab::ImportExport::JSON::StreamingSerializer.batch_size(exportable)
+ end
end
end
end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index 263c49c509f..31d1f7b48bd 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -49,7 +49,7 @@ module Gitlab
def ensure_default_member!
return if user_already_member?
- @importable.members.destroy_all # rubocop: disable DestroyAll
+ @importable.members.destroy_all # rubocop: disable Cop/DestroyAll
relation_class.create!(user: @user, access_level: highest_access_level, source_id: @importable.id, importing: true)
rescue => e
diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb
index f735b9612aa..4643742b607 100644
--- a/lib/gitlab/import_export/merge_request_parser.rb
+++ b/lib/gitlab/import_export/merge_request_parser.rb
@@ -41,7 +41,13 @@ module Gitlab
def create_source_branch
@project.repository.create_branch(@merge_request.source_branch, @diff_head_sha)
rescue => err
- Rails.logger.warn("Import/Export warning: Failed to create source branch #{@merge_request.source_branch} => #{@diff_head_sha} for MR #{@merge_request.iid}: #{err}") # rubocop:disable Gitlab/RailsLogger
+ Gitlab::Import::Logger.warn(
+ message: 'Import warning: Failed to create source branch',
+ source_branch: @merge_request.source_branch,
+ diff_head_sha: @diff_head_sha,
+ merge_request_iid: @merge_request.iid,
+ error: err.message
+ )
end
def create_target_branch
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 8851b106ad5..f0b733d7e95 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -312,6 +312,7 @@ excluded_attributes:
- :pipeline_schedule_id
- :merge_request_id
- :external_pull_request_id
+ - :ci_ref_id
stages:
- :pipeline_id
merge_access_levels:
@@ -397,3 +398,4 @@ ee:
- protected_environments:
- :deploy_access_levels
- :service_desk_setting
+ - :security_setting
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index 3123687453f..9e10e7aea13 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -16,6 +16,8 @@ module Gitlab
repository.create_from_bundle(path_to_bundle)
rescue => e
+ Repositories::DestroyService.new(repository).execute
+
shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index ae82c380755..e4724659eff 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -11,14 +11,16 @@ module Gitlab
def initialize(exportable:, shared:)
@exportable = exportable
- @shared = shared
+ @shared = shared
end
def save
if compress_and_save
- remove_export_path
-
- Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
+ Gitlab::Export::Logger.info(
+ message: 'Export archive saved',
+ exportable_class: @exportable.class.to_s,
+ archive_file: archive_file
+ )
save_upload
else
@@ -29,8 +31,7 @@ module Gitlab
@shared.error(e)
false
ensure
- remove_archive
- remove_export_path
+ remove_base_tmp_dir
end
private
@@ -39,12 +40,8 @@ module Gitlab
tar_czf(archive: archive_file, dir: @shared.export_path)
end
- def remove_export_path
- FileUtils.rm_rf(@shared.export_path)
- end
-
- def remove_archive
- FileUtils.rm_rf(@shared.archive_path)
+ def remove_base_tmp_dir
+ FileUtils.rm_rf(@shared.base_path)
end
def archive_file
diff --git a/lib/gitlab/import_export/snippet_repo_restorer.rb b/lib/gitlab/import_export/snippet_repo_restorer.rb
index b58ea14a3a8..334d13a13ae 100644
--- a/lib/gitlab/import_export/snippet_repo_restorer.rb
+++ b/lib/gitlab/import_export/snippet_repo_restorer.rb
@@ -5,6 +5,8 @@ module Gitlab
class SnippetRepoRestorer < RepoRestorer
attr_reader :snippet
+ SnippetRepositoryError = Class.new(StandardError)
+
def initialize(snippet:, user:, shared:, path_to_bundle:)
@snippet = snippet
@user = user
@@ -34,14 +36,11 @@ module Gitlab
end
def create_repository_from_db
- snippet.create_repository
-
- commit_attrs = {
- branch_name: 'master',
- message: 'Initial commit'
- }
+ Gitlab::BackgroundMigration::BackfillSnippetRepositories.new.perform_by_ids([snippet.id])
- repository.create_file(@user, snippet.file_name, snippet.content, commit_attrs)
+ unless snippet.reset.snippet_repository
+ raise SnippetRepositoryError, _("Error creating repository for snippet with id %{snippet_id}") % { snippet_id: snippet.id }
+ end
end
end
end
diff --git a/lib/gitlab/import_export/snippets_repo_restorer.rb b/lib/gitlab/import_export/snippets_repo_restorer.rb
index 9ff3e74a6b1..5ab28f8dd83 100644
--- a/lib/gitlab/import_export/snippets_repo_restorer.rb
+++ b/lib/gitlab/import_export/snippets_repo_restorer.rb
@@ -10,15 +10,13 @@ module Gitlab
end
def restore
- return true unless Dir.exist?(snippets_repo_bundle_path)
-
- @project.snippets.find_each.all? do |snippet|
+ @project.snippets.find_each.map do |snippet|
Gitlab::ImportExport::SnippetRepoRestorer.new(snippet: snippet,
user: @user,
shared: @shared,
path_to_bundle: snippet_repo_bundle_path(snippet))
.restore
- end
+ end.all?(true)
end
private
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
index 86ea7a30e69..4154d4fe775 100644
--- a/lib/gitlab/import_export/version_checker.rb
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -36,7 +36,11 @@ module Gitlab
def different_version?(version)
Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version)
rescue => e
- Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
+ Gitlab::Import::Logger.error(
+ message: 'Import error',
+ error: e.message
+ )
+
raise Gitlab::ImportExport::Error.new('Incorrect VERSION format')
end
end
diff --git a/lib/gitlab/instrumentation/elasticsearch_transport.rb b/lib/gitlab/instrumentation/elasticsearch_transport.rb
new file mode 100644
index 00000000000..deee0127c0c
--- /dev/null
+++ b/lib/gitlab/instrumentation/elasticsearch_transport.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'elasticsearch-transport'
+
+module Gitlab
+ module Instrumentation
+ module ElasticsearchTransportInterceptor
+ def perform_request(*args)
+ start = Time.now
+ super
+ ensure
+ if ::Gitlab::SafeRequestStore.active?
+ duration = (Time.now - start)
+
+ ::Gitlab::Instrumentation::ElasticsearchTransport.increment_request_count
+ ::Gitlab::Instrumentation::ElasticsearchTransport.add_duration(duration)
+ ::Gitlab::Instrumentation::ElasticsearchTransport.add_call_details(duration, args)
+ end
+ end
+ end
+
+ class ElasticsearchTransport
+ ELASTICSEARCH_REQUEST_COUNT = :elasticsearch_request_count
+ ELASTICSEARCH_CALL_DURATION = :elasticsearch_call_duration
+ ELASTICSEARCH_CALL_DETAILS = :elasticsearch_call_details
+
+ def self.get_request_count
+ ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] || 0
+ end
+
+ def self.increment_request_count
+ ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] ||= 0
+ ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] += 1
+ end
+
+ def self.detail_store
+ ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DETAILS] ||= []
+ end
+
+ def self.query_time
+ query_time = ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] || 0
+ query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION)
+ end
+
+ def self.add_duration(duration)
+ ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] ||= 0
+ ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] += duration
+ end
+
+ def self.add_call_details(duration, args)
+ return unless Gitlab::PerformanceBar.enabled_for_request?
+
+ detail_store << {
+ method: args[0],
+ path: args[1],
+ params: args[2],
+ body: args[3],
+ duration: duration,
+ backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller)
+ }
+ end
+ end
+ end
+end
+
+class ::Elasticsearch::Transport::Client
+ prepend ::Gitlab::Instrumentation::ElasticsearchTransportInterceptor
+end
diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb
index cc99e828251..82b4701872f 100644
--- a/lib/gitlab/instrumentation/redis.rb
+++ b/lib/gitlab/instrumentation/redis.rb
@@ -1,67 +1,46 @@
# frozen_string_literal: true
-require 'redis'
-
module Gitlab
module Instrumentation
- module RedisInterceptor
- def call(*args, &block)
- start = Time.now
- super(*args, &block)
- ensure
- duration = (Time.now - start)
-
- if ::RequestStore.active?
- ::Gitlab::Instrumentation::Redis.increment_request_count
- ::Gitlab::Instrumentation::Redis.add_duration(duration)
- ::Gitlab::Instrumentation::Redis.add_call_details(duration, args)
- end
- end
- end
-
+ # Aggregates Redis measurements from different request storage sources.
class Redis
- REDIS_REQUEST_COUNT = :redis_request_count
- REDIS_CALL_DURATION = :redis_call_duration
- REDIS_CALL_DETAILS = :redis_call_details
+ ActionCable = Class.new(RedisBase)
+ Cache = Class.new(RedisBase)
+ Queues = Class.new(RedisBase)
+ SharedState = Class.new(RedisBase)
- def self.get_request_count
- ::RequestStore[REDIS_REQUEST_COUNT] || 0
- end
+ STORAGES = [ActionCable, Cache, Queues, SharedState].freeze
- def self.increment_request_count
- ::RequestStore[REDIS_REQUEST_COUNT] ||= 0
- ::RequestStore[REDIS_REQUEST_COUNT] += 1
- end
+ # Milliseconds represented in seconds (from 1 millisecond to 2 seconds).
+ QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze
- def self.detail_store
- ::RequestStore[REDIS_CALL_DETAILS] ||= []
- end
+ class << self
+ include ::Gitlab::Instrumentation::RedisPayload
- def self.query_time
- query_time = ::RequestStore[REDIS_CALL_DURATION] || 0
- query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION)
- end
+ def storage_key
+ nil
+ end
- def self.add_duration(duration)
- total_time = query_time + duration
- ::RequestStore[REDIS_CALL_DURATION] = total_time
- end
+ def known_payload_keys
+ super + STORAGES.flat_map(&:known_payload_keys)
+ end
- def self.add_call_details(duration, args)
- return unless Gitlab::PerformanceBar.enabled_for_request?
- # redis-rb passes an array (e.g. [:get, key])
- return unless args.length == 1
+ def payload
+ super.merge(*STORAGES.flat_map(&:payload))
+ end
- detail_store << {
- cmd: args.first,
- duration: duration,
- backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller)
- }
+ def detail_store
+ STORAGES.flat_map do |storage|
+ storage.detail_store.map { |details| details.merge(storage: storage.name.demodulize) }
+ end
+ end
+
+ %i[get_request_count query_time read_bytes write_bytes].each do |method|
+ define_method method do
+ STORAGES.sum(&method) # rubocop:disable CodeReuse/ActiveRecord
+ end
+ end
end
end
end
end
-
-class ::Redis::Client
- prepend ::Gitlab::Instrumentation::RedisInterceptor
-end
diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb
new file mode 100644
index 00000000000..012543e1645
--- /dev/null
+++ b/lib/gitlab/instrumentation/redis_base.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'redis'
+
+module Gitlab
+ module Instrumentation
+ class RedisBase
+ class << self
+ include ::Gitlab::Utils::StrongMemoize
+ include ::Gitlab::Instrumentation::RedisPayload
+
+ # TODO: To be used by https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/395
+ # as a 'label' alias.
+ def storage_key
+ self.name.demodulize.underscore
+ end
+
+ def add_duration(duration)
+ ::RequestStore[call_duration_key] ||= 0
+ ::RequestStore[call_duration_key] += duration
+ end
+
+ def add_call_details(duration, args)
+ return unless Gitlab::PerformanceBar.enabled_for_request?
+ # redis-rb passes an array (e.g. [[:get, key]])
+ return unless args.length == 1
+
+ # TODO: Add information about current Redis client
+ # being instrumented.
+ # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/316.
+ detail_store << {
+ cmd: args.first,
+ duration: duration,
+ backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller)
+ }
+ end
+
+ def increment_request_count
+ ::RequestStore[request_count_key] ||= 0
+ ::RequestStore[request_count_key] += 1
+ end
+
+ def increment_read_bytes(num_bytes)
+ ::RequestStore[read_bytes_key] ||= 0
+ ::RequestStore[read_bytes_key] += num_bytes
+ end
+
+ def increment_write_bytes(num_bytes)
+ ::RequestStore[write_bytes_key] ||= 0
+ ::RequestStore[write_bytes_key] += num_bytes
+ end
+
+ def get_request_count
+ ::RequestStore[request_count_key] || 0
+ end
+
+ def read_bytes
+ ::RequestStore[read_bytes_key] || 0
+ end
+
+ def write_bytes
+ ::RequestStore[write_bytes_key] || 0
+ end
+
+ def detail_store
+ ::RequestStore[call_details_key] ||= []
+ end
+
+ def query_time
+ query_time = ::RequestStore[call_duration_key] || 0
+ query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION)
+ end
+
+ private
+
+ def request_count_key
+ strong_memoize(:request_count_key) { build_key(:redis_request_count) }
+ end
+
+ def read_bytes_key
+ strong_memoize(:read_bytes_key) { build_key(:redis_read_bytes) }
+ end
+
+ def write_bytes_key
+ strong_memoize(:write_bytes_key) { build_key(:redis_write_bytes) }
+ end
+
+ def call_duration_key
+ strong_memoize(:call_duration_key) { build_key(:redis_call_duration) }
+ end
+
+ def call_details_key
+ strong_memoize(:call_details_key) { build_key(:redis_call_details) }
+ end
+
+ def build_key(namespace)
+ "#{storage_key}_#{namespace}"
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb
new file mode 100644
index 00000000000..a36aade59c3
--- /dev/null
+++ b/lib/gitlab/instrumentation/redis_interceptor.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'redis'
+
+module Gitlab
+ module Instrumentation
+ module RedisInterceptor
+ def call(*args, &block)
+ start = Time.now
+ super(*args, &block)
+ ensure
+ duration = (Time.now - start)
+
+ if ::RequestStore.active?
+ instrumentation_class.increment_request_count
+ instrumentation_class.add_duration(duration)
+ instrumentation_class.add_call_details(duration, args)
+ end
+ end
+
+ def write(command)
+ measure_write_size(command) if ::RequestStore.active?
+ super
+ end
+
+ def read
+ result = super
+ measure_read_size(result) if ::RequestStore.active?
+ result
+ end
+
+ private
+
+ def measure_write_size(command)
+ size = 0
+
+ # Mimic what happens in
+ # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/command_helper.rb#L8.
+ # This count is an approximation that omits the Redis protocol overhead
+ # of type prefixes, length prefixes and line endings.
+ command.each do |x|
+ size += begin
+ if x.is_a? Array
+ x.inject(0) { |sum, y| sum + y.to_s.bytesize }
+ else
+ x.to_s.bytesize
+ end
+ end
+ end
+
+ instrumentation_class.increment_write_bytes(size)
+ end
+
+ def measure_read_size(result)
+ # The Connection::Ruby#read class can return one of four types of results from read:
+ # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/ruby.rb#L406
+ #
+ # 1. Error (exception, will not reach this line)
+ # 2. Status (string)
+ # 3. Integer (will be converted to string by to_s.bytesize and thrown away)
+ # 4. "Binary" string (i.e. may contain zero byte)
+ # 5. Array of binary string
+
+ if result.is_a? Array
+ # Redis can return nested arrays, e.g. from XRANGE or GEOPOS, so we use recursion here.
+ result.each { |x| measure_read_size(x) }
+ else
+ # This count is an approximation that omits the Redis protocol overhead
+ # of type prefixes, length prefixes and line endings.
+ instrumentation_class.increment_read_bytes(result.to_s.bytesize)
+ end
+ end
+
+ # That's required so it knows which GitLab Redis instance
+ # it's interacting with in order to categorize accordingly.
+ #
+ def instrumentation_class
+ @options[:instrumentation_class] # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ end
+ end
+ end
+end
+
+class ::Redis::Client
+ prepend ::Gitlab::Instrumentation::RedisInterceptor
+end
diff --git a/lib/gitlab/instrumentation/redis_payload.rb b/lib/gitlab/instrumentation/redis_payload.rb
new file mode 100644
index 00000000000..69aafffd124
--- /dev/null
+++ b/lib/gitlab/instrumentation/redis_payload.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Instrumentation
+ module RedisPayload
+ include ::Gitlab::Utils::StrongMemoize
+
+ # Fetches payload keys from the lazy payload (this avoids
+ # unnecessary processing of the values).
+ def known_payload_keys
+ to_lazy_payload.keys
+ end
+
+ def payload
+ to_lazy_payload.transform_values do |value|
+ result = value.call
+ result if result > 0
+ end.compact
+ end
+
+ private
+
+ def to_lazy_payload
+ strong_memoize(:to_lazy_payload) do
+ key_prefix = storage_key ? "redis_#{storage_key}" : 'redis'
+
+ {
+ "#{key_prefix}_calls": -> { get_request_count },
+ "#{key_prefix}_duration_s": -> { query_time },
+ "#{key_prefix}_read_bytes": -> { read_bytes },
+ "#{key_prefix}_write_bytes": -> { write_bytes }
+ }.symbolize_keys
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index 7c5a601cd5b..3a29d2e7efa 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -4,30 +4,56 @@ module Gitlab
module InstrumentationHelper
extend self
- KEYS = %i(gitaly_calls gitaly_duration_s rugged_calls rugged_duration_s redis_calls redis_duration_s).freeze
DURATION_PRECISION = 6 # microseconds
+ def keys
+ @keys ||= [:gitaly_calls,
+ :gitaly_duration_s,
+ :rugged_calls,
+ :rugged_duration_s,
+ :elasticsearch_calls,
+ :elasticsearch_duration_s,
+ *::Gitlab::Instrumentation::Redis.known_payload_keys]
+ end
+
def add_instrumentation_data(payload)
+ instrument_gitaly(payload)
+ instrument_rugged(payload)
+ instrument_redis(payload)
+ instrument_elasticsearch(payload)
+ end
+
+ def instrument_gitaly(payload)
gitaly_calls = Gitlab::GitalyClient.get_request_count
- if gitaly_calls > 0
- payload[:gitaly_calls] = gitaly_calls
- payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time
- end
+ return if gitaly_calls == 0
+
+ payload[:gitaly_calls] = gitaly_calls
+ payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time
+ end
+ def instrument_rugged(payload)
rugged_calls = Gitlab::RuggedInstrumentation.query_count
- if rugged_calls > 0
- payload[:rugged_calls] = rugged_calls
- payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time
- end
+ return if rugged_calls == 0
+
+ payload[:rugged_calls] = rugged_calls
+ payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time
+ end
+
+ def instrument_redis(payload)
+ payload.merge! ::Gitlab::Instrumentation::Redis.payload
+ end
+
+ def instrument_elasticsearch(payload)
+ # Elasticsearch integration is only available in EE but instrumentation
+ # only depends on the Gem which is also available in FOSS.
+ elasticsearch_calls = Gitlab::Instrumentation::ElasticsearchTransport.get_request_count
- redis_calls = Gitlab::Instrumentation::Redis.get_request_count
+ return if elasticsearch_calls == 0
- if redis_calls > 0
- payload[:redis_calls] = redis_calls
- payload[:redis_duration_s] = Gitlab::Instrumentation::Redis.query_time
- end
+ payload[:elasticsearch_calls] = elasticsearch_calls
+ payload[:elasticsearch_duration_s] = Gitlab::Instrumentation::ElasticsearchTransport.query_time
end
# Returns the queuing duration for a Sidekiq job in seconds, as a float, if the
diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb
index 6f760751b0f..e946fc00c4d 100644
--- a/lib/gitlab/issuable_metadata.rb
+++ b/lib/gitlab/issuable_metadata.rb
@@ -1,8 +1,52 @@
# frozen_string_literal: true
module Gitlab
- module IssuableMetadata
- def issuable_meta_data(issuable_collection, collection_type, user = nil)
+ class IssuableMetadata
+ include Gitlab::Utils::StrongMemoize
+
+ # data structure to store issuable meta data like
+ # upvotes, downvotes, notes and closing merge requests counts for issues and merge requests
+ # this avoiding n+1 queries when loading issuable collections on frontend
+ IssuableMeta = Struct.new(:upvotes, :downvotes, :user_notes_count, :mrs_count) do
+ def merge_requests_count(user = nil)
+ mrs_count
+ end
+ end
+
+ attr_reader :current_user, :issuable_collection
+
+ def initialize(current_user, issuable_collection)
+ @current_user = current_user
+ @issuable_collection = issuable_collection
+
+ validate_collection!
+ end
+
+ def data
+ return {} if issuable_ids.empty?
+
+ issuable_ids.each_with_object({}) do |id, issuable_meta|
+ issuable_meta[id] = metadata_for_issuable(id)
+ end
+ end
+
+ private
+
+ def metadata_for_issuable(id)
+ downvotes = group_issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? }
+ upvotes = group_issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? }
+ notes = grouped_issuable_notes_count.find { |notes| notes.noteable_id == id }
+ merge_requests = grouped_issuable_merge_requests_count.find { |mr| mr.first == id }
+
+ IssuableMeta.new(
+ upvotes.try(:count).to_i,
+ downvotes.try(:count).to_i,
+ notes.try(:count).to_i,
+ merge_requests.try(:last).to_i
+ )
+ end
+
+ def validate_collection!
# ActiveRecord uses Object#extend for null relations.
if !(issuable_collection.singleton_class < ActiveRecord::NullRelation) &&
issuable_collection.respond_to?(:limit_value) &&
@@ -10,36 +54,43 @@ module Gitlab
raise 'Collection must have a limit applied for preloading meta-data'
end
+ end
- # map has to be used here since using pluck or select will
- # throw an error when ordering issuables by priority which inserts
- # a new order into the collection.
- # We cannot use reorder to not mess up the paginated collection.
- issuable_ids = issuable_collection.map(&:id)
+ def issuable_ids
+ strong_memoize(:issuable_ids) do
+ # map has to be used here since using pluck or select will
+ # throw an error when ordering issuables by priority which inserts
+ # a new order into the collection.
+ # We cannot use reorder to not mess up the paginated collection.
+ issuable_collection.map(&:id)
+ end
+ end
- return {} if issuable_ids.empty?
+ def collection_type
+ # Supports relations or paginated arrays
+ issuable_collection.try(:model)&.name ||
+ issuable_collection.first&.model_name.to_s
+ end
- issuable_notes_count = ::Note.count_for_collection(issuable_ids, collection_type)
- issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type)
- issuable_merge_requests_count =
+ def group_issuable_votes_count
+ strong_memoize(:group_issuable_votes_count) do
+ AwardEmoji.votes_for_collection(issuable_ids, collection_type)
+ end
+ end
+
+ def grouped_issuable_notes_count
+ strong_memoize(:grouped_issuable_notes_count) do
+ ::Note.count_for_collection(issuable_ids, collection_type)
+ end
+ end
+
+ def grouped_issuable_merge_requests_count
+ strong_memoize(:grouped_issuable_merge_requests_count) do
if collection_type == 'Issue'
- ::MergeRequestsClosingIssues.count_for_collection(issuable_ids, user)
+ ::MergeRequestsClosingIssues.count_for_collection(issuable_ids, current_user)
else
[]
end
-
- issuable_ids.each_with_object({}) do |id, issuable_meta|
- downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? }
- upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? }
- notes = issuable_notes_count.find { |notes| notes.noteable_id == id }
- merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id }
-
- issuable_meta[id] = ::Issuable::IssuableMeta.new(
- upvotes.try(:count).to_i,
- downvotes.try(:count).to_i,
- notes.try(:count).to_i,
- merge_requests.try(:last).to_i
- )
end
end
end
diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb
index b973244a531..c09d8170d17 100644
--- a/lib/gitlab/jira/http_client.rb
+++ b/lib/gitlab/jira/http_client.rb
@@ -12,12 +12,7 @@ module Gitlab
def request(*args)
result = make_request(*args)
- unless result.response.is_a?(Net::HTTPSuccess)
- Gitlab::ErrorTracking.track_and_raise_exception(
- JIRA::HTTPError.new(result.response),
- response: result.body
- )
- end
+ raise JIRA::HTTPError.new(result.response) unless result.response.is_a?(Net::HTTPSuccess)
result
end
diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb
index 3f56094956a..75d6fdc07b6 100644
--- a/lib/gitlab/jira_import.rb
+++ b/lib/gitlab/jira_import.rb
@@ -7,11 +7,30 @@ module Gitlab
FAILED_ISSUES_COUNTER_KEY = 'jira-import/failed/%{project_id}/%{collection_type}'
NEXT_ITEMS_START_AT_KEY = 'jira-import/paginator/%{project_id}/%{collection_type}'
JIRA_IMPORT_LABEL = 'jira-import/import-label/%{project_id}'
- ITEMS_MAPPER_CACHE_KEY = 'jira-import/items-mapper/%{project_id}/%{collection_type}/%{jira_isssue_id}'
+ ITEMS_MAPPER_CACHE_KEY = 'jira-import/items-mapper/%{project_id}/%{collection_type}/%{jira_item_id}'
+ USERS_MAPPER_KEY_PREFIX = 'jira-import/items-mapper/%{project_id}/users/'
ALREADY_IMPORTED_ITEMS_CACHE_KEY = 'jira-importer/already-imported/%{project}/%{collection_type}'
- def self.jira_issue_cache_key(project_id, jira_issue_id)
- ITEMS_MAPPER_CACHE_KEY % { project_id: project_id, collection_type: :issues, jira_isssue_id: jira_issue_id }
+ def self.validate_project_settings!(project, user: nil, configuration_check: true)
+ if user
+ raise Projects::ImportService::Error, _('Cannot import because issues are not available in this project.') unless project.feature_available?(:issues, user)
+ raise Projects::ImportService::Error, _('You do not have permissions to run the import.') unless user.can?(:admin_project, project)
+ end
+
+ return unless configuration_check
+
+ jira_service = project.jira_service
+
+ raise Projects::ImportService::Error, _('Jira integration not configured.') unless jira_service&.active?
+ raise Projects::ImportService::Error, _('Unable to connect to the Jira instance. Please check your Jira integration configuration.') unless jira_service&.valid_connection?
+ end
+
+ def self.jira_item_cache_key(project_id, jira_item_id, collection_type)
+ ITEMS_MAPPER_CACHE_KEY % { project_id: project_id, collection_type: collection_type, jira_item_id: jira_item_id }
+ end
+
+ def self.jira_user_key_prefix(project_id)
+ USERS_MAPPER_KEY_PREFIX % { project_id: project_id }
end
def self.already_imported_cache_key(collection_type, project_id)
@@ -48,7 +67,7 @@ module Gitlab
end
def self.cache_issue_mapping(issue_id, jira_issue_id, project_id)
- cache_key = JiraImport.jira_issue_cache_key(project_id, jira_issue_id)
+ cache_key = JiraImport.jira_item_cache_key(project_id, jira_issue_id, :issues)
cache_class.write(cache_key, issue_id)
end
@@ -67,6 +86,19 @@ module Gitlab
cache_class.expire(self.already_imported_cache_key(:issues, project_id), JIRA_IMPORT_CACHE_TIMEOUT)
end
+ # Caches the mapping of jira_account_id -> gitlab user id
+ # project_id - id of a project
+ # mapping - hash in format of jira_account_id -> gitlab user id
+ def self.cache_users_mapping(project_id, mapping)
+ cache_class.write_multiple(mapping, key_prefix: jira_user_key_prefix(project_id))
+ end
+
+ def self.get_user_mapping(project_id, jira_account_id)
+ cache_key = JiraImport.jira_item_cache_key(project_id, jira_account_id, :users)
+
+ cache_class.read(cache_key)&.to_i
+ end
+
def self.cache_class
Gitlab::Cache::Import::Caching
end
diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb
index 306736df30f..688254bf91f 100644
--- a/lib/gitlab/jira_import/base_importer.rb
+++ b/lib/gitlab/jira_import/base_importer.rb
@@ -6,7 +6,7 @@ module Gitlab
attr_reader :project, :client, :formatter, :jira_project_key, :running_import
def initialize(project)
- project.validate_jira_import_settings!
+ Gitlab::JiraImport.validate_project_settings!(project)
@running_import = project.latest_jira_import
@jira_project_key = running_import&.jira_project_key
diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb
index 8c18e58d9df..26fa01755d1 100644
--- a/lib/gitlab/jira_import/issues_importer.rb
+++ b/lib/gitlab/jira_import/issues_importer.rb
@@ -57,17 +57,27 @@ module Gitlab
# For such cases we exit early if issue was already imported.
next if already_imported?(jira_issue.id)
- issue_attrs = IssueSerializer.new(project, jira_issue, running_import.user_id, { iid: next_iid }).execute
- Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key)
-
- job_waiter.jobs_remaining += 1
- next_iid += 1
-
- # Mark the issue as imported immediately so we don't end up
- # importing it multiple times within same import.
- # These ids are cleaned-up when import finishes.
- # see Gitlab::JiraImport::Stage::FinishImportWorker
- mark_as_imported(jira_issue.id)
+ begin
+ issue_attrs = IssueSerializer.new(project, jira_issue, running_import.user_id, { iid: next_iid }).execute
+
+ Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key)
+
+ job_waiter.jobs_remaining += 1
+ next_iid += 1
+
+ # Mark the issue as imported immediately so we don't end up
+ # importing it multiple times within same import.
+ # These ids are cleaned-up when import finishes.
+ # see Gitlab::JiraImport::Stage::FinishImportWorker
+ mark_as_imported(jira_issue.id)
+ rescue => ex
+ # handle exceptionn here and skip the failed to import issue, instead of
+ # failing to import the entire batch of issues
+
+ # track the failed to import issue.
+ Gitlab::ErrorTracking.track_exception(ex, project_id: project.id)
+ JiraImport.increment_issue_failures(project.id)
+ end
end
job_waiter
diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb
index 00ab7109267..9507f7bc117 100644
--- a/lib/gitlab/kubernetes/helm.rb
+++ b/lib/gitlab/kubernetes/helm.rb
@@ -10,12 +10,6 @@ module Gitlab
SERVICE_ACCOUNT = 'tiller'
CLUSTER_ROLE_BINDING = 'tiller-admin'
CLUSTER_ROLE = 'cluster-admin'
-
- MANAGED_APPS_LOCAL_TILLER_FEATURE_FLAG = :managed_apps_local_tiller
-
- def self.local_tiller_enabled?
- Feature.enabled?(MANAGED_APPS_LOCAL_TILLER_FEATURE_FLAG)
- end
end
end
end
diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb
index 31cd21f17e0..f27ad05599e 100644
--- a/lib/gitlab/kubernetes/helm/base_command.rb
+++ b/lib/gitlab/kubernetes/helm/base_command.rb
@@ -3,7 +3,24 @@
module Gitlab
module Kubernetes
module Helm
- module BaseCommand
+ class BaseCommand
+ attr_reader :name, :files
+
+ def initialize(rbac:, name:, files:, local_tiller_enabled:)
+ @rbac = rbac
+ @name = name
+ @files = files
+ @local_tiller_enabled = local_tiller_enabled
+ end
+
+ def rbac?
+ @rbac
+ end
+
+ def local_tiller_enabled?
+ @local_tiller_enabled
+ end
+
def pod_resource
pod_service_account_name = rbac? ? service_account_name : nil
@@ -46,18 +63,6 @@ module Gitlab
files.keys
end
- def name
- raise "Not implemented"
- end
-
- def rbac?
- raise "Not implemented"
- end
-
- def files
- raise "Not implemented"
- end
-
private
def files_dir
diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb
index e7ade7e4d39..24458e1b4b3 100644
--- a/lib/gitlab/kubernetes/helm/client_command.rb
+++ b/lib/gitlab/kubernetes/helm/client_command.rb
@@ -57,10 +57,6 @@ module Gitlab
'--tls-key', "#{files_dir}/key.pem"
]
end
-
- def local_tiller_enabled?
- ::Gitlab::Kubernetes::Helm.local_tiller_enabled?
- end
end
end
end
diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb
index 771444ee9ee..3bb41d09994 100644
--- a/lib/gitlab/kubernetes/helm/delete_command.rb
+++ b/lib/gitlab/kubernetes/helm/delete_command.rb
@@ -3,17 +3,13 @@
module Gitlab
module Kubernetes
module Helm
- class DeleteCommand
- include BaseCommand
+ class DeleteCommand < BaseCommand
include ClientCommand
attr_reader :predelete, :postdelete
- attr_accessor :name, :files
- def initialize(name:, rbac:, files:, predelete: nil, postdelete: nil)
- @name = name
- @files = files
- @rbac = rbac
+ def initialize(predelete: nil, postdelete: nil, **args)
+ super(**args)
@predelete = predelete
@postdelete = postdelete
end
@@ -32,10 +28,6 @@ module Gitlab
"uninstall-#{name}"
end
- def rbac?
- @rbac
- end
-
def delete_command
command = ['helm', 'delete', '--purge', name] + tls_flags_if_remote_tiller
diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb
index 058f38f2c9c..e4844e255c5 100644
--- a/lib/gitlab/kubernetes/helm/init_command.rb
+++ b/lib/gitlab/kubernetes/helm/init_command.rb
@@ -3,27 +3,13 @@
module Gitlab
module Kubernetes
module Helm
- class InitCommand
- include BaseCommand
-
- attr_reader :name, :files
-
- def initialize(name:, files:, rbac:)
- @name = name
- @files = files
- @rbac = rbac
- end
-
+ class InitCommand < BaseCommand
def generate_script
super + [
init_helm_command
].join("\n")
end
- def rbac?
- @rbac
- end
-
private
def init_helm_command
diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb
index 3784aecccb5..cf6d993cad4 100644
--- a/lib/gitlab/kubernetes/helm/install_command.rb
+++ b/lib/gitlab/kubernetes/helm/install_command.rb
@@ -3,19 +3,16 @@
module Gitlab
module Kubernetes
module Helm
- class InstallCommand
- include BaseCommand
+ class InstallCommand < BaseCommand
include ClientCommand
- attr_reader :name, :files, :chart, :repository, :preinstall, :postinstall
+ attr_reader :chart, :repository, :preinstall, :postinstall
attr_accessor :version
- def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil, preinstall: nil, postinstall: nil)
- @name = name
+ def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args)
+ super(**args)
@chart = chart
@version = version
- @rbac = rbac
- @files = files
@repository = repository
@preinstall = preinstall
@postinstall = postinstall
@@ -33,10 +30,6 @@ module Gitlab
].compact.join("\n")
end
- def rbac?
- @rbac
- end
-
private
# Uses `helm upgrade --install` which means we can use this for both
diff --git a/lib/gitlab/kubernetes/helm/patch_command.rb b/lib/gitlab/kubernetes/helm/patch_command.rb
index ed7a5c2b2d6..1a5fab116bd 100644
--- a/lib/gitlab/kubernetes/helm/patch_command.rb
+++ b/lib/gitlab/kubernetes/helm/patch_command.rb
@@ -5,23 +5,21 @@
module Gitlab
module Kubernetes
module Helm
- class PatchCommand
- include BaseCommand
+ class PatchCommand < BaseCommand
include ClientCommand
- attr_reader :name, :files, :chart, :repository
+ attr_reader :chart, :repository
attr_accessor :version
- def initialize(name:, chart:, files:, rbac:, version:, repository: nil)
+ def initialize(chart:, version:, repository: nil, **args)
+ super(**args)
+
# version is mandatory to prevent chart mismatches
# we do not want our values interpreted in the context of the wrong version
raise ArgumentError, 'version is required' if version.blank?
- @name = name
@chart = chart
@version = version
- @rbac = rbac
- @files = files
@repository = repository
end
@@ -35,10 +33,6 @@ module Gitlab
].compact.join("\n")
end
- def rbac?
- @rbac
- end
-
private
def upgrade_command
diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb
index 13176360227..f1f7938039c 100644
--- a/lib/gitlab/kubernetes/helm/reset_command.rb
+++ b/lib/gitlab/kubernetes/helm/reset_command.rb
@@ -3,18 +3,9 @@
module Gitlab
module Kubernetes
module Helm
- class ResetCommand
- include BaseCommand
+ class ResetCommand < BaseCommand
include ClientCommand
- attr_reader :name, :files
-
- def initialize(name:, rbac:, files:)
- @name = name
- @files = files
- @rbac = rbac
- end
-
def generate_script
super + [
reset_helm_command,
@@ -23,10 +14,6 @@ module Gitlab
].join("\n")
end
- def rbac?
- @rbac
- end
-
def pod_name
"uninstall-#{name}"
end
diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb
index ea25d81cbd2..dc13a614551 100644
--- a/lib/gitlab/kubernetes/network_policy.rb
+++ b/lib/gitlab/kubernetes/network_policy.rb
@@ -3,9 +3,12 @@
module Gitlab
module Kubernetes
class NetworkPolicy
- def initialize(name:, namespace:, pod_selector:, ingress:, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil)
+ DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by'
+
+ def initialize(name:, namespace:, pod_selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil)
@name = name
@namespace = namespace
+ @labels = labels
@creation_timestamp = creation_timestamp
@pod_selector = pod_selector
@policy_types = policy_types
@@ -24,6 +27,7 @@ module Gitlab
self.new(
name: metadata[:name],
namespace: metadata[:namespace],
+ labels: metadata[:labels],
pod_selector: spec[:podSelector],
policy_types: spec[:policyTypes],
ingress: spec[:ingress],
@@ -42,6 +46,7 @@ module Gitlab
self.new(
name: metadata[:name],
namespace: metadata[:namespace],
+ labels: metadata[:labels]&.to_h,
creation_timestamp: metadata[:creationTimestamp],
pod_selector: spec[:podSelector],
policy_types: spec[:policyTypes],
@@ -62,16 +67,48 @@ module Gitlab
name: name,
namespace: namespace,
creation_timestamp: creation_timestamp,
- manifest: manifest
+ manifest: manifest,
+ is_autodevops: autodevops?,
+ is_enabled: enabled?
}
end
+ def autodevops?
+ return false unless labels
+
+ !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-')
+ end
+
+ # podSelector selects pods that should be targeted by this
+ # policy. We can narrow selection by requiring this policy to
+ # match our custom labels. Since DISABLED_BY label will not be
+ # on any pod a policy will be effectively disabled.
+ def enabled?
+ return true unless pod_selector&.key?(:matchLabels)
+
+ !pod_selector[:matchLabels]&.key?(DISABLED_BY_LABEL)
+ end
+
+ def enable
+ return if enabled?
+
+ pod_selector[:matchLabels].delete(DISABLED_BY_LABEL)
+ end
+
+ def disable
+ @pod_selector ||= {}
+ pod_selector[:matchLabels] ||= {}
+ pod_selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab')
+ end
+
private
- attr_reader :name, :namespace, :creation_timestamp, :pod_selector, :policy_types, :ingress, :egress
+ attr_reader :name, :namespace, :labels, :creation_timestamp, :pod_selector, :policy_types, :ingress, :egress
def metadata
- { name: name, namespace: namespace }
+ meta = { name: name, namespace: namespace }
+ meta[:labels] = labels if labels
+ meta
end
def spec
diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb
index e90f3f05a33..c7f2adb27d1 100644
--- a/lib/gitlab/lfs_token.rb
+++ b/lib/gitlab/lfs_token.rb
@@ -14,7 +14,7 @@ module Gitlab
include LfsTokenHelper
- DEFAULT_EXPIRE_TIME = 1800
+ DEFAULT_EXPIRE_TIME = 7200 # Default value 2 hours
attr_accessor :actor
diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb
index 55c46c365f6..17a36c292c0 100644
--- a/lib/gitlab/lograge/custom_options.rb
+++ b/lib/gitlab/lograge/custom_options.rb
@@ -12,24 +12,24 @@ module Gitlab
params = event
.payload[:params]
.each_with_object([]) { |(k, v), array| array << { key: k, value: v } unless IGNORE_PARAMS.include?(k) }
-
payload = {
time: Time.now.utc.iso8601(3),
params: Gitlab::Utils::LogLimitedArray.log_limited_array(params, sentinel: LIMITED_ARRAY_SENTINEL),
remote_ip: event.payload[:remote_ip],
user_id: event.payload[:user_id],
username: event.payload[:username],
- ua: event.payload[:ua],
- queue_duration_s: event.payload[:queue_duration_s]
+ ua: event.payload[:ua]
}
+ add_db_counters!(payload)
payload.merge!(event.payload[:metadata]) if event.payload[:metadata]
::Gitlab::InstrumentationHelper.add_instrumentation_data(payload)
+ payload[:queue_duration_s] = event.payload[:queue_duration_s] if event.payload[:queue_duration_s]
payload[:response] = event.payload[:response] if event.payload[:response]
payload[:etag_route] = event.payload[:etag_route] if event.payload[:etag_route]
- payload[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id
+ payload[Labkit::Correlation::CorrelationId::LOG_KEY] = event.payload[Labkit::Correlation::CorrelationId::LOG_KEY] || Labkit::Correlation::CorrelationId.current_id
if cpu_s = Gitlab::Metrics::System.thread_cpu_duration(::Gitlab::RequestContext.instance.start_thread_cpu_time)
payload[:cpu_s] = cpu_s.round(2)
@@ -46,6 +46,16 @@ module Gitlab
payload
end
+
+ def self.add_db_counters!(payload)
+ current_transaction = Gitlab::Metrics::Transaction.current
+ if current_transaction
+ payload[:db_count] = current_transaction.get(:db_count, :counter).to_i
+ payload[:db_write_count] = current_transaction.get(:db_write_count, :counter).to_i
+ payload[:db_cached_count] = current_transaction.get(:db_cached_count, :counter).to_i
+ end
+ end
+ private_class_method :add_db_counters!
end
end
end
diff --git a/lib/gitlab/looping_batcher.rb b/lib/gitlab/looping_batcher.rb
deleted file mode 100644
index adf0aeda506..00000000000
--- a/lib/gitlab/looping_batcher.rb
+++ /dev/null
@@ -1,99 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- # Returns an ID range within a table so it can be iterated over. Repeats from
- # the beginning after it reaches the end.
- #
- # Used by Geo in particular to iterate over a replicable and its registry
- # table.
- #
- # Tracks a cursor for each table, by "key". If the table is smaller than
- # batch_size, then a range for the whole table is returned on every call.
- class LoopingBatcher
- # @param [Class] model_class the class of the table to iterate on
- # @param [String] key to identify the cursor. Note, cursor is already unique
- # per table.
- # @param [Integer] batch_size to limit the number of records in a batch
- def initialize(model_class, key:, batch_size: 1000)
- @model_class = model_class
- @key = key
- @batch_size = batch_size
- end
-
- # @return [Range] a range of IDs. `nil` if 0 records at or after the cursor.
- def next_range!
- return unless @model_class.any?
-
- batch_first_id = cursor_id
-
- batch_last_id = get_batch_last_id(batch_first_id)
- return unless batch_last_id
-
- batch_first_id..batch_last_id
- end
-
- private
-
- # @private
- #
- # Get the last ID of the batch. Increment the cursor or reset it if at end.
- #
- # @param [Integer] batch_first_id the first ID of the batch
- # @return [Integer] batch_last_id the last ID of the batch (not the table)
- def get_batch_last_id(batch_first_id)
- batch_last_id, more_rows = run_query(@model_class.table_name, @model_class.primary_key, batch_first_id, @batch_size)
-
- if more_rows
- increment_batch(batch_last_id)
- else
- reset if batch_first_id > 1
- end
-
- batch_last_id
- end
-
- def run_query(table, primary_key, batch_first_id, batch_size)
- sql = <<~SQL
- SELECT MAX(batch.id) AS batch_last_id,
- EXISTS (
- SELECT #{primary_key}
- FROM #{table}
- WHERE #{primary_key} > MAX(batch.id)
- ) AS more_rows
- FROM (
- SELECT #{primary_key}
- FROM #{table}
- WHERE #{primary_key} >= #{batch_first_id}
- ORDER BY #{primary_key}
- LIMIT #{batch_size}) AS batch;
- SQL
-
- result = ActiveRecord::Base.connection.exec_query(sql).first
-
- [result["batch_last_id"], result["more_rows"]]
- end
-
- def reset
- set_cursor_id(1)
- end
-
- def increment_batch(batch_last_id)
- set_cursor_id(batch_last_id + 1)
- end
-
- # @private
- #
- # @return [Integer] the cursor ID, or 1 if it is not set
- def cursor_id
- Rails.cache.fetch("#{cache_key}:cursor_id") || 1
- end
-
- def set_cursor_id(id)
- Rails.cache.write("#{cache_key}:cursor_id", id)
- end
-
- def cache_key
- @cache_key ||= "#{self.class.name.parameterize}:#{@model_class.name.parameterize}:#{@key}:cursor_id"
- end
- end
-end
diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
new file mode 100644
index 00000000000..ff36f7b605e
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ module Dashboard
+ module Stages
+ class UrlValidator < BaseStage
+ def transform!
+ dashboard[:links]&.each do |link|
+ Gitlab::UrlBlocker.validate!(link[:url])
+ rescue Gitlab::UrlBlocker::BlockedUrlError
+ link[:url] = ''
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/elasticsearch_rack_middleware.rb b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb
new file mode 100644
index 00000000000..6830eed68d5
--- /dev/null
+++ b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ # Rack middleware for tracking Elasticsearch metrics from Grape and Web requests.
+ class ElasticsearchRackMiddleware
+ HISTOGRAM_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60].freeze
+
+ def initialize(app)
+ @app = app
+
+ @requests_total_counter = Gitlab::Metrics.counter(:http_elasticsearch_requests_total,
+ 'Amount of calls to Elasticsearch servers during web requests',
+ Gitlab::Metrics::Transaction::BASE_LABELS)
+ @requests_duration_histogram = Gitlab::Metrics.histogram(:http_elasticsearch_requests_duration_seconds,
+ 'Query time for Elasticsearch servers during web requests',
+ Gitlab::Metrics::Transaction::BASE_LABELS,
+ HISTOGRAM_BUCKETS)
+ end
+
+ def call(env)
+ transaction = Gitlab::Metrics.current_transaction
+
+ @app.call(env)
+ ensure
+ record_metrics(transaction)
+ end
+
+ private
+
+ def record_metrics(transaction)
+ labels = transaction.labels
+ query_time = ::Gitlab::Instrumentation::ElasticsearchTransport.query_time
+ request_count = ::Gitlab::Instrumentation::ElasticsearchTransport.get_request_count
+
+ @requests_total_counter.increment(labels, request_count)
+ @requests_duration_histogram.observe(labels, query_time)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb
index cee601ff14c..5955987541c 100644
--- a/lib/gitlab/metrics/methods.rb
+++ b/lib/gitlab/metrics/methods.rb
@@ -52,7 +52,7 @@ module Gitlab
end
def disabled_by_feature(options)
- options.with_feature && !::Feature.get(options.with_feature).enabled?
+ options.with_feature && !::Feature.enabled?(options.with_feature)
end
def build_metric!(type, name, options)
diff --git a/lib/gitlab/metrics/redis_rack_middleware.rb b/lib/gitlab/metrics/redis_rack_middleware.rb
new file mode 100644
index 00000000000..f0f99c5f45d
--- /dev/null
+++ b/lib/gitlab/metrics/redis_rack_middleware.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Metrics
+ # Rack middleware for tracking Redis metrics from Grape and Web requests.
+ class RedisRackMiddleware
+ def initialize(app)
+ @app = app
+
+ @requests_total_counter = Gitlab::Metrics.counter(:http_redis_requests_total,
+ 'Amount of calls to Redis servers during web requests',
+ Gitlab::Metrics::Transaction::BASE_LABELS)
+ @requests_duration_histogram = Gitlab::Metrics.histogram(:http_redis_requests_duration_seconds,
+ 'Query time for Redis servers during web requests',
+ Gitlab::Metrics::Transaction::BASE_LABELS,
+ Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS)
+ end
+
+ def call(env)
+ transaction = Gitlab::Metrics.current_transaction
+
+ @app.call(env)
+ ensure
+ record_metrics(transaction)
+ end
+
+ private
+
+ def record_metrics(transaction)
+ labels = transaction.labels
+ query_time = Gitlab::Instrumentation::Redis.query_time
+ request_count = Gitlab::Instrumentation::Redis.get_request_count
+
+ @requests_total_counter.increment(labels, request_count)
+ @requests_duration_histogram.observe(labels, query_time)
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb
index 90051f85f31..ff3e7be567f 100644
--- a/lib/gitlab/metrics/samplers/base_sampler.rb
+++ b/lib/gitlab/metrics/samplers/base_sampler.rb
@@ -6,8 +6,10 @@ module Gitlab
module Metrics
module Samplers
class BaseSampler < Daemon
+ attr_reader :interval
+
# interval - The sampling interval in seconds.
- def initialize(interval)
+ def initialize(interval = self.class::SAMPLING_INTERVAL_SECONDS)
interval_half = interval.to_f / 2
@interval = interval
diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb
index 98dd517ee3b..b5343d5e66a 100644
--- a/lib/gitlab/metrics/samplers/puma_sampler.rb
+++ b/lib/gitlab/metrics/samplers/puma_sampler.rb
@@ -4,6 +4,8 @@ module Gitlab
module Metrics
module Samplers
class PumaSampler < BaseSampler
+ SAMPLING_INTERVAL_SECONDS = 5
+
def metrics
@metrics ||= init_metrics
end
diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb
index df59c06911b..dac9fbd1247 100644
--- a/lib/gitlab/metrics/samplers/ruby_sampler.rb
+++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb
@@ -6,9 +6,10 @@ module Gitlab
module Metrics
module Samplers
class RubySampler < BaseSampler
+ SAMPLING_INTERVAL_SECONDS = 60
GC_REPORT_BUCKETS = [0.005, 0.01, 0.02, 0.04, 0.07, 0.1, 0.5].freeze
- def initialize(interval)
+ def initialize(*)
GC::Profiler.clear
metrics[:process_start_time_seconds].set(labels, Time.now.to_i)
diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb
index 8dfb61046c4..de8e1ca3256 100644
--- a/lib/gitlab/metrics/sidekiq_middleware.rb
+++ b/lib/gitlab/metrics/sidekiq_middleware.rb
@@ -6,19 +6,30 @@ module Gitlab
#
# This middleware is intended to be used as a server-side middleware.
class SidekiqMiddleware
- def call(worker, message, queue)
+ def call(worker, payload, queue)
trans = BackgroundTransaction.new(worker.class)
begin
# Old gitlad-shell messages don't provide enqueued_at/created_at attributes
- trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0))
+ enqueued_at = payload['enqueued_at'] || payload['created_at'] || 0
+ trans.set(:sidekiq_queue_duration, Time.current.to_f - enqueued_at)
trans.run { yield }
rescue Exception => error # rubocop: disable Lint/RescueException
trans.add_event(:sidekiq_exception)
raise error
+ ensure
+ add_info_to_payload(payload, trans)
end
end
+
+ private
+
+ def add_info_to_payload(payload, trans)
+ payload[:db_count] = trans.get(:db_count, :counter).to_i
+ payload[:db_write_count] = trans.get(:db_write_count, :counter).to_i
+ payload[:db_cached_count] = trans.get(:db_cached_count, :counter).to_i
+ end
end
end
end
diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb
index a02dd850582..1628eeb5a95 100644
--- a/lib/gitlab/metrics/subscribers/active_record.rb
+++ b/lib/gitlab/metrics/subscribers/active_record.rb
@@ -9,6 +9,7 @@ module Gitlab
attach_to :active_record
IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze
+ DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze
def sql(event)
return unless current_transaction
@@ -19,8 +20,7 @@ module Gitlab
self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0)
- current_transaction.increment(:sql_duration, event.duration, false)
- current_transaction.increment(:sql_count, 1, false)
+ increment_db_counters(payload)
end
private
@@ -31,6 +31,20 @@ module Gitlab
buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
end
+ def select_sql_command?(payload)
+ payload[:sql].match(/\A((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i)
+ end
+
+ def increment_db_counters(payload)
+ current_transaction.increment(:db_count, 1)
+
+ if payload.fetch(:cached, payload[:name] == 'CACHE')
+ current_transaction.increment(:db_cached_count, 1)
+ end
+
+ current_transaction.increment(:db_write_count, 1) unless select_sql_command?(payload)
+ end
+
def current_transaction
Transaction.current
end
diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb
index b126efd2dd5..822f5243e9d 100644
--- a/lib/gitlab/metrics/transaction.rb
+++ b/lib/gitlab/metrics/transaction.rb
@@ -16,7 +16,7 @@ module Gitlab
# The series to store events (e.g. Git pushes) in.
EVENT_SERIES = 'events'
- attr_reader :tags, :method
+ attr_reader :method
def self.current
Thread.current[THREAD_KEY]
@@ -28,8 +28,6 @@ module Gitlab
@started_at = nil
@finished_at = nil
- @tags = {}
-
@memory_before = 0
@memory_after = 0
end
@@ -94,6 +92,12 @@ module Gitlab
self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus
end
+ def get(name, type, tags = {})
+ metric = self.class.transaction_metric(name, type)
+
+ metric.get(filter_tags(tags).merge(labels))
+ end
+
def labels
BASE_LABELS
end
diff --git a/lib/gitlab/middleware/handle_ip_spoof_attack_error.rb b/lib/gitlab/middleware/handle_ip_spoof_attack_error.rb
new file mode 100644
index 00000000000..2fc08db9b4d
--- /dev/null
+++ b/lib/gitlab/middleware/handle_ip_spoof_attack_error.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Middleware
+ # ActionDispatch::RemoteIp tries to set the `request.ip` for controllers by
+ # looking at the request IP and headers. It needs to see through any reverse
+ # proxies to get the right answer, but there are some security issues with
+ # that.
+ #
+ # Proxies can specify `Client-Ip` or `X-Forwarded-For`, and the security of
+ # that is determined at the edge. If both headers are present, it's likely
+ # that the edge is securing one, but ignoring the other. Rails blocks this,
+ # which is correct, because we don't know which header is the safe one - but
+ # we want the block to be a 400, rather than 500, error.
+ #
+ # This middleware needs to go before ActionDispatch::RemoteIp in the chain.
+ class HandleIpSpoofAttackError
+ attr_reader :app
+
+ def initialize(app)
+ @app = app
+ end
+
+ def call(env)
+ app.call(env)
+ rescue ActionDispatch::RemoteIp::IpSpoofAttackError => err
+ Gitlab::ErrorTracking.track_exception(err)
+
+ [400, { 'Content-Type' => 'text/plain' }, ['Bad Request']]
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/monitor/demo_projects.rb b/lib/gitlab/monitor/demo_projects.rb
new file mode 100644
index 00000000000..c617f895e4c
--- /dev/null
+++ b/lib/gitlab/monitor/demo_projects.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Monitor
+ # See Demo Project documentation
+ # https://about.gitlab.com/handbook/engineering/development/ops/monitor/#demo-environments
+ module DemoProjects
+ # [https://gitlab.com/gitlab-org/monitor/tanuki-inc, https://gitlab.com/gitlab-org/monitor/monitor-sandbox]
+ DOT_COM_IDS = [14986497, 12507547].freeze
+ # [https://staging.gitlab.com/gitlab-org/monitor/monitor-sandbox]
+ STAGING_IDS = [4422333].freeze
+
+ def self.primary_keys
+ # .com? returns true for staging
+ if ::Gitlab.com? && !::Gitlab.staging?
+ DOT_COM_IDS
+ elsif ::Gitlab.staging?
+ STAGING_IDS
+ elsif ::Gitlab.dev_or_test_env?
+ Project.limit(100).pluck(:id) # rubocop: disable CodeReuse/ActiveRecord
+ else
+ []
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/pagination/keyset/request_context.rb b/lib/gitlab/pagination/keyset/request_context.rb
index 8c8138b3076..070fa844347 100644
--- a/lib/gitlab/pagination/keyset/request_context.rb
+++ b/lib/gitlab/pagination/keyset/request_context.rb
@@ -24,7 +24,9 @@ module Gitlab
end
def apply_headers(next_page)
- request.header('Links', pagination_links(next_page))
+ link = pagination_links(next_page)
+ request.header('Links', link)
+ request.header('Link', link)
end
private
diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb
index 11a5ef4e518..8796dd4d7ec 100644
--- a/lib/gitlab/pagination/offset_pagination.rb
+++ b/lib/gitlab/pagination/offset_pagination.rb
@@ -19,7 +19,13 @@ module Gitlab
private
def paginate_with_limit_optimization(relation)
- pagination_data = relation.page(params[:page]).per(params[:per_page])
+ # do not paginate relation if it is already paginated
+ pagination_data = if relation.respond_to?(:current_page) && relation.current_page == params[:page] && relation.limit_value == params[:per_page]
+ relation
+ else
+ relation.page(params[:page]).per(params[:per_page])
+ end
+
return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation)
return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit)
diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb
index 6a2841b6a8e..7aba3cf26fd 100644
--- a/lib/gitlab/phabricator_import/cache/map.rb
+++ b/lib/gitlab/phabricator_import/cache/map.rb
@@ -63,7 +63,7 @@ module Gitlab
def timeout
# Setting the timeout to the same one as we do for clearing stuck jobs
# this makes sure all cache is available while the import is running.
- StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION
+ Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION
end
end
end
diff --git a/lib/gitlab/phabricator_import/worker_state.rb b/lib/gitlab/phabricator_import/worker_state.rb
index 38829e34509..ffa2d3d7a43 100644
--- a/lib/gitlab/phabricator_import/worker_state.rb
+++ b/lib/gitlab/phabricator_import/worker_state.rb
@@ -40,7 +40,7 @@ module Gitlab
def timeout
# Make sure we get rid of all the information after a job is marked
# as failed/succeeded
- StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION
+ Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION
end
end
end
diff --git a/lib/gitlab/process_memory_cache/helper.rb b/lib/gitlab/process_memory_cache/helper.rb
new file mode 100644
index 00000000000..ee4b81a9a19
--- /dev/null
+++ b/lib/gitlab/process_memory_cache/helper.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Gitlab
+ class ProcessMemoryCache
+ module Helper
+ def fetch_memory_cache(key, &payload)
+ cache = cache_backend.read(key)
+
+ if cache && !stale_cache?(key, cache)
+ cache[:data]
+ else
+ store_cache(key, &payload)
+ end
+ end
+
+ def invalidate_memory_cache(key)
+ touch_cache_timestamp(key)
+ end
+
+ private
+
+ def touch_cache_timestamp(key, time = Time.current.to_f)
+ shared_backend.write(key, time)
+ end
+
+ def stale_cache?(key, cache_info)
+ shared_timestamp = shared_backend.read(key)
+ return true unless shared_timestamp
+
+ shared_timestamp.to_f > cache_info[:cached_at].to_f
+ end
+
+ def store_cache(key)
+ data = yield
+ time = Time.current.to_f
+
+ cache_backend.write(key, data: data, cached_at: time)
+ touch_cache_timestamp(key, time)
+ data
+ end
+
+ def shared_backend
+ Rails.cache
+ end
+
+ def cache_backend
+ ::Gitlab::ProcessMemoryCache.cache_backend
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index fbdfe166645..e6b25e71eb3 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -11,7 +11,7 @@ module Gitlab
@query = query
end
- def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE)
+ def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil)
case scope
when 'notes'
notes.page(page).per(per_page)
@@ -20,7 +20,7 @@ module Gitlab
when 'wiki_blobs'
paginated_wiki_blobs(wiki_blobs(limit: limit_up_to_page(page, per_page)), page, per_page)
when 'commits'
- Kaminari.paginate_array(commits).page(page).per(per_page)
+ paginated_commits(page, per_page)
when 'users'
users.page(page).per(per_page)
else
@@ -37,7 +37,7 @@ module Gitlab
when 'wiki_blobs'
wiki_blobs_count.to_s
when 'commits'
- commits_count.to_s
+ formatted_limited_count(commits_count)
else
super
end
@@ -72,7 +72,7 @@ module Gitlab
end
def commits_count
- @commits_count ||= commits.count
+ @commits_count ||= commits(limit: count_limit).count
end
def single_commit_result?
@@ -86,6 +86,12 @@ module Gitlab
private
+ def paginated_commits(page, per_page)
+ results = commits(limit: limit_up_to_page(page, per_page))
+
+ Kaminari.paginate_array(results).page(page).per(per_page)
+ end
+
def paginated_blobs(blobs, page, per_page)
results = Kaminari.paginate_array(blobs).page(page).per(per_page)
@@ -139,21 +145,21 @@ module Gitlab
end
# rubocop: enable CodeReuse/ActiveRecord
- def commits
- @commits ||= find_commits(query)
+ def commits(limit:)
+ @commits ||= find_commits(query, limit: limit)
end
- def find_commits(query)
+ def find_commits(query, limit:)
return [] unless Ability.allowed?(@current_user, :download_code, @project)
- commits = find_commits_by_message(query)
+ commits = find_commits_by_message(query, limit: limit)
commit_by_sha = find_commit_by_sha(query)
commits |= [commit_by_sha] if commit_by_sha
commits
end
- def find_commits_by_message(query)
- project.repository.find_commits_by_message(query)
+ def find_commits_by_message(query, limit:)
+ project.repository.find_commits_by_message(query, repository_project_ref, nil, limit)
end
def find_commit_by_sha(query)
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
index 38adfc03ea7..fdb3fbc03bc 100644
--- a/lib/gitlab/project_template.rb
+++ b/lib/gitlab/project_template.rb
@@ -44,13 +44,13 @@ module Gitlab
ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift', 'illustrations/logos/swift.svg'),
ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'),
ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps.'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'),
- ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro'),
+ ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'),
ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby.'), 'https://gitlab.com/pages/gatsby'),
- ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo'),
- ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll'),
+ ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'),
+ ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'),
ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'),
- ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook'),
- ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo'),
+ ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'),
+ ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'),
ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman'),
ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'),
ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'),
diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb
index 4d48c4a3af7..5b688f83545 100644
--- a/lib/gitlab/prometheus/query_variables.rb
+++ b/lib/gitlab/prometheus/query_variables.rb
@@ -3,8 +3,10 @@
module Gitlab
module Prometheus
module QueryVariables
- def self.call(environment)
+ # start_time and end_time should be Time objects.
+ def self.call(environment, start_time: nil, end_time: nil)
{
+ __range: range(start_time, end_time),
ci_environment_slug: environment.slug,
kube_namespace: environment.deployment_namespace || '',
environment_filter: %{container_name!="POD",environment="#{environment.slug}"},
@@ -14,6 +16,16 @@ module Gitlab
ci_environment_name: environment.name
}
end
+
+ private
+
+ def self.range(start_time, end_time)
+ if start_time && end_time
+ range_seconds = (end_time - start_time).to_i
+ "#{range_seconds}s"
+ end
+ end
+ private_class_method :range
end
end
end
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index b03d8a4d254..213e3ba835d 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -71,6 +71,19 @@ module Gitlab
end
end
+ # Queries Prometheus with the given aggregate query and groups the results by mapping
+ # metric labels to their respective values.
+ #
+ # @return [Hash] mapping labels to their aggregate numeric values, or the empty hash if no results were found
+ def aggregate(aggregate_query, time: Time.now)
+ response = query(aggregate_query, time: time)
+ response.to_h do |result|
+ key = block_given? ? yield(result['metric']) : result['metric']
+ _timestamp, value = result['value']
+ [key, value.to_i]
+ end
+ end
+
def label_values(name = '__name__')
json_api_get("label/#{name}/values")
end
diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb
index 7c06698ffec..98db8ff761e 100644
--- a/lib/gitlab/quick_actions/merge_request_actions.rb
+++ b/lib/gitlab/quick_actions/merge_request_actions.rb
@@ -104,6 +104,23 @@ module Gitlab
command :target_branch do |branch_name|
@updates[:target_branch] = branch_name if project.repository.branch_exists?(branch_name)
end
+
+ desc _('Submit a review')
+ explanation _('Submit the current review.')
+ types MergeRequest
+ condition do
+ quick_action_target.persisted?
+ end
+ command :submit_review do
+ next if params[:review_id]
+
+ result = DraftNotes::PublishService.new(quick_action_target, current_user).execute
+ @execution_message[:submit_review] = if result[:status] == :success
+ _('Submitted the current review.')
+ else
+ result[:message]
+ end
+ end
end
def merge_orchestration_service
diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb
index 6e31c506438..a634f12345a 100644
--- a/lib/gitlab/redis/cache.rb
+++ b/lib/gitlab/redis/cache.rb
@@ -27,6 +27,10 @@ module Gitlab
# this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
super
end
+
+ def instrumentation_class
+ ::Gitlab::Instrumentation::Redis::Cache
+ end
end
end
end
diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb
index 0375e4a221a..42d5167beb3 100644
--- a/lib/gitlab/redis/queues.rb
+++ b/lib/gitlab/redis/queues.rb
@@ -28,6 +28,10 @@ module Gitlab
# this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
super
end
+
+ def instrumentation_class
+ ::Gitlab::Instrumentation::Redis::Queues
+ end
end
end
end
diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb
index 35356083f26..8ab53700932 100644
--- a/lib/gitlab/redis/shared_state.rb
+++ b/lib/gitlab/redis/shared_state.rb
@@ -30,6 +30,10 @@ module Gitlab
# this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent
super
end
+
+ def instrumentation_class
+ ::Gitlab::Instrumentation::Redis::SharedState
+ end
end
end
end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index 06ee81ba172..5584323789b 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -71,6 +71,10 @@ module Gitlab
# nil will force use of DEFAULT_REDIS_URL when config file is absent
nil
end
+
+ def instrumentation_class
+ raise NotImplementedError
+ end
end
def initialize(rails_env = nil)
@@ -100,6 +104,8 @@ module Gitlab
redis_url = config.delete(:url)
redis_uri = URI.parse(redis_url)
+ config[:instrumentation_class] ||= self.class.instrumentation_class
+
if redis_uri.scheme == 'unix'
# Redis::Store does not handle Unix sockets well, so let's do it for them
config[:path] = redis_uri.path
diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb
index d07d6440c6b..01aff48b08b 100644
--- a/lib/gitlab/reference_extractor.rb
+++ b/lib/gitlab/reference_extractor.rb
@@ -4,7 +4,7 @@ module Gitlab
# Extract possible GFM references from an arbitrary String for further processing.
class ReferenceExtractor < Banzai::ReferenceExtractor
REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project
- merge_request snippet commit commit_range directly_addressed_user epic).freeze
+ merge_request snippet commit commit_range directly_addressed_user epic iteration).freeze
attr_accessor :project, :current_user, :author
# This counter is increased by a number of references filtered out by
# banzai reference exctractor. Note that this counter is stateful and
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index c8b04ce2a5c..4caff8ae679 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -2,7 +2,93 @@
module Gitlab
module Regex
+ module Packages
+ CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt].freeze
+ CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze
+
+ def conan_file_name_regex
+ @conan_file_name_regex ||=
+ %r{\A#{(CONAN_RECIPE_FILES + CONAN_PACKAGE_FILES).join("|")}\z}.freeze
+ end
+
+ def conan_package_reference_regex
+ @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z}.freeze
+ end
+
+ def conan_revision_regex
+ @conan_revision_regex ||= %r{\A0\z}.freeze
+ end
+
+ def conan_recipe_component_regex
+ @conan_recipe_component_regex ||= %r{\A[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}\z}.freeze
+ end
+
+ def composer_package_version_regex
+ @composer_package_version_regex ||= %r{^v?(\d+(\.(\d+|x))*(-.+)?)}.freeze
+ end
+
+ def package_name_regex
+ @package_name_regex ||= %r{\A\@?(([\w\-\.\+]*)\/)*([\w\-\.]+)@?(([\w\-\.\+]*)\/)*([\w\-\.]*)\z}.freeze
+ end
+
+ def maven_file_name_regex
+ @maven_file_name_regex ||= %r{\A[A-Za-z0-9\.\_\-\+]+\z}.freeze
+ end
+
+ def maven_path_regex
+ @maven_path_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.\+]*)\z}.freeze
+ end
+
+ def maven_app_name_regex
+ @maven_app_name_regex ||= /\A[\w\-\.]+\z/.freeze
+ end
+
+ def maven_app_group_regex
+ maven_app_name_regex
+ end
+
+ def unbounded_semver_regex
+ # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+
+ # The order of the alternatives in <prerelease> are intentionally
+ # reordered to be greedy. Without this change, the unbounded regex would
+ # only partially match "v0.0.0-20201230123456-abcdefabcdef".
+ @unbounded_semver_regex ||= /
+ (?<major>0|[1-9]\d*)
+ \.(?<minor>0|[1-9]\d*)
+ \.(?<patch>0|[1-9]\d*)
+ (?:-(?<prerelease>(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0)(?:\.(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0))*))?
+ (?:\+(?<build>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?
+ /x.freeze
+ end
+
+ def semver_regex
+ @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options)
+ end
+
+ def go_package_regex
+ # A Go package name looks like a URL but is not; it:
+ # - Must not have a scheme, such as http:// or https://
+ # - Must not have a port number, such as :8080 or :8443
+
+ @go_package_regex ||= /
+ \b (?# word boundary)
+ (?<domain>
+ [0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])? (?# first domain)
+ (?:\.[0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])?)* (?# inner domains)
+ \.[a-z]{2,} (?# top-level domain)
+ )
+ (?<path>\/(?:
+ [-\/$_.+!*'(),0-9a-z] (?# plain URL character)
+ | %[0-9a-f]{2})* (?# URL encoded character)
+ )? (?# path)
+ \b (?# word boundary)
+ /ix.freeze
+ end
+ end
+
extend self
+ extend Packages
def project_name_regex
# The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff}
@@ -163,6 +249,10 @@ module Gitlab
def issue
@issue ||= /(?<issue>\d+\b)/
end
+
+ def base64_regex
+ @base64_regex ||= /(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?/.freeze
+ end
end
end
diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb
index 84885be9bda..cad127922df 100644
--- a/lib/gitlab/routing.rb
+++ b/lib/gitlab/routing.rb
@@ -4,6 +4,36 @@ module Gitlab
module Routing
extend ActiveSupport::Concern
+ class LegacyRedirector
+ # @params path_type [symbol] type of path to do "-" redirection
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/16854
+ def initialize(path_type)
+ @path_type = path_type
+ end
+
+ def call(_params, request)
+ ensure_valid_uri!(request)
+
+ # Only replace the last occurrence of `path`.
+ #
+ # `request.fullpath` includes the querystring
+ new_path = request.path.sub(%r{/#{@path_type}(/*)(?!.*#{@path_type})}, "/-/#{@path_type}\\1")
+ new_path = "#{new_path}?#{request.query_string}" if request.query_string.present?
+
+ new_path
+ end
+
+ private
+
+ def ensure_valid_uri!(request)
+ URI.parse(request.path)
+ rescue URI::InvalidURIError => e
+ # If url is invalid, raise custom error,
+ # which can be ignored by monitoring tools.
+ raise ActionController::RoutingError.new(e.message)
+ end
+ end
+
mattr_accessor :_includers
self._includers = []
@@ -44,20 +74,10 @@ module Gitlab
end
def self.redirect_legacy_paths(router, *paths)
- build_redirect_path = lambda do |request, _params, path|
- # Only replace the last occurrence of `path`.
- #
- # `request.fullpath` includes the querystring
- new_path = request.path.sub(%r{/#{path}(/*)(?!.*#{path})}, "/-/#{path}\\1")
- new_path = "#{new_path}?#{request.query_string}" if request.query_string.present?
-
- new_path
- end
-
paths.each do |path|
router.match "/#{path}(/*rest)",
via: [:get, :post, :patch, :delete],
- to: router.redirect { |params, request| build_redirect_path.call(request, params, path) },
+ to: router.redirect(LegacyRedirector.new(path)),
as: "legacy_#{path}_redirect"
end
end
diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb
index 9a5917ffba9..36a3a491de6 100644
--- a/lib/gitlab/rugged_instrumentation.rb
+++ b/lib/gitlab/rugged_instrumentation.rb
@@ -3,12 +3,13 @@
module Gitlab
module RuggedInstrumentation
def self.query_time
- query_time = SafeRequestStore[:rugged_query_time] ||= 0
+ query_time = SafeRequestStore[:rugged_query_time] || 0
query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
end
- def self.query_time=(duration)
- SafeRequestStore[:rugged_query_time] = duration
+ def self.add_query_time(duration)
+ SafeRequestStore[:rugged_query_time] ||= 0
+ SafeRequestStore[:rugged_query_time] += duration
end
def self.query_time_ms
diff --git a/lib/gitlab/search_context.rb b/lib/gitlab/search_context.rb
new file mode 100644
index 00000000000..c3bb0ff26f2
--- /dev/null
+++ b/lib/gitlab/search_context.rb
@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Gitlab
+ # Holds the contextual data used by navbar search component to
+ # determine the search scope, whether to search for code, or if
+ # a search should target snippets.
+ #
+ # Use the SearchContext::Builder to create an instance of this class
+ class SearchContext
+ attr_accessor :project, :project_metadata, :ref,
+ :group, :group_metadata,
+ :snippets,
+ :scope, :search_url
+
+ def initialize
+ @ref = nil
+ @project = nil
+ @project_metadata = {}
+ @group = nil
+ @group_metadata = {}
+ @snippets = []
+ @scope = nil
+ @search_url = nil
+ end
+
+ def for_project?
+ project.present? && project.persisted?
+ end
+
+ def for_group?
+ group.present? && group.persisted?
+ end
+
+ def for_snippets?
+ snippets.any?
+ end
+
+ def code_search?
+ project.present? && scope.nil?
+ end
+
+ class Builder
+ def initialize(view_context)
+ @view_context = view_context
+ @snippets = []
+ end
+
+ def with_snippet(snippet)
+ @snippets << snippet
+
+ self
+ end
+
+ def with_project(project)
+ @project = project
+ with_group(project&.group)
+
+ self
+ end
+
+ def with_group(group)
+ @group = group
+
+ self
+ end
+
+ def with_ref(ref)
+ @ref = ref
+
+ self
+ end
+
+ def build!
+ SearchContext.new.tap do |context|
+ context.project = @project
+ context.group = @group
+ context.ref = @ref
+ context.snippets = @snippets.dup
+ context.scope = search_scope
+ context.search_url = search_url
+ context.group_metadata = group_search_metadata(@group)
+ context.project_metadata = project_search_metadata(@project)
+ end
+ end
+
+ private
+
+ attr_accessor :view_context
+
+ def project_search_metadata(project)
+ return {} unless project
+
+ {
+ project_path: project.path,
+ name: project.name,
+ issues_path: view_context.project_issues_path(project),
+ mr_path: view_context.project_merge_requests_path(project),
+ issues_disabled: !project.issues_enabled?
+ }
+ end
+
+ def group_search_metadata(group)
+ return {} unless group
+
+ {
+ group_path: group.path,
+ name: group.name,
+ issues_path: view_context.issues_group_path(group),
+ mr_path: view_context.merge_requests_group_path(group)
+ }
+ end
+
+ def search_url
+ if @project.present?
+ view_context.search_path(project_id: @project.id)
+ elsif @group.present?
+ view_context.search_path(group_id: @group.id)
+ else
+ view_context.search_path
+ end
+ end
+
+ def search_scope
+ if view_context.current_controller?(:issues)
+ 'issues'
+ elsif view_context.current_controller?(:merge_requests)
+ 'merge_requests'
+ elsif view_context.current_controller?(:wikis)
+ 'wiki_blobs'
+ elsif view_context.current_controller?(:commits)
+ 'commits'
+ else nil
+ end
+ end
+ end
+
+ module ControllerConcern
+ extend ActiveSupport::Concern
+
+ included do
+ helper_method :search_context
+ end
+
+ # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ #
+ # Introspect the current controller's assignments and
+ # and builds the proper SearchContext object for it.
+ def search_context
+ builder = Builder.new(view_context)
+
+ builder.with_snippet(@snippet) if @snippet.present?
+ @snippets.each(&builder.method(:with_snippet)) if @snippets.present?
+ builder.with_project(@project) if @project.present? && @project.persisted?
+ builder.with_group(@group) if @group.present? && @group.persisted?
+ builder.with_ref(@ref) if @ref.present?
+
+ builder.build!
+ end
+ # rubocop:enable Gitlab/ModuleWithInstanceVariables
+ end
+ end
+end
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index c35ee62163a..6239158ef06 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -26,7 +26,7 @@ module Gitlab
@default_project_filter = default_project_filter
end
- def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true)
+ def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true, preload_method: nil)
collection = case scope
when 'projects'
projects
diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb
index 3f36725cb66..64a30fbe16c 100644
--- a/lib/gitlab/setup_helper.rb
+++ b/lib/gitlab/setup_helper.rb
@@ -79,6 +79,7 @@ module Gitlab
config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby
config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path }
config[:bin_dir] = Gitlab.config.gitaly.client_path
+ config[:gitlab] = { url: Gitlab.config.gitlab.url }
TomlRB.dump(config)
end
@@ -97,7 +98,8 @@ module Gitlab
def configuration_toml(gitaly_dir, storage_paths)
nodes = [{ storage: 'default', address: "unix:#{gitaly_dir}/gitaly.socket", primary: true, token: 'secret' }]
storages = [{ name: 'default', node: nodes }]
- config = { socket_path: "#{gitaly_dir}/praefect.socket", virtual_storage: storages }
+ failover = { enabled: false }
+ config = { socket_path: "#{gitaly_dir}/praefect.socket", memory_queue_enabled: true, virtual_storage: storages, failover: failover }
config[:token] = 'secret' if Rails.env.test?
TomlRB.dump(config)
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index 4e0d3da1868..633291dcdf3 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -14,8 +14,8 @@ module Gitlab
].compact.freeze
DEFAULT_WORKERS = [
- DummyWorker.new('default', weight: 1),
- DummyWorker.new('mailers', weight: 2)
+ DummyWorker.new('default', weight: 1, tags: []),
+ DummyWorker.new('mailers', weight: 2, tags: [])
].map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze
class << self
diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb
index 0d0efe8ffbd..a256632bc12 100644
--- a/lib/gitlab/sidekiq_config/cli_methods.rb
+++ b/lib/gitlab/sidekiq_config/cli_methods.rb
@@ -28,6 +28,7 @@ module Gitlab
has_external_dependencies: lambda { |value| value == 'true' },
name: :to_s,
resource_boundary: :to_sym,
+ tags: :to_sym,
urgency: :to_sym
}.freeze
@@ -117,7 +118,11 @@ module Gitlab
raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block
- lambda { |queue| values.map(&values_block).include?(queue[lhs.to_sym]) }
+ lambda do |queue|
+ comparator = Array(queue[lhs.to_sym]).to_set
+
+ values.map(&values_block).to_set.intersect?(comparator)
+ end
end
end
end
diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb
index bd205c81931..7568840410b 100644
--- a/lib/gitlab/sidekiq_config/dummy_worker.rb
+++ b/lib/gitlab/sidekiq_config/dummy_worker.rb
@@ -12,7 +12,8 @@ module Gitlab
urgency: :get_urgency,
resource_boundary: :get_worker_resource_boundary,
idempotent: :idempotent?,
- weight: :get_weight
+ weight: :get_weight,
+ tags: :get_tags
}.freeze
def initialize(queue, attributes = {})
diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb
index ec7a82f6459..46fa0aa5be1 100644
--- a/lib/gitlab/sidekiq_config/worker.rb
+++ b/lib/gitlab/sidekiq_config/worker.rb
@@ -6,7 +6,7 @@ module Gitlab
include Comparable
attr_reader :klass
- delegate :feature_category_not_owned?, :get_feature_category,
+ delegate :feature_category_not_owned?, :get_feature_category, :get_tags,
:get_urgency, :get_weight, :get_worker_resource_boundary,
:idempotent?, :queue, :queue_namespace,
:worker_has_external_dependencies?,
@@ -52,7 +52,8 @@ module Gitlab
urgency: get_urgency,
resource_boundary: get_worker_resource_boundary,
weight: get_weight,
- idempotent: idempotent?
+ idempotent: idempotent?,
+ tags: get_tags
}
end
diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb
index 64782e1e1d1..8894b48417c 100644
--- a/lib/gitlab/sidekiq_logging/json_formatter.rb
+++ b/lib/gitlab/sidekiq_logging/json_formatter.rb
@@ -18,10 +18,15 @@ module Gitlab
when String
output[:message] = data
when Hash
- convert_to_iso8601!(data)
- convert_retry_to_integer!(data)
- stringify_args!(data)
output.merge!(data)
+
+ # jobstr is redundant and can include information we wanted to
+ # exclude (like arguments)
+ output.delete(:jobstr)
+
+ convert_to_iso8601!(output)
+ convert_retry_to_integer!(output)
+ process_args!(output)
end
output.to_json + "\n"
@@ -56,8 +61,11 @@ module Gitlab
end
end
- def stringify_args!(payload)
- payload['args'] = Gitlab::Utils::LogLimitedArray.log_limited_array(payload['args'].map(&:to_s)) if payload['args']
+ def process_args!(payload)
+ return unless payload['args']
+
+ payload['args'] = Gitlab::ErrorTracking::Processor::SidekiqProcessor
+ .loggable_arguments(payload['args'], payload['class'])
end
end
end
diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb
index 4e39120f8a7..eb845c5ff8d 100644
--- a/lib/gitlab/sidekiq_logging/structured_logger.rb
+++ b/lib/gitlab/sidekiq_logging/structured_logger.rb
@@ -27,7 +27,7 @@ module Gitlab
private
def add_instrumentation_keys!(job, output_payload)
- output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS))
+ output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper.keys))
end
def add_logging_extras!(job, output_payload)
@@ -36,6 +36,10 @@ module Gitlab
)
end
+ def add_db_counters!(job, output_payload)
+ output_payload.merge!(job.slice(*::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS))
+ end
+
def log_job_start(payload)
payload['message'] = "#{base_message(payload)}: start"
payload['job_status'] = 'start'
@@ -50,6 +54,7 @@ module Gitlab
payload = payload.dup
add_instrumentation_keys!(job, payload)
add_logging_extras!(job, payload)
+ add_db_counters!(job, payload)
elapsed_time = elapsed(started_time)
add_time_keys!(elapsed_time, payload)
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb
index ddd1b91410b..bb0c18735bb 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb
@@ -5,9 +5,6 @@ module Gitlab
module DuplicateJobs
class Client
def call(worker_class, job, queue, _redis_pool, &block)
- # We don't try to deduplicate jobs that are scheduled in the future
- return yield if job['at']
-
DuplicateJob.new(job, queue).schedule(&block)
end
end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index fa742d07af2..0dc53c61e84 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -18,13 +18,13 @@ module Gitlab
# When new jobs can be scheduled again, the strategy calls `#delete`.
class DuplicateJob
DUPLICATE_KEY_TTL = 6.hours
+ DEFAULT_STRATEGY = :until_executing
attr_reader :existing_jid
- def initialize(job, queue_name, strategy: :until_executing)
+ def initialize(job, queue_name)
@job = job
@queue_name = queue_name
- @strategy = strategy
end
# This will continue the middleware chain if the job should be scheduled
@@ -41,12 +41,12 @@ module Gitlab
end
# This method will return the jid that was set in redis
- def check!
+ def check!(expiry = DUPLICATE_KEY_TTL)
read_jid = nil
Sidekiq.redis do |redis|
redis.multi do |multi|
- redis.set(idempotency_key, jid, ex: DUPLICATE_KEY_TTL, nx: true)
+ redis.set(idempotency_key, jid, ex: expiry, nx: true)
read_jid = redis.get(idempotency_key)
end
end
@@ -60,6 +60,10 @@ module Gitlab
end
end
+ def scheduled?
+ scheduled_at.present?
+ end
+
def duplicate?
raise "Call `#check!` first to check for existing duplicates" unless existing_jid
@@ -67,14 +71,36 @@ module Gitlab
end
def droppable?
- idempotent? && duplicate? && ::Feature.disabled?("disable_#{queue_name}_deduplication")
+ idempotent? && ::Feature.disabled?("disable_#{queue_name}_deduplication")
+ end
+
+ def scheduled_at
+ job['at']
+ end
+
+ def options
+ return {} unless worker_klass
+ return {} unless worker_klass.respond_to?(:get_deduplication_options)
+
+ worker_klass.get_deduplication_options
end
private
- attr_reader :queue_name, :strategy, :job
+ attr_reader :queue_name, :job
attr_writer :existing_jid
+ def worker_klass
+ @worker_klass ||= worker_class_name.to_s.safe_constantize
+ end
+
+ def strategy
+ return DEFAULT_STRATEGY unless worker_klass
+ return DEFAULT_STRATEGY unless worker_klass.respond_to?(:idempotent?)
+
+ worker_klass.get_deduplicate_strategy
+ end
+
def worker_class_name
job['class']
end
@@ -104,11 +130,10 @@ module Gitlab
end
def idempotent?
- worker_class = worker_class_name.to_s.safe_constantize
- return false unless worker_class
- return false unless worker_class.respond_to?(:idempotent?)
+ return false unless worker_klass
+ return false unless worker_klass.respond_to?(:idempotent?)
- worker_class.idempotent?
+ worker_klass.idempotent?
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
index 674e436b714..0ed4912c4cc 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb
@@ -13,13 +13,13 @@ module Gitlab
end
def schedule(job)
- if duplicate_job.check! && duplicate_job.duplicate?
+ if deduplicatable_job? && check! && duplicate_job.duplicate?
job['duplicate-of'] = duplicate_job.existing_jid
- end
- if duplicate_job.droppable?
- Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(job, "dropped until executing")
- return false
+ if duplicate_job.droppable?
+ Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(job, "dropped until executing")
+ return false
+ end
end
yield
@@ -34,6 +34,22 @@ module Gitlab
private
attr_reader :duplicate_job
+
+ def deduplicatable_job?
+ !duplicate_job.scheduled? || duplicate_job.options[:including_scheduled]
+ end
+
+ def check!
+ duplicate_job.check!(expiry)
+ end
+
+ def expiry
+ return DuplicateJob::DUPLICATE_KEY_TTL unless duplicate_job.scheduled?
+
+ time_diff = duplicate_job.scheduled_at.to_i - Time.now.to_i
+
+ time_diff > 0 ? time_diff : DuplicateJob::DUPLICATE_KEY_TTL
+ end
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb
index 61ed2fe1a06..6a942a6ce06 100644
--- a/lib/gitlab/sidekiq_middleware/server_metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb
@@ -47,6 +47,10 @@ module Gitlab
@metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time)
@metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000)
@metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(job))
+ @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(job))
+ @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(job))
+ @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(job))
+ @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(job))
end
end
@@ -54,15 +58,19 @@ module Gitlab
def init_metrics
{
- sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'),
- sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'),
- sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all),
- sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all)
+ sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
+ sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
+ sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
+ sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
+ sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS),
+ sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS),
+ sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS),
+ sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'),
+ sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'),
+ sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'),
+ sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'),
+ sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all),
+ sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all)
}
end
@@ -70,6 +78,22 @@ module Gitlab
defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0
end
+ def get_redis_time(job)
+ job.fetch(:redis_duration_s, 0)
+ end
+
+ def get_redis_calls(job)
+ job.fetch(:redis_calls, 0)
+ end
+
+ def get_elasticsearch_time(job)
+ job.fetch(:elasticsearch_duration_s, 0)
+ end
+
+ def get_elasticsearch_calls(job)
+ job.fetch(:elasticsearch_calls, 0)
+ end
+
def get_gitaly_time(job)
job.fetch(:gitaly_duration_s, 0)
end
diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb
index 342dae456a8..2d8df2ca204 100644
--- a/lib/gitlab/slash_commands/presenters/help.rb
+++ b/lib/gitlab/slash_commands/presenters/help.rb
@@ -21,7 +21,7 @@ module Gitlab
This chatops integration does not have any commands that can be
executed.
- #{footer}
+ #{help_footer}
MESSAGE
end
diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb
index 9911f9e62a6..1d253ca90f3 100644
--- a/lib/gitlab/snippet_search_results.rb
+++ b/lib/gitlab/snippet_search_results.rb
@@ -11,7 +11,7 @@ module Gitlab
@query = query
end
- def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE)
+ def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil)
paginated_objects(snippet_titles, page, per_page)
end
diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb
index d0f12c8364a..ec404ebd309 100644
--- a/lib/gitlab/sourcegraph.rb
+++ b/lib/gitlab/sourcegraph.rb
@@ -19,7 +19,7 @@ module Gitlab
private
def feature
- Feature.get(:sourcegraph)
+ Feature.get(:sourcegraph) # rubocop:disable Gitlab/AvoidFeatureGet
end
end
end
diff --git a/lib/gitlab/suggestions/commit_message.rb b/lib/gitlab/suggestions/commit_message.rb
new file mode 100644
index 00000000000..d59a8fc3730
--- /dev/null
+++ b/lib/gitlab/suggestions/commit_message.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Suggestions
+ class CommitMessage
+ DEFAULT_SUGGESTION_COMMIT_MESSAGE =
+ 'Apply %{suggestions_count} suggestion(s) to %{files_count} file(s)'
+
+ def initialize(user, suggestion_set)
+ @user = user
+ @suggestion_set = suggestion_set
+ end
+
+ def message
+ project = suggestion_set.project
+ user_defined_message = project.suggestion_commit_message.presence
+ message = user_defined_message || DEFAULT_SUGGESTION_COMMIT_MESSAGE
+
+ Gitlab::StringPlaceholderReplacer
+ .replace_string_placeholders(message, PLACEHOLDERS_REGEX) do |key|
+ PLACEHOLDERS[key].call(user, suggestion_set)
+ end
+ end
+
+ def self.format_paths(paths)
+ paths.sort.join(', ')
+ end
+
+ private_class_method :format_paths
+
+ private
+
+ attr_reader :user, :suggestion_set
+
+ PLACEHOLDERS = {
+ 'branch_name' => ->(user, suggestion_set) { suggestion_set.branch },
+ 'files_count' => ->(user, suggestion_set) { suggestion_set.file_paths.length },
+ 'file_paths' => ->(user, suggestion_set) { format_paths(suggestion_set.file_paths) },
+ 'project_name' => ->(user, suggestion_set) { suggestion_set.project.name },
+ 'project_path' => ->(user, suggestion_set) { suggestion_set.project.path },
+ 'user_full_name' => ->(user, suggestion_set) { user.name },
+ 'username' => ->(user, suggestion_set) { user.username },
+ 'suggestions_count' => ->(user, suggestion_set) { suggestion_set.suggestions.size }
+ }.freeze
+
+ # This regex is built dynamically using the keys from the PLACEHOLDER struct.
+ # So, we can easily add new placeholder just by modifying the PLACEHOLDER hash.
+ # This regex will build the new PLACEHOLDER_REGEX with the new information
+ PLACEHOLDERS_REGEX = Regexp.union(PLACEHOLDERS.keys.map do |key|
+ Regexp.new(Regexp.escape(key))
+ end).freeze
+ end
+ end
+end
diff --git a/lib/gitlab/suggestions/file_suggestion.rb b/lib/gitlab/suggestions/file_suggestion.rb
new file mode 100644
index 00000000000..73b9800f0b8
--- /dev/null
+++ b/lib/gitlab/suggestions/file_suggestion.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Suggestions
+ class FileSuggestion
+ include Gitlab::Utils::StrongMemoize
+
+ SuggestionForDifferentFileError = Class.new(StandardError)
+
+ def initialize
+ @suggestions = []
+ end
+
+ def add_suggestion(new_suggestion)
+ if for_different_file?(new_suggestion)
+ raise SuggestionForDifferentFileError,
+ 'Only add suggestions for the same file.'
+ end
+
+ suggestions << new_suggestion
+ end
+
+ def line_conflict?
+ strong_memoize(:line_conflict) do
+ _line_conflict?
+ end
+ end
+
+ def new_content
+ @new_content ||= _new_content
+ end
+
+ def file_path
+ @file_path ||= _file_path
+ end
+
+ private
+
+ attr_accessor :suggestions
+
+ def blob
+ first_suggestion&.diff_file&.new_blob
+ end
+
+ def blob_data_lines
+ blob.load_all_data!
+ blob.data.lines
+ end
+
+ def current_content
+ @current_content ||= blob.nil? ? [''] : blob_data_lines
+ end
+
+ def _new_content
+ current_content.tap do |content|
+ suggestions.each do |suggestion|
+ range = line_range(suggestion)
+ content[range] = suggestion.to_content
+ end
+ end.join
+ end
+
+ def line_range(suggestion)
+ suggestion.from_line_index..suggestion.to_line_index
+ end
+
+ def for_different_file?(suggestion)
+ file_path && file_path != suggestion_file_path(suggestion)
+ end
+
+ def suggestion_file_path(suggestion)
+ suggestion&.diff_file&.file_path
+ end
+
+ def first_suggestion
+ suggestions.first
+ end
+
+ def _file_path
+ suggestion_file_path(first_suggestion)
+ end
+
+ def _line_conflict?
+ has_conflict = false
+
+ suggestions.each_with_object([]) do |suggestion, ranges|
+ range_in_test = line_range(suggestion)
+
+ if has_range_conflict?(range_in_test, ranges)
+ has_conflict = true
+ break
+ end
+
+ ranges << range_in_test
+ end
+
+ has_conflict
+ end
+
+ def has_range_conflict?(range_in_test, ranges)
+ ranges.any? do |range|
+ range.overlaps?(range_in_test)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/suggestions/suggestion_set.rb b/lib/gitlab/suggestions/suggestion_set.rb
new file mode 100644
index 00000000000..22abef98bf0
--- /dev/null
+++ b/lib/gitlab/suggestions/suggestion_set.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Suggestions
+ class SuggestionSet
+ attr_reader :suggestions
+
+ def initialize(suggestions)
+ @suggestions = suggestions
+ end
+
+ def project
+ first_suggestion.project
+ end
+
+ def branch
+ first_suggestion.branch
+ end
+
+ def valid?
+ error_message.nil?
+ end
+
+ def error_message
+ @error_message ||= _error_message
+ end
+
+ def actions
+ @actions ||= suggestions_per_file.map do |file_path, file_suggestion|
+ {
+ action: 'update',
+ file_path: file_path,
+ content: file_suggestion.new_content
+ }
+ end
+ end
+
+ def file_paths
+ @file_paths ||= suggestions.map(&:file_path).uniq
+ end
+
+ private
+
+ def first_suggestion
+ suggestions.first
+ end
+
+ def suggestions_per_file
+ @suggestions_per_file ||= _suggestions_per_file
+ end
+
+ def _suggestions_per_file
+ suggestions.each_with_object({}) do |suggestion, result|
+ file_path = suggestion.diff_file.file_path
+ file_suggestion = result[file_path] ||= FileSuggestion.new
+ file_suggestion.add_suggestion(suggestion)
+ end
+ end
+
+ def file_suggestions
+ suggestions_per_file.values
+ end
+
+ def first_file_suggestion
+ file_suggestions.first
+ end
+
+ def _error_message
+ suggestions.each do |suggestion|
+ message = error_for_suggestion(suggestion)
+
+ return message if message
+ end
+
+ has_line_conflict = file_suggestions.any? do |file_suggestion|
+ file_suggestion.line_conflict?
+ end
+
+ if has_line_conflict
+ return _('Suggestions are not applicable as their lines cannot overlap.')
+ end
+
+ nil
+ end
+
+ def error_for_suggestion(suggestion)
+ unless suggestion.diff_file
+ return _('A file was not found.')
+ end
+
+ unless on_same_branch?(suggestion)
+ return _('Suggestions must all be on the same branch.')
+ end
+
+ unless suggestion.appliable?(cached: false)
+ return _('A suggestion is not applicable.')
+ end
+
+ unless latest_source_head?(suggestion)
+ return _('A file has been changed.')
+ end
+
+ nil
+ end
+
+ def on_same_branch?(suggestion)
+ branch == suggestion.branch
+ end
+
+ # Checks whether the latest source branch HEAD matches with
+ # the position HEAD we're using to update the file content. Since
+ # the persisted HEAD is updated async (for MergeRequest),
+ # it's more consistent to fetch this data directly from the
+ # repository.
+ def latest_source_head?(suggestion)
+ suggestion.position.head_sha == suggestion.noteable.source_branch_sha
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 5992f24f4e9..72783a2d682 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -23,7 +23,8 @@ module Gitlab
Theme.new(9, 'Red', 'ui-red'),
Theme.new(10, 'Light Red', 'ui-light-red'),
Theme.new(2, 'Dark', 'ui-dark'),
- Theme.new(3, 'Light', 'ui-light')
+ Theme.new(3, 'Light', 'ui-light'),
+ Theme.new(11, 'Dark Mode (alpha)', 'gl-dark')
].freeze
# Convenience method to get a space-separated String of all the theme
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 329f87d8be8..cd15130cee6 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -41,7 +41,7 @@ module Gitlab
when Wiki
wiki_url(object, **options)
when WikiPage
- instance.project_wiki_url(object.wiki.project, object.slug, **options)
+ wiki_page_url(object.wiki, object, **options)
when ::DesignManagement::Design
design_url(object, **options)
else
@@ -78,12 +78,21 @@ module Gitlab
end
end
- def wiki_url(object, **options)
- if object.container.is_a?(Project)
- instance.project_wiki_url(object.container, Wiki::HOMEPAGE, **options)
- else
- raise NotImplementedError.new("No URL builder defined for #{object.inspect}")
- end
+ def wiki_url(wiki, **options)
+ return wiki_page_url(wiki, Wiki::HOMEPAGE, **options) unless options[:action]
+
+ options[:controller] = 'projects/wikis'
+ options[:namespace_id] = wiki.container.namespace
+ options[:project_id] = wiki.container
+
+ instance.url_for(**options)
+ end
+
+ def wiki_page_url(wiki, page, **options)
+ options[:action] ||= :show
+ options[:id] = page
+
+ wiki_url(wiki, **options)
end
def design_url(design, **options)
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index e60c786b52c..7b6f5e69ee1 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -1,27 +1,25 @@
# frozen_string_literal: true
-# For hardening usage ping and make it easier to add measures there is in place
-# * alt_usage_data method
-# handles StandardError and fallbacks into -1 this way not all measures fail if we encounter one exception
+# When developing usage data metrics use the below usage data interface methods
+# unless you have good reasons to implement custom usage data
+# See `lib/gitlab/utils/usage_data.rb`
#
-# Examples:
-# alt_usage_data { Gitlab::VERSION }
-# alt_usage_data { Gitlab::CurrentSettings.uuid }
-#
-# * redis_usage_data method
-# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
-# returns -1 when a block is sent or hash with all values -1 when a counter is sent
-# different behaviour due to 2 different implementations of redis counter
-#
-# Examples:
-# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
-# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
+# Examples
+# issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
+# active_user_count: count(User.active)
+# alt_usage_data { Gitlab::VERSION }
+# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
+# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
+
module Gitlab
class UsageData
BATCH_SIZE = 100
- FALLBACK = -1
class << self
+ include Gitlab::Utils::UsageData
+ include Gitlab::Utils::StrongMemoize
+ include Gitlab::UsageDataConcerns::Topology
+
def data(force_refresh: false)
Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do
uncached_data
@@ -29,12 +27,15 @@ module Gitlab
end
def uncached_data
+ clear_memoized_limits
+
license_usage_data
.merge(system_usage_data)
.merge(features_usage_data)
.merge(components_usage_data)
.merge(cycle_analytics_usage_data)
.merge(object_store_usage_data)
+ .merge(topology_usage_data)
.merge(recording_ce_finish_data)
end
@@ -44,7 +45,7 @@ module Gitlab
def license_usage_data
{
- recorded_at: Time.now, # should be calculated very first
+ recorded_at: recorded_at,
uuid: alt_usage_data { Gitlab::CurrentSettings.uuid },
hostname: alt_usage_data { Gitlab.config.gitlab.host },
version: alt_usage_data { Gitlab::VERSION },
@@ -54,6 +55,10 @@ module Gitlab
}
end
+ def recorded_at
+ Time.now
+ end
+
def recording_ce_finish_data
{
recording_ce_finished_at: Time.now
@@ -64,6 +69,7 @@ module Gitlab
# rubocop: disable CodeReuse/ActiveRecord
def system_usage_data
alert_bot_incident_count = count(::Issue.authored(::User.alert_bot))
+ issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot))
{
counts: {
@@ -114,7 +120,9 @@ module Gitlab
issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data,
- issues_created_gitlab_alerts: count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot)),
+ issues_created_from_alerts: total_alert_issues,
+ issues_created_gitlab_alerts: issues_created_manually_from_alerts,
+ issues_created_manually_from_alerts: issues_created_manually_from_alerts,
incident_issues: alert_bot_incident_count,
alert_bot_incident_issues: alert_bot_incident_count,
incident_labeled_issues: count(::Issue.with_label_attributes(IncidentManagement::CreateIssueService::INCIDENT_LABEL)),
@@ -131,11 +139,17 @@ module Gitlab
projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
projects_with_alerts_service_enabled: count(AlertsService.active),
projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id),
+ projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id),
+ projects_with_terraform_states: distinct_count(::Terraform::State, :project_id),
protected_branches: count(ProtectedBranch),
releases: count(Release),
remote_mirrors: count(RemoteMirror),
snippets: count(Snippet),
+ personal_snippets: count(PersonalSnippet),
+ project_snippets: count(ProjectSnippet),
suggestions: count(Suggestion),
+ terraform_reports: count(::Ci::JobArtifact.terraform_reports),
+ terraform_states: count(::Terraform::State),
todos: count(Todo),
uploads: count(Upload),
web_hooks: count(WebHook),
@@ -147,7 +161,8 @@ module Gitlab
usage_counters,
user_preferences_usage,
ingress_modsecurity_usage,
- container_expiration_policies_usage
+ container_expiration_policies_usage,
+ merge_requests_usage(default_time_period)
)
}
end
@@ -174,19 +189,20 @@ module Gitlab
def features_usage_data_ce
{
- container_registry_enabled: alt_usage_data { Gitlab.config.registry.enabled },
+ instance_auto_devops_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.auto_devops_enabled? },
+ container_registry_enabled: alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled },
dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled,
- gitlab_shared_runners_enabled: alt_usage_data { Gitlab.config.gitlab_ci.shared_runners_enabled },
- gravatar_enabled: alt_usage_data { Gitlab::CurrentSettings.gravatar_enabled? },
- ldap_enabled: alt_usage_data { Gitlab.config.ldap.enabled },
- mattermost_enabled: alt_usage_data { Gitlab.config.mattermost.enabled },
- omniauth_enabled: alt_usage_data { Gitlab::Auth.omniauth_enabled? },
- prometheus_metrics_enabled: alt_usage_data { Gitlab::Metrics.prometheus_metrics_enabled? },
- reply_by_email_enabled: alt_usage_data { Gitlab::IncomingEmail.enabled? },
- signup_enabled: alt_usage_data { Gitlab::CurrentSettings.allow_signup? },
- web_ide_clientside_preview_enabled: alt_usage_data { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? },
+ gitlab_shared_runners_enabled: alt_usage_data(fallback: nil) { Gitlab.config.gitlab_ci.shared_runners_enabled },
+ gravatar_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gravatar_enabled? },
+ ldap_enabled: alt_usage_data(fallback: nil) { Gitlab.config.ldap.enabled },
+ mattermost_enabled: alt_usage_data(fallback: nil) { Gitlab.config.mattermost.enabled },
+ omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? },
+ prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? },
+ reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? },
+ signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? },
+ web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? },
ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity),
- grafana_link_enabled: alt_usage_data { Gitlab::CurrentSettings.grafana_enabled? }
+ grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }
}
end
@@ -213,14 +229,15 @@ module Gitlab
def components_usage_data
{
- git: { version: alt_usage_data { Gitlab::Git.version } },
+ git: { version: alt_usage_data(fallback: { major: -1 }) { Gitlab::Git.version } },
gitaly: {
version: alt_usage_data { Gitaly::Server.all.first.server_version },
servers: alt_usage_data { Gitaly::Server.count },
- filesystems: alt_usage_data { Gitaly::Server.filesystems }
+ clusters: alt_usage_data { Gitaly::Server.gitaly_clusters },
+ filesystems: alt_usage_data(fallback: ["-1"]) { Gitaly::Server.filesystems }
},
gitlab_pages: {
- enabled: alt_usage_data { Gitlab.config.pages.enabled },
+ enabled: alt_usage_data(fallback: nil) { Gitlab.config.pages.enabled },
version: alt_usage_data { Gitlab::Pages::VERSION }
},
database: {
@@ -382,65 +399,67 @@ module Gitlab
{} # augmented in EE
end
- def count(relation, column = nil, batch: true, start: nil, finish: nil)
- if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
- Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish)
- else
- relation.count
- end
- rescue ActiveRecord::StatementInvalid
- FALLBACK
- end
+ # rubocop: disable CodeReuse/ActiveRecord
+ def merge_requests_usage(time_period)
+ query =
+ Event
+ .where(target_type: Event::TARGET_TYPES[:merge_request].to_s)
+ .where(time_period)
+
+ merge_request_users = distinct_count(
+ query,
+ :author_id,
+ batch_size: 5_000, # Based on query performance, this is the optimal batch size.
+ start: User.minimum(:id),
+ finish: User.maximum(:id)
+ )
- def distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)
- if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
- Gitlab::Database::BatchCount.batch_distinct_count(relation, column, start: start, finish: finish)
- else
- relation.distinct_count_by(column)
- end
- rescue ActiveRecord::StatementInvalid
- FALLBACK
+ {
+ merge_requests_users: merge_request_users
+ }
end
+ # rubocop: enable CodeReuse/ActiveRecord
- def alt_usage_data(value = nil, fallback: FALLBACK, &block)
- if block_given?
- yield
+ def installation_type
+ if Rails.env.production?
+ Gitlab::INSTALLATION_TYPE
else
- value
+ "gitlab-development-kit"
end
- rescue
- fallback
end
- def redis_usage_data(counter = nil, &block)
- if block_given?
- redis_usage_counter(&block)
- elsif counter.present?
- redis_usage_data_totals(counter)
- end
+ def default_time_period
+ { created_at: 28.days.ago..Time.current }
end
private
- def redis_usage_counter
- yield
- rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
- FALLBACK
+ def total_alert_issues
+ # Remove prometheus table queries once they are deprecated
+ # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407.
+ [
+ count(Issue.with_alert_management_alerts),
+ count(::Issue.with_self_managed_prometheus_alert_events),
+ count(::Issue.with_prometheus_alert_events)
+ ].reduce(:+)
end
- def redis_usage_data_totals(counter)
- counter.totals
- rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
- counter.fallback_totals
+ def user_minimum_id
+ strong_memoize(:user_minimum_id) do
+ ::User.minimum(:id)
+ end
end
- def installation_type
- if Rails.env.production?
- Gitlab::INSTALLATION_TYPE
- else
- "gitlab-development-kit"
+ def user_maximum_id
+ strong_memoize(:user_maximum_id) do
+ ::User.maximum(:id)
end
end
+
+ def clear_memoized_limits
+ clear_memoization(:user_minimum_id)
+ clear_memoization(:user_maximum_id)
+ end
end
end
end
diff --git a/lib/gitlab/usage_data_concerns/topology.rb b/lib/gitlab/usage_data_concerns/topology.rb
new file mode 100644
index 00000000000..6e1d29f2a17
--- /dev/null
+++ b/lib/gitlab/usage_data_concerns/topology.rb
@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module UsageDataConcerns
+ module Topology
+ include Gitlab::Utils::UsageData
+
+ JOB_TO_SERVICE_NAME = {
+ 'gitlab-rails' => 'web',
+ 'gitlab-sidekiq' => 'sidekiq',
+ 'gitlab-workhorse' => 'workhorse',
+ 'redis' => 'redis',
+ 'postgres' => 'postgres',
+ 'gitaly' => 'gitaly',
+ 'prometheus' => 'prometheus',
+ 'node' => 'node-exporter'
+ }.freeze
+
+ def topology_usage_data
+ topology_data, duration = measure_duration do
+ alt_usage_data(fallback: {}) do
+ {
+ nodes: topology_node_data
+ }.compact
+ end
+ end
+ { topology: topology_data.merge(duration_s: duration) }
+ end
+
+ private
+
+ def topology_node_data
+ with_prometheus_client do |client|
+ # node-level data
+ by_instance_mem = topology_node_memory(client)
+ by_instance_cpus = topology_node_cpus(client)
+ # service-level data
+ by_instance_by_job_by_metric_memory = topology_all_service_memory(client)
+ by_instance_by_job_process_count = topology_all_service_process_count(client)
+
+ instances = Set.new(by_instance_mem.keys + by_instance_cpus.keys)
+ instances.map do |instance|
+ {
+ node_memory_total_bytes: by_instance_mem[instance],
+ node_cpus: by_instance_cpus[instance],
+ node_services:
+ topology_node_services(instance, by_instance_by_job_process_count, by_instance_by_job_by_metric_memory)
+ }.compact
+ end
+ end
+ end
+
+ def topology_node_memory(client)
+ aggregate_single(client, 'avg (node_memory_MemTotal_bytes) by (instance)')
+ end
+
+ def topology_node_cpus(client)
+ aggregate_single(client, 'count (node_cpu_seconds_total{mode="idle"}) by (instance)')
+ end
+
+ def topology_all_service_memory(client)
+ aggregate_many(
+ client,
+ 'avg ({__name__ =~ "(ruby_){0,1}process_(resident|unique|proportional)_memory_bytes", job != "gitlab_exporter_process"}) by (instance, job, __name__)'
+ )
+ end
+
+ def topology_all_service_process_count(client)
+ aggregate_many(client, 'count ({__name__ =~ "(ruby_){0,1}process_start_time_seconds", job != "gitlab_exporter_process"}) by (instance, job)')
+ end
+
+ def topology_node_services(instance, all_process_counts, all_process_memory)
+ # returns all node service data grouped by service name as the key
+ instance_service_data =
+ topology_instance_service_process_count(instance, all_process_counts)
+ .deep_merge(topology_instance_service_memory(instance, all_process_memory))
+
+ # map to list of hashes where service names become values instead, and remove
+ # unknown services, since they might not be ours
+ instance_service_data.each_with_object([]) do |entry, list|
+ service, service_metrics = entry
+ gitlab_service = JOB_TO_SERVICE_NAME[service.to_s]
+ next unless gitlab_service
+
+ list << { name: gitlab_service }.merge(service_metrics)
+ end
+ end
+
+ def topology_instance_service_process_count(instance, all_instance_data)
+ topology_data_for_instance(instance, all_instance_data).to_h do |metric, count|
+ [metric['job'], { process_count: count }]
+ end
+ end
+
+ def topology_instance_service_memory(instance, all_instance_data)
+ topology_data_for_instance(instance, all_instance_data).each_with_object({}) do |entry, hash|
+ metric, memory = entry
+ job = metric['job']
+ key =
+ case metric['__name__']
+ when match_process_memory_metric_for_type('resident') then :process_memory_rss
+ when match_process_memory_metric_for_type('unique') then :process_memory_uss
+ when match_process_memory_metric_for_type('proportional') then :process_memory_pss
+ end
+
+ hash[job] ||= {}
+ hash[job][key] ||= memory
+ end
+ end
+
+ def match_process_memory_metric_for_type(type)
+ /(ruby_){0,1}process_#{type}_memory_bytes/
+ end
+
+ def topology_data_for_instance(instance, all_instance_data)
+ all_instance_data.filter { |metric, _value| metric['instance'] == instance }
+ end
+
+ def drop_port(instance)
+ instance.gsub(/:.+$/, '')
+ end
+
+ # Will retain a single `instance` key that values are mapped to
+ def aggregate_single(client, query)
+ client.aggregate(query) { |metric| drop_port(metric['instance']) }
+ end
+
+ # Will retain a composite key that values are mapped to
+ def aggregate_many(client, query)
+ client.aggregate(query) do |metric|
+ metric['instance'] = drop_port(metric['instance'])
+ metric
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb
index 96898e5189c..44893645cc2 100644
--- a/lib/gitlab/usage_data_counters/base_counter.rb
+++ b/lib/gitlab/usage_data_counters/base_counter.rb
@@ -8,7 +8,7 @@ module Gitlab::UsageDataCounters
class << self
def redis_key(event)
- Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new, event: event) unless known_events.include?(event.to_s)
+ require_known_event(event)
"USAGE_#{prefix}_#{event}".upcase
end
@@ -31,6 +31,10 @@ module Gitlab::UsageDataCounters
private
+ def require_known_event(event)
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new, event: event) unless known_events.include?(event.to_s)
+ end
+
def counter_key(event)
"#{prefix}_#{event}".to_sym
end
diff --git a/lib/gitlab/usage_data_counters/designs_counter.rb b/lib/gitlab/usage_data_counters/designs_counter.rb
index 801fb8f3b3d..22188b555d2 100644
--- a/lib/gitlab/usage_data_counters/designs_counter.rb
+++ b/lib/gitlab/usage_data_counters/designs_counter.rb
@@ -4,7 +4,7 @@ module Gitlab::UsageDataCounters
class DesignsCounter
extend Gitlab::UsageDataCounters::RedisCounter
- KNOWN_EVENTS = %w[create update delete].map(&:freeze).freeze
+ KNOWN_EVENTS = %w[create update delete].freeze
UnknownEvent = Class.new(StandardError)
diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb
index b9e3a5c0104..61f98887adc 100644
--- a/lib/gitlab/usage_data_counters/search_counter.rb
+++ b/lib/gitlab/usage_data_counters/search_counter.rb
@@ -2,28 +2,20 @@
module Gitlab
module UsageDataCounters
- class SearchCounter
- extend RedisCounter
-
- NAVBAR_SEARCHES_COUNT_KEY = 'NAVBAR_SEARCHES_COUNT'
+ class SearchCounter < BaseCounter
+ KNOWN_EVENTS = %w[all_searches navbar_searches].freeze
class << self
- def increment_navbar_searches_count
- increment(NAVBAR_SEARCHES_COUNT_KEY)
- end
+ def redis_key(event)
+ require_known_event(event)
- def total_navbar_searches_count
- total_count(NAVBAR_SEARCHES_COUNT_KEY)
+ "#{event}_COUNT".upcase
end
- def totals
- {
- navbar_searches: total_navbar_searches_count
- }
- end
+ private
- def fallback_totals
- { navbar_searches: -1 }
+ def counter_key(event)
+ "#{event}".to_sym
end
end
end
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index d46601fa2e8..e80cc51dc3b 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -3,6 +3,7 @@
module Gitlab
module Utils
extend self
+ PathTraversalAttackError ||= Class.new(StandardError)
# Ensure that the relative path will not traverse outside the base directory
# We url decode the path to avoid passing invalid paths forward in url encoded format.
@@ -17,7 +18,7 @@ module Gitlab
path.end_with?("#{File::SEPARATOR}..") ||
(!allowed_absolute && Pathname.new(path).absolute?)
- raise StandardError.new("Invalid path")
+ raise PathTraversalAttackError.new('Invalid path')
end
path
@@ -159,5 +160,23 @@ module Gitlab
Addressable::URI.parse(uri_string)
rescue Addressable::URI::InvalidURIError, TypeError
end
+
+ # Invert a hash, collecting all keys that map to a given value in an array.
+ #
+ # Unlike `Hash#invert`, where the last encountered pair wins, and which has the
+ # type `Hash[k, v] => Hash[v, k]`, `multiple_key_invert` does not lose any
+ # information, has the type `Hash[k, v] => Hash[v, Array[k]]`, and the original
+ # hash can always be reconstructed.
+ #
+ # example:
+ #
+ # multiple_key_invert({ a: 1, b: 2, c: 1 })
+ # # => { 1 => [:a, :c], 2 => [:b] }
+ #
+ def multiple_key_invert(hash)
+ hash.flat_map { |k, v| Array.wrap(v).zip([k].cycle) }
+ .group_by(&:first)
+ .transform_values { |kvs| kvs.map(&:last) }
+ end
end
end
diff --git a/lib/gitlab/utils/log_limited_array.rb b/lib/gitlab/utils/log_limited_array.rb
index e0589c3df4c..fbbba568d14 100644
--- a/lib/gitlab/utils/log_limited_array.rb
+++ b/lib/gitlab/utils/log_limited_array.rb
@@ -9,14 +9,14 @@ module Gitlab
# to around 10 kilobytes. Once we hit the limit, add the sentinel
# value as the last item in the returned array.
def self.log_limited_array(array, sentinel: '...')
- return [] unless array.is_a?(Array)
+ return [] unless array.is_a?(Array) || array.is_a?(Enumerator::Lazy)
total_length = 0
limited_array = array.take_while do |arg|
total_length += JsonSizeEstimator.estimate(arg)
total_length <= MAXIMUM_ARRAY_LENGTH
- end
+ end.to_a
limited_array.push(sentinel) if total_length > MAXIMUM_ARRAY_LENGTH
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb
new file mode 100644
index 00000000000..afc4e000977
--- /dev/null
+++ b/lib/gitlab/utils/usage_data.rb
@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+# Usage data utilities
+#
+# * distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)
+# Does a distinct batch count, smartly reduces batch_size and handles errors
+#
+# Examples:
+# issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
+#
+# * count(relation, column = nil, batch: true, start: nil, finish: nil)
+# Does a non-distinct batch count, smartly reduces batch_size and handles errors
+#
+# Examples:
+# active_user_count: count(User.active)
+#
+# * alt_usage_data method
+# handles StandardError and fallbacks by default into -1 this way not all measures fail if we encounter one exception
+# there might be cases where we need to set a specific fallback in order to be aligned wih what version app is expecting as a type
+#
+# Examples:
+# alt_usage_data { Gitlab::VERSION }
+# alt_usage_data { Gitlab::CurrentSettings.uuid }
+# alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled }
+#
+# * redis_usage_data method
+# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
+# returns -1 when a block is sent or hash with all values -1 when a counter is sent
+# different behaviour due to 2 different implementations of redis counter
+#
+# Examples:
+# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
+# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
+
+module Gitlab
+ module Utils
+ module UsageData
+ extend self
+
+ FALLBACK = -1
+
+ def count(relation, column = nil, batch: true, start: nil, finish: nil)
+ if batch
+ Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish)
+ else
+ relation.count
+ end
+ rescue ActiveRecord::StatementInvalid
+ FALLBACK
+ end
+
+ def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)
+ if batch
+ Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish)
+ else
+ relation.distinct_count_by(column)
+ end
+ rescue ActiveRecord::StatementInvalid
+ FALLBACK
+ end
+
+ def alt_usage_data(value = nil, fallback: FALLBACK, &block)
+ if block_given?
+ yield
+ else
+ value
+ end
+ rescue
+ fallback
+ end
+
+ def redis_usage_data(counter = nil, &block)
+ if block_given?
+ redis_usage_counter(&block)
+ elsif counter.present?
+ redis_usage_data_totals(counter)
+ end
+ end
+
+ def with_prometheus_client
+ if Gitlab::Prometheus::Internal.prometheus_enabled?
+ prometheus_address = Gitlab::Prometheus::Internal.uri
+ yield Gitlab::PrometheusClient.new(prometheus_address, allow_local_requests: true)
+ end
+ end
+
+ def measure_duration
+ result = nil
+ duration = Benchmark.realtime do
+ result = yield
+ end
+ [result, duration]
+ end
+
+ private
+
+ def redis_usage_counter
+ yield
+ rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
+ FALLBACK
+ end
+
+ def redis_usage_data_totals(counter)
+ counter.totals
+ rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
+ counter.fallback_totals
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/web_ide/config.rb b/lib/gitlab/web_ide/config.rb
new file mode 100644
index 00000000000..3b1fa162b53
--- /dev/null
+++ b/lib/gitlab/web_ide/config.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module WebIde
+ #
+ # Base GitLab WebIde Configuration facade
+ #
+ class Config
+ ConfigError = Class.new(StandardError)
+
+ def initialize(config, opts = {})
+ @config = build_config(config, opts)
+
+ @global = Entry::Global.new(@config,
+ with_image_ports: true)
+ @global.compose!
+ rescue Gitlab::Config::Loader::FormatError => e
+ raise Config::ConfigError, e.message
+ end
+
+ def valid?
+ @global.valid?
+ end
+
+ def errors
+ @global.errors
+ end
+
+ def to_hash
+ @config
+ end
+
+ def terminal_value
+ @global.terminal_value
+ end
+
+ private
+
+ def build_config(config, opts = {})
+ Gitlab::Config::Loader::Yaml.new(config).load!
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/web_ide/config/entry/global.rb b/lib/gitlab/web_ide/config/entry/global.rb
new file mode 100644
index 00000000000..50c3f2d294f
--- /dev/null
+++ b/lib/gitlab/web_ide/config/entry/global.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module WebIde
+ class Config
+ module Entry
+ ##
+ # This class represents a global entry - root Entry for entire
+ # GitLab WebIde Configuration file.
+ #
+ class Global < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Configurable
+ include ::Gitlab::Config::Entry::Attributable
+
+ ALLOWED_KEYS = %i[terminal].freeze
+
+ validations do
+ validates :config, allowed_keys: ALLOWED_KEYS
+ end
+
+ entry :terminal, Entry::Terminal,
+ description: 'Configuration of the webide terminal.'
+
+ attributes :terminal
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/web_ide/config/entry/terminal.rb b/lib/gitlab/web_ide/config/entry/terminal.rb
new file mode 100644
index 00000000000..403e308d45b
--- /dev/null
+++ b/lib/gitlab/web_ide/config/entry/terminal.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module WebIde
+ class Config
+ module Entry
+ ##
+ # Entry that represents a concrete CI/CD job.
+ #
+ class Terminal < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Configurable
+ include ::Gitlab::Config::Entry::Attributable
+
+ # By default the build will finish in a few seconds, not giving the webide
+ # enough time to connect to the terminal. This default script provides
+ # those seconds blocking the build from finishing inmediately.
+ DEFAULT_SCRIPT = ['sleep 60'].freeze
+
+ ALLOWED_KEYS = %i[image services tags before_script script variables].freeze
+
+ validations do
+ validates :config, allowed_keys: ALLOWED_KEYS
+ validates :config, job_port_unique: { data: ->(record) { record.ports } }
+
+ with_options allow_nil: true do
+ validates :tags, array_of_strings: true
+ end
+ end
+
+ entry :before_script, ::Gitlab::Ci::Config::Entry::Script,
+ description: 'Global before script overridden in this job.'
+
+ entry :script, ::Gitlab::Ci::Config::Entry::Commands,
+ description: 'Commands that will be executed in this job.'
+
+ entry :image, ::Gitlab::Ci::Config::Entry::Image,
+ description: 'Image that will be used to execute this job.'
+
+ entry :services, ::Gitlab::Ci::Config::Entry::Services,
+ description: 'Services that will be used to execute this job.'
+
+ entry :variables, ::Gitlab::Ci::Config::Entry::Variables,
+ description: 'Environment variables available for this job.'
+
+ attributes :tags
+
+ def value
+ to_hash.compact
+ end
+
+ private
+
+ def to_hash
+ { tag_list: tags || [],
+ yaml_variables: yaml_variables,
+ options: {
+ image: image_value,
+ services: services_value,
+ before_script: before_script_value,
+ script: script_value || DEFAULT_SCRIPT
+ }.compact }
+ end
+
+ def yaml_variables
+ return unless variables_value
+
+ variables_value.map do |key, value|
+ { key: key.to_s, value: value, public: true }
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/milestone_array.rb b/lib/milestone_array.rb
deleted file mode 100644
index 461e73e9670..00000000000
--- a/lib/milestone_array.rb
+++ /dev/null
@@ -1,42 +0,0 @@
-# frozen_string_literal: true
-
-module MilestoneArray
- class << self
- def sort(array, sort_method)
- case sort_method
- when 'due_date_asc'
- sort_asc_nulls_last(array, 'due_date')
- when 'due_date_desc'
- sort_desc_nulls_last(array, 'due_date')
- when 'start_date_asc'
- sort_asc_nulls_last(array, 'start_date')
- when 'start_date_desc'
- sort_desc_nulls_last(array, 'start_date')
- when 'name_asc'
- sort_asc(array, 'title')
- when 'name_desc'
- sort_asc(array, 'title').reverse
- else
- array
- end
- end
-
- private
-
- def sort_asc_nulls_last(array, attribute)
- attribute = attribute.to_sym
-
- array.select(&attribute).sort_by(&attribute) + array.reject(&attribute)
- end
-
- def sort_desc_nulls_last(array, attribute)
- attribute = attribute.to_sym
-
- array.select(&attribute).sort_by(&attribute).reverse + array.reject(&attribute)
- end
-
- def sort_asc(array, attribute)
- array.sort_by(&attribute.to_sym)
- end
- end
-end
diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb
index fd26663fef0..5eab882039d 100644
--- a/lib/object_storage/direct_upload.rb
+++ b/lib/object_storage/direct_upload.rb
@@ -2,7 +2,7 @@
module ObjectStorage
#
- # The DirectUpload c;ass generates a set of presigned URLs
+ # The DirectUpload class generates a set of presigned URLs
# that can be used to upload data to object storage from untrusted component: Workhorse, Runner?
#
# For Google it assumes that the platform supports variable Content-Length.
@@ -46,7 +46,7 @@ module ObjectStorage
MultipartUpload: multipart_upload_hash,
CustomPutHeaders: true,
PutHeaders: upload_options
- }.compact
+ }.merge(workhorse_client_hash).compact
end
def multipart_upload_hash
@@ -60,6 +60,32 @@ module ObjectStorage
}
end
+ def workhorse_client_hash
+ return {} unless aws?
+
+ {
+ UseWorkhorseClient: use_workhorse_s3_client?,
+ RemoteTempObjectID: object_name,
+ ObjectStorage: {
+ Provider: 'AWS',
+ S3Config: {
+ Bucket: bucket_name,
+ Region: credentials[:region],
+ Endpoint: credentials[:endpoint],
+ PathStyle: credentials.fetch(:path_style, false),
+ UseIamProfile: credentials.fetch(:use_iam_profile, false)
+ }
+ }
+ }
+ end
+
+ def use_workhorse_s3_client?
+ Feature.enabled?(:use_workhorse_s3_client, default_enabled: true) &&
+ credentials.fetch(:use_iam_profile, false) &&
+ # The Golang AWS SDK does not support V2 signatures
+ credentials.fetch(:aws_signature_version, 4).to_i >= 4
+ end
+
def provider
credentials[:provider].to_s
end
diff --git a/lib/peek/views/bullet_detailed.rb b/lib/peek/views/bullet_detailed.rb
new file mode 100644
index 00000000000..8e6f72f565e
--- /dev/null
+++ b/lib/peek/views/bullet_detailed.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Peek
+ module Views
+ class BulletDetailed < DetailedView
+ WARNING_MESSAGE = "Unoptimized queries detected"
+
+ def key
+ 'bullet'
+ end
+
+ def results
+ return {} unless ::Bullet.enable?
+ return {} unless calls > 0
+
+ {
+ calls: calls,
+ details: details,
+ warnings: [WARNING_MESSAGE]
+ }
+ end
+
+ private
+
+ def details
+ notifications.map do |notification|
+ # there is no public method which returns pure backtace:
+ # https://github.com/flyerhzm/bullet/blob/9cda9c224a46786ecfa894480c4dd4d304db2adb/lib/bullet/notification/n_plus_one_query.rb
+ backtrace = notification.body_with_caller
+
+ {
+ notification: "#{notification.title}: #{notification.body}",
+ backtrace: backtrace
+ }
+ end
+ end
+
+ def calls
+ notifications.size
+ end
+
+ def notifications
+ ::Bullet.notification_collector&.collection || []
+ end
+ end
+ end
+end
diff --git a/lib/peek/views/elasticsearch.rb b/lib/peek/views/elasticsearch.rb
new file mode 100644
index 00000000000..626a6fb1316
--- /dev/null
+++ b/lib/peek/views/elasticsearch.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Peek
+ module Views
+ class Elasticsearch < DetailedView
+ DEFAULT_THRESHOLDS = {
+ calls: 5,
+ duration: 1000,
+ individual_call: 1000
+ }.freeze
+
+ THRESHOLDS = {
+ production: {
+ calls: 5,
+ duration: 1000,
+ individual_call: 1000
+ }
+ }.freeze
+
+ def key
+ 'es'
+ end
+
+ def self.thresholds
+ @thresholds ||= THRESHOLDS.fetch(Rails.env.to_sym, DEFAULT_THRESHOLDS)
+ end
+
+ private
+
+ def duration
+ ::Gitlab::Instrumentation::ElasticsearchTransport.query_time * 1000
+ end
+
+ def calls
+ ::Gitlab::Instrumentation::ElasticsearchTransport.get_request_count
+ end
+
+ def call_details
+ ::Gitlab::Instrumentation::ElasticsearchTransport.detail_store
+ end
+
+ def format_call_details(call)
+ super.merge(request: "#{call[:method]} #{call[:path]}")
+ end
+ end
+ end
+end
diff --git a/lib/peek/views/gitaly.rb b/lib/peek/views/gitaly.rb
index 7dc00b16cc0..566ca4496c4 100644
--- a/lib/peek/views/gitaly.rb
+++ b/lib/peek/views/gitaly.rb
@@ -40,12 +40,6 @@ module Peek
super.merge(request: pretty_request || {})
end
-
- def setup_subscribers
- subscribe 'start_processing.action_controller' do
- ::Gitlab::GitalyClient.query_time = 0
- end
- end
end
end
end
diff --git a/lib/peek/views/redis_detailed.rb b/lib/peek/views/redis_detailed.rb
index 79845044d75..44ec0ec0f68 100644
--- a/lib/peek/views/redis_detailed.rb
+++ b/lib/peek/views/redis_detailed.rb
@@ -9,10 +9,15 @@ module Peek
'redis'
end
+ def detail_store
+ ::Gitlab::Instrumentation::Redis.detail_store
+ end
+
private
def format_call_details(call)
- super.merge(cmd: format_command(call[:cmd]))
+ super.merge(cmd: format_command(call[:cmd]),
+ instance: call[:storage])
end
def format_command(cmd)
diff --git a/lib/quality/test_level.rb b/lib/quality/test_level.rb
index 97b86fa8c2e..334643fd0d3 100644
--- a/lib/quality/test_level.rb
+++ b/lib/quality/test_level.rb
@@ -44,6 +44,7 @@ module Quality
views
workers
elastic_integration
+ tooling
],
integration: %w[
controllers
diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab
index 8e9f220ec85..98cac0b0d1d 100755
--- a/lib/support/init.d/gitlab
+++ b/lib/support/init.d/gitlab
@@ -67,13 +67,13 @@ gitaly_log="$app_root/log/gitaly.log"
test -f /etc/default/gitlab && . /etc/default/gitlab
# Switch to the app_user if it is not they who are running the script.
-if [ `whoami` != "$app_user" ]; then
+if [ $(whoami) != "$app_user" ]; then
eval su - "$app_user" -c $(echo \")$shell_path -l -c \'$0 "$@"\'$(echo \"); exit;
fi
# Switch to the gitlab path, exit on failure.
if ! cd "$app_root" ; then
- echo "Failed to cd into $app_root, exiting!"; exit 1
+ echo "Failed to cd into $app_root, exiting!"; exit 1
fi
if [ -z "$SIDEKIQ_WORKERS" ]; then
@@ -341,7 +341,7 @@ start_gitlab() {
echo "Gitaly is already running with pid $gapid, not restarting"
else
$app_root/bin/daemon_with_pidfile $gitaly_pid_path \
- $gitaly_dir/gitaly $gitaly_dir/config.toml >> $gitaly_log 2>&1 &
+ $gitaly_dir/gitaly $gitaly_dir/config.toml >> $gitaly_log 2>&1 &
fi
fi
@@ -413,39 +413,39 @@ print_status() {
return
fi
if [ "$web_status" = "0" ]; then
- echo "The GitLab web server with pid $wpid is running."
+ echo "The GitLab web server with pid $wpid is running."
else
- printf "The GitLab web server is \033[31mnot running\033[0m.\n"
+ printf "The GitLab web server is \033[31mnot running\033[0m.\n"
fi
if [ "$sidekiq_status" = "0" ]; then
- echo "The GitLab Sidekiq job dispatcher with pid $spid is running."
+ echo "The GitLab Sidekiq job dispatcher with pid $spid is running."
else
- printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n"
+ printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n"
fi
if [ "$gitlab_workhorse_status" = "0" ]; then
- echo "The GitLab Workhorse with pid $hpid is running."
+ echo "The GitLab Workhorse with pid $hpid is running."
else
- printf "The GitLab Workhorse is \033[31mnot running\033[0m.\n"
+ printf "The GitLab Workhorse is \033[31mnot running\033[0m.\n"
fi
if [ "$mail_room_enabled" = true ]; then
if [ "$mail_room_status" = "0" ]; then
- echo "The GitLab MailRoom email processor with pid $mpid is running."
+ echo "The GitLab MailRoom email processor with pid $mpid is running."
else
- printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n"
+ printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n"
fi
fi
if [ "$gitlab_pages_enabled" = true ]; then
if [ "$gitlab_pages_status" = "0" ]; then
- echo "The GitLab Pages with pid $gppid is running."
+ echo "The GitLab Pages with pid $gppid is running."
else
- printf "The GitLab Pages is \033[31mnot running\033[0m.\n"
+ printf "The GitLab Pages is \033[31mnot running\033[0m.\n"
fi
fi
if [ "$gitaly_enabled" = true ]; then
if [ "$gitaly_status" = "0" ]; then
- echo "Gitaly with pid $gapid is running."
+ echo "Gitaly with pid $gapid is running."
else
- printf "Gitaly is \033[31mnot running\033[0m.\n"
+ printf "Gitaly is \033[31mnot running\033[0m.\n"
fi
fi
if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" = "0" ]; }; then
@@ -490,25 +490,25 @@ restart_gitlab(){
case "$1" in
start)
- start_gitlab
- ;;
+ start_gitlab
+ ;;
stop)
- stop_gitlab
- ;;
+ stop_gitlab
+ ;;
restart)
- restart_gitlab
- ;;
+ restart_gitlab
+ ;;
reload|force-reload)
- reload_gitlab
- ;;
+ reload_gitlab
+ ;;
status)
- print_status
- exit $gitlab_status
- ;;
+ print_status
+ exit $gitlab_status
+ ;;
*)
- echo "Usage: service gitlab {start|stop|restart|reload|status}"
- exit 1
- ;;
+ echo "Usage: service gitlab {start|stop|restart|reload|status}"
+ exit 1
+ ;;
esac
exit
diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake
index 3833689e07e..85393bba9a6 100644
--- a/lib/tasks/gemojione.rake
+++ b/lib/tasks/gemojione.rake
@@ -32,6 +32,7 @@ namespace :gemojione do
dir = Gemojione.images_path
resultant_emoji_map = {}
+ resultant_emoji_map_new = {}
Gitlab::Emoji.emojis.each do |name, emoji_hash|
# Ignore aliases
@@ -53,6 +54,16 @@ namespace :gemojione do
}
resultant_emoji_map[name] = entry
+
+ # Our new map is only characters to make the json substantially smaller
+ new_entry = {
+ c: category,
+ e: emoji_hash['moji'],
+ d: emoji_hash['description'],
+ u: Gitlab::Emoji.emoji_unicode_version(name)
+ }
+
+ resultant_emoji_map_new[name] = new_entry
end
end
@@ -60,6 +71,11 @@ namespace :gemojione do
File.open(out, 'w') do |handle|
handle.write(Gitlab::Json.pretty_generate(resultant_emoji_map))
end
+
+ out_new = File.join(Rails.root, 'public', '-', 'emojis', '1', 'emojis.json')
+ File.open(out_new, 'w') do |handle|
+ handle.write(Gitlab::Json.pretty_generate(resultant_emoji_map_new))
+ end
end
# This task will generate a standard and Retina sprite of all of the current
diff --git a/lib/tasks/gitlab/container_registry.rake b/lib/tasks/gitlab/container_registry.rake
new file mode 100644
index 00000000000..7687cb237cc
--- /dev/null
+++ b/lib/tasks/gitlab/container_registry.rake
@@ -0,0 +1,35 @@
+namespace :gitlab do
+ namespace :container_registry do
+ desc "GitLab | Container Registry | Configure"
+ task configure: :gitlab_environment do
+ configure
+ end
+
+ def configure
+ registry_config = Gitlab.config.registry
+
+ unless registry_config.enabled && registry_config.api_url.presence
+ puts "Registry is not enabled or registry api url is not present.".color(:yellow)
+ return
+ end
+
+ warn_user_is_not_gitlab
+
+ url = registry_config.api_url
+ # registry_info will query the /v2 route of the registry API. This route
+ # requires authentication, but not authorization (the response has no body,
+ # only headers that show the version of the registry). There is no
+ # associated user when running this rake, so we need to generate a valid
+ # JWT token with no access permissions to authenticate as a trusted client.
+ token = Auth::ContainerRegistryAuthenticationService.access_token([], [])
+ client = ContainerRegistry::Client.new(url, token: token)
+ info = client.registry_info
+
+ Gitlab::CurrentSettings.update!(
+ container_registry_vendor: info[:vendor] || '',
+ container_registry_version: info[:version] || '',
+ container_registry_features: info[:features] || []
+ )
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake
index 506027aa866..4917d496d07 100644
--- a/lib/tasks/gitlab/db.rake
+++ b/lib/tasks/gitlab/db.rake
@@ -92,9 +92,42 @@ namespace :gitlab do
Rake::Task[task_name].reenable
end
- # Inform Rake that gitlab:schema:clean_structure_sql should be run every time rake db:structure:dump is run
+ desc 'This dumps GitLab specific database details - it runs after db:structure:dump'
+ task :dump_custom_structure do |task_name|
+ Gitlab::Database::CustomStructure.new.dump
+
+ # Allow this task to be called multiple times, as happens when running db:migrate:redo
+ Rake::Task[task_name].reenable
+ end
+
+ desc 'This loads GitLab specific database details - runs after db:structure:dump'
+ task :load_custom_structure do
+ configuration = Rails.application.config_for(:database)
+
+ ENV['PGHOST'] = configuration['host'] if configuration['host']
+ ENV['PGPORT'] = configuration['port'].to_s if configuration['port']
+ ENV['PGPASSWORD'] = configuration['password'].to_s if configuration['password']
+ ENV['PGUSER'] = configuration['username'].to_s if configuration['username']
+
+ command = 'psql'
+ dump_filepath = Gitlab::Database::CustomStructure.custom_dump_filepath.to_path
+ args = ['-v', 'ON_ERROR_STOP=1', '-q', '-X', '-f', dump_filepath, configuration['database']]
+
+ unless Kernel.system(command, *args)
+ raise "failed to execute:\n#{command} #{args.join(' ')}\n\n" \
+ "Please ensure `#{command}` is installed in your PATH and has proper permissions.\n\n"
+ end
+ end
+
+ # Inform Rake that custom tasks should be run every time rake db:structure:dump is run
Rake::Task['db:structure:dump'].enhance do
Rake::Task['gitlab:db:clean_structure_sql'].invoke
+ Rake::Task['gitlab:db:dump_custom_structure'].invoke
+ end
+
+ # Inform Rake that custom tasks should be run every time rake db:structure:load is run
+ Rake::Task['db:structure:load'].enhance do
+ Rake::Task['gitlab:db:load_custom_structure'].invoke
end
end
end
diff --git a/lib/tasks/gitlab/doctor/secrets.rake b/lib/tasks/gitlab/doctor/secrets.rake
new file mode 100644
index 00000000000..3fdef9dfc80
--- /dev/null
+++ b/lib/tasks/gitlab/doctor/secrets.rake
@@ -0,0 +1,12 @@
+namespace :gitlab do
+ namespace :doctor do
+ desc "GitLab | Check if the database encrypted values can be decrypted using current secrets"
+ task secrets: :gitlab_environment do
+ logger = Logger.new(STDOUT)
+
+ logger.level = Gitlab::Utils.to_boolean(ENV['VERBOSE']) ? Logger::DEBUG : Logger::INFO
+
+ Gitlab::Doctor::Secrets.new(logger).run!
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/features.rake b/lib/tasks/gitlab/features.rake
index 9cf568c07fe..2309aa5d214 100644
--- a/lib/tasks/gitlab/features.rake
+++ b/lib/tasks/gitlab/features.rake
@@ -21,7 +21,7 @@ namespace :gitlab do
Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.each do |flag|
case status
when nil
- Feature.get(flag).remove
+ Feature.remove(flag)
when true
Feature.enable(flag)
when false
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index d6e62a5c550..edbaec85bd9 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -21,25 +21,12 @@ namespace :gitlab do
gitlab_url: gitlab_url,
http_settings: { self_signed_cert: false }.stringify_keys,
auth_file: File.join(user_home, ".ssh", "authorized_keys"),
- redis: {
- bin: `which redis-cli`.chomp,
- namespace: "resque:gitlab"
- }.stringify_keys,
log_level: "INFO",
audit_usernames: false
}.stringify_keys
- redis_url = URI.parse(ENV['REDIS_URL'] || "redis://localhost:6379")
-
- if redis_url.scheme == 'unix'
- config['redis']['socket'] = redis_url.path
- else
- config['redis']['host'] = redis_url.host
- config['redis']['port'] = redis_url.port
- end
-
# Generate config.yml based on existing gitlab settings
- File.open("config.yml", "w+") {|f| f.puts config.to_yaml}
+ File.open("config.yml", "w+") {|f| f.puts config.to_yaml }
[
%w(bin/install) + repository_storage_paths_args,