diff options
Diffstat (limited to 'config')
46 files changed, 620 insertions, 237 deletions
diff --git a/config/application.rb b/config/application.rb index 524827226e7..772b3b042c1 100644 --- a/config/application.rb +++ b/config/application.rb @@ -17,6 +17,7 @@ module Gitlab class Application < Rails::Application require_dependency Rails.root.join('lib/gitlab') require_dependency Rails.root.join('lib/gitlab/utils') + require_dependency Rails.root.join('lib/gitlab/action_cable/config') require_dependency Rails.root.join('lib/gitlab/redis/wrapper') require_dependency Rails.root.join('lib/gitlab/redis/cache') require_dependency Rails.root.join('lib/gitlab/redis/queues') @@ -157,6 +158,8 @@ module Gitlab # Webpack dev server configuration is handled in initializers/static_files.rb config.webpack.dev_server.enabled = false + config.action_mailer.delivery_job = "ActionMailer::MailDeliveryJob" + # Enable the asset pipeline config.assets.enabled = true diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml index ff5ccbb3c1b..3724fbb767b 100644 --- a/config/dependency_decisions.yml +++ b/config/dependency_decisions.yml @@ -454,12 +454,6 @@ :why: https://github.com/jaredhanson/utils-merge/blob/v1.0.0/LICENSE :versions: [] :when: 2017-09-16 05:18:26.193764000 Z -- - :approve - - svg4everybody - - :who: Tim Zallmann - :why: CC0 1.0 - https://github.com/jonathantneal/svg4everybody/blob/master/LICENSE.md - :versions: [] - :when: 2017-09-13 17:31:16.425819400 Z - - :license - "@gitlab/svgs" - MIT diff --git a/config/environments/development.rb b/config/environments/development.rb index 25d57467060..9d4fc6ba5e9 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -49,8 +49,6 @@ Rails.application.configure do # Do not log asset requests config.assets.quiet = true - config.allow_concurrency = Gitlab::Runtime.multi_threaded? - # BetterErrors live shell (REPL) on every stack frame BetterErrors::Middleware.allow_ip!("127.0.0.1/0") diff --git a/config/environments/production.rb b/config/environments/production.rb index c03421040a3..393a274606e 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -77,6 +77,4 @@ Rails.application.configure do config.action_mailer.raise_delivery_errors = true config.eager_load = true - - config.allow_concurrency = Gitlab::Runtime.multi_threaded? end diff --git a/config/environments/test.rb b/config/environments/test.rb index c130eb84baa..e08e2a34ff4 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -54,4 +54,8 @@ Rails.application.configure do config.logger = ActiveSupport::TaggedLogging.new(Logger.new(nil)) config.log_level = :fatal end + + # Mount the ActionCable Engine in-app so that we don't have to spawn another Puma + # process for feature specs + ENV['ACTION_CABLE_IN_APP'] = 'true' end diff --git a/config/feature_categories.yml b/config/feature_categories.yml index 7cbc90497a4..fd4cc8bf2a5 100644 --- a/config/feature_categories.yml +++ b/config/feature_categories.yml @@ -8,6 +8,7 @@ # --- - accessibility_testing +- advanced_deployments - alert_management - analysis - api @@ -28,7 +29,7 @@ - code_testing - collection - compliance_management -- container_behavior_analytics +- container_host_security - container_network_security - container_registry - container_scanning @@ -39,6 +40,7 @@ - dependency_proxy - dependency_scanning - design_management +- design_system - devops_reports - digital_experience_management - disaster_recovery @@ -60,7 +62,6 @@ - helm_chart_registry - importers - incident_management -- incremental_rollout - infrastructure_as_code - integrations - interactive_application_security_testing @@ -79,17 +80,18 @@ - malware_scanning - merge_trains - metrics +- navigation - omnibus_package - package_registry - pages - pki_management - planning_analytics - product_analytics +- projects - quality_management - release_evidence - release_orchestration - requirements_management -- responsible_disclosure - review_apps - roadmaps - runbooks diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index bd328d9919a..7ba256b39cd 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -205,6 +205,33 @@ production: &base # Whether to expunge (permanently remove) messages from the mailbox when they are deleted after delivery expunge_deleted: false + ## Consolidated object store config + ## This will only take effect if the object_store sections are not defined + ## within the types (e.g. artifacts, lfs, etc.). + # object_store: + # enabled: false + # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage + # connection: + # provider: AWS # Only AWS supported at the moment + # aws_access_key_id: AWS_ACCESS_KEY_ID + # aws_secret_access_key: AWS_SECRET_ACCESS_KEY + # region: us-east-1 + # aws_signature_version: 4 # For creation of signed URLs. Set to 2 if provider does not support v4. + # endpoint: 'https://s3.amazonaws.com' # default: nil - Useful for S3 compliant services such as DigitalOcean Spaces + # objects: + # artifacts: + # bucket: artifacts + # external_diffs: + # bucket: external-diffs + # lfs: + # bucket: lfs-objects + # uploads: + # bucket: uploads + # packages: + # bucket: packages + # dependency_proxy: + # bucket: dependency_proxy + ## Build Artifacts artifacts: enabled: true @@ -333,7 +360,7 @@ production: &base # storage_path: shared/terraform_state object_store: enabled: false - remote_directory: terraform_state # The bucket name + remote_directory: terraform # The bucket name connection: provider: AWS aws_access_key_id: AWS_ACCESS_KEY_ID @@ -474,11 +501,6 @@ production: &base geo_registry_sync_worker: cron: "*/1 * * * *" - # GitLab Geo migrated local files clean up worker - # NOTE: This will only take effect if Geo is enabled (secondary nodes only) - geo_migrated_local_files_clean_up_worker: - cron: "15 */6 * * *" - # Export pseudonymized data in CSV format for analysis pseudonymizer_worker: cron: "0 * * * *" @@ -487,12 +509,17 @@ production: &base # NOTE: This will only take effect if elasticsearch is enabled. elastic_index_bulk_cron_worker: cron: "*/1 * * * *" - + # Elasticsearch bulk updater for initial updates. # NOTE: This will only take effect if elasticsearch is enabled. elastic_index_initial_bulk_cron_worker: cron: "*/1 * * * *" + # Elasticsearch reindexing worker + # NOTE: This will only take effect if elasticsearch is enabled. + elastic_index_initial_bulk_cron_worker: + cron: "*/10 * * * *" + registry: # enabled: true # host: registry.example.com @@ -1078,11 +1105,6 @@ production: &base # host: localhost # port: 3808 - ## ActionCable settings - action_cable: - # Number of threads used to process ActionCable connection callbacks and channel actions - # worker_pool_size: 4 - ## Monitoring # Built in monitoring settings monitoring: @@ -1181,7 +1203,7 @@ test: # has been pushed). # when: always # The location where external diffs are stored (default: shared/external-diffs). - # storage_path: shared/external-diffs + storage_path: tmp/tests/external-diffs object_store: enabled: false remote_directory: external-diffs # The bucket name @@ -1231,7 +1253,7 @@ test: storage_path: tmp/tests/terraform_state object_store: enabled: false - remote_directory: terraform_state + remote_directory: terraform connection: provider: AWS # Only AWS supported at the moment aws_access_key_id: AWS_ACCESS_KEY_ID diff --git a/config/initializers/01_secret_token.rb b/config/initializers/01_secret_token.rb index 8b96727a2a1..5949f463457 100644 --- a/config/initializers/01_secret_token.rb +++ b/config/initializers/01_secret_token.rb @@ -1,13 +1,5 @@ -# WARNING: If you add a new secret to this file, make sure you also -# update Omnibus GitLab or updates will fail. Omnibus is responsible for -# writing the `secrets.yml` file. If Omnibus doesn't know about a -# secret, Rails will attempt to write to the file, but this will fail -# because Rails doesn't have write access. -# -# As an example: -# * https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/27581 -# * https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/3267 -# +# WARNING: Before you make a change to secrets.yml, read the development guide for GitLab secrets +# doc/development/application_secrets.md. # # This file needs to be loaded BEFORE any initializers that attempt to # prepend modules that require access to secrets (e.g. EE's 0_as_concern.rb). diff --git a/config/initializers/0_inject_feature_flags.rb b/config/initializers/0_inject_feature_flags.rb new file mode 100644 index 00000000000..45e6546e294 --- /dev/null +++ b/config/initializers/0_inject_feature_flags.rb @@ -0,0 +1,5 @@ +# This needs to be loaded after +# config/initializers/0_inject_enterprise_edition_module.rb + +Feature.register_feature_groups +Feature.register_definitions diff --git a/config/initializers/1_postgresql_only.rb b/config/initializers/1_postgresql_only.rb index be771bebf47..415fc6f2cae 100644 --- a/config/initializers/1_postgresql_only.rb +++ b/config/initializers/1_postgresql_only.rb @@ -2,3 +2,5 @@ raise "PostgreSQL is the only supported database from GitLab 12.1" unless Gitlab::Database.postgresql? + +Gitlab::Database.check_postgres_version_and_print_warning diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 0afd43634c3..b7432c4cbe6 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -254,7 +254,7 @@ Settings.artifacts['storage_path'] = Settings.absolute(Settings.artifacts.values # Settings.artifact['path'] is deprecated, use `storage_path` instead Settings.artifacts['path'] = Settings.artifacts['storage_path'] Settings.artifacts['max_size'] ||= 100 # in megabytes -Settings.artifacts['object_store'] = ObjectStoreSettings.parse(Settings.artifacts['object_store']) +Settings.artifacts['object_store'] = ObjectStoreSettings.legacy_parse(Settings.artifacts['object_store']) # # Registry @@ -325,7 +325,7 @@ Settings['external_diffs'] ||= Settingslogic.new({}) Settings.external_diffs['enabled'] = false if Settings.external_diffs['enabled'].nil? Settings.external_diffs['when'] = 'always' if Settings.external_diffs['when'].nil? Settings.external_diffs['storage_path'] = Settings.absolute(Settings.external_diffs['storage_path'] || File.join(Settings.shared['path'], 'external-diffs')) -Settings.external_diffs['object_store'] = ObjectStoreSettings.parse(Settings.external_diffs['object_store']) +Settings.external_diffs['object_store'] = ObjectStoreSettings.legacy_parse(Settings.external_diffs['object_store']) # # Git LFS @@ -333,7 +333,7 @@ Settings.external_diffs['object_store'] = ObjectStoreSettings.parse(Settings.ext Settings['lfs'] ||= Settingslogic.new({}) Settings.lfs['enabled'] = true if Settings.lfs['enabled'].nil? Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects")) -Settings.lfs['object_store'] = ObjectStoreSettings.parse(Settings.lfs['object_store']) +Settings.lfs['object_store'] = ObjectStoreSettings.legacy_parse(Settings.lfs['object_store']) # # Uploads @@ -341,18 +341,16 @@ Settings.lfs['object_store'] = ObjectStoreSettings.parse(Settings.lfs['object_st Settings['uploads'] ||= Settingslogic.new({}) Settings.uploads['storage_path'] = Settings.absolute(Settings.uploads['storage_path'] || 'public') Settings.uploads['base_dir'] = Settings.uploads['base_dir'] || 'uploads/-/system' -Settings.uploads['object_store'] = ObjectStoreSettings.parse(Settings.uploads['object_store']) +Settings.uploads['object_store'] = ObjectStoreSettings.legacy_parse(Settings.uploads['object_store']) Settings.uploads['object_store']['remote_directory'] ||= 'uploads' # # Packages # -Gitlab.ee do - Settings['packages'] ||= Settingslogic.new({}) - Settings.packages['enabled'] = true if Settings.packages['enabled'].nil? - Settings.packages['storage_path'] = Settings.absolute(Settings.packages['storage_path'] || File.join(Settings.shared['path'], "packages")) - Settings.packages['object_store'] = ObjectStoreSettings.parse(Settings.packages['object_store']) -end +Settings['packages'] ||= Settingslogic.new({}) +Settings.packages['enabled'] = true if Settings.packages['enabled'].nil? +Settings.packages['storage_path'] = Settings.absolute(Settings.packages['storage_path'] || File.join(Settings.shared['path'], "packages")) +Settings.packages['object_store'] = ObjectStoreSettings.legacy_parse(Settings.packages['object_store']) # # Dependency Proxy @@ -361,7 +359,7 @@ Gitlab.ee do Settings['dependency_proxy'] ||= Settingslogic.new({}) Settings.dependency_proxy['enabled'] = true if Settings.dependency_proxy['enabled'].nil? Settings.dependency_proxy['storage_path'] = Settings.absolute(Settings.dependency_proxy['storage_path'] || File.join(Settings.shared['path'], "dependency_proxy")) - Settings.dependency_proxy['object_store'] = ObjectStoreSettings.parse(Settings.dependency_proxy['object_store']) + Settings.dependency_proxy['object_store'] = ObjectStoreSettings.legacy_parse(Settings.dependency_proxy['object_store']) # For first iteration dependency proxy uses Rails server to download blobs. # To ensure acceptable performance we only allow feature to be used with @@ -376,7 +374,7 @@ end Settings['terraform_state'] ||= Settingslogic.new({}) Settings.terraform_state['enabled'] = true if Settings.terraform_state['enabled'].nil? Settings.terraform_state['storage_path'] = Settings.absolute(Settings.terraform_state['storage_path'] || File.join(Settings.shared['path'], "terraform_state")) -Settings.terraform_state['object_store'] = ObjectStoreSettings.parse(Settings.terraform_state['object_store']) +Settings.terraform_state['object_store'] = ObjectStoreSettings.legacy_parse(Settings.terraform_state['object_store']) # # Mattermost @@ -502,6 +500,12 @@ Settings.cron_jobs['users_create_statistics_worker']['job_class'] = 'Users::Crea Settings.cron_jobs['authorized_project_update_periodic_recalculate_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['authorized_project_update_periodic_recalculate_worker']['cron'] ||= '45 1 * * 6' Settings.cron_jobs['authorized_project_update_periodic_recalculate_worker']['job_class'] = 'AuthorizedProjectUpdate::PeriodicRecalculateWorker' +Settings.cron_jobs['update_container_registry_info_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['update_container_registry_info_worker']['cron'] ||= '0 0 * * *' +Settings.cron_jobs['update_container_registry_info_worker']['job_class'] = 'UpdateContainerRegistryInfoWorker' +Settings.cron_jobs['postgres_dynamic_partitions_creator'] ||= Settingslogic.new({}) +Settings.cron_jobs['postgres_dynamic_partitions_creator']['cron'] ||= '21 */6 * * *' +Settings.cron_jobs['postgres_dynamic_partitions_creator']['job_class'] ||= 'PartitionCreationWorker' Gitlab.ee do Settings.cron_jobs['adjourned_group_deletion_worker'] ||= Settingslogic.new({}) @@ -522,9 +526,6 @@ Gitlab.ee do Settings.cron_jobs['geo_metrics_update_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['geo_metrics_update_worker']['cron'] ||= '*/1 * * * *' Settings.cron_jobs['geo_metrics_update_worker']['job_class'] ||= 'Geo::MetricsUpdateWorker' - Settings.cron_jobs['geo_migrated_local_files_clean_up_worker'] ||= Settingslogic.new({}) - Settings.cron_jobs['geo_migrated_local_files_clean_up_worker']['cron'] ||= '15 */6 * * *' - Settings.cron_jobs['geo_migrated_local_files_clean_up_worker']['job_class'] ||= 'Geo::MigratedLocalFilesCleanUpWorker' Settings.cron_jobs['geo_prune_event_log_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['geo_prune_event_log_worker']['cron'] ||= '*/5 * * * *' Settings.cron_jobs['geo_prune_event_log_worker']['job_class'] ||= 'Geo::PruneEventLogWorker' @@ -567,12 +568,27 @@ Gitlab.ee do Settings.cron_jobs['elastic_index_initial_bulk_cron_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['elastic_index_initial_bulk_cron_worker']['cron'] ||= '*/1 * * * *' Settings.cron_jobs['elastic_index_initial_bulk_cron_worker']['job_class'] ||= 'ElasticIndexInitialBulkCronWorker' + Settings.cron_jobs['elastic_cluster_reindexing_cron_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['elastic_cluster_reindexing_cron_worker']['cron'] ||= '*/10 * * * *' + Settings.cron_jobs['elastic_cluster_reindexing_cron_worker']['job_class'] ||= 'ElasticClusterReindexingCronWorker' Settings.cron_jobs['sync_seat_link_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['sync_seat_link_worker']['cron'] ||= "#{rand(60)} 0 * * *" Settings.cron_jobs['sync_seat_link_worker']['job_class'] = 'SyncSeatLinkWorker' Settings.cron_jobs['web_application_firewall_metrics_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['web_application_firewall_metrics_worker']['cron'] ||= '0 1 * * 0' Settings.cron_jobs['web_application_firewall_metrics_worker']['job_class'] = 'IngressModsecurityCounterMetricsWorker' + Settings.cron_jobs['users_create_statistics_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['users_create_statistics_worker']['cron'] ||= '2 15 * * *' + Settings.cron_jobs['users_create_statistics_worker']['job_class'] = 'Users::CreateStatisticsWorker' + Settings.cron_jobs['network_policy_metrics_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['network_policy_metrics_worker']['cron'] ||= '0 3 * * 0' + Settings.cron_jobs['network_policy_metrics_worker']['job_class'] = 'NetworkPolicyMetricsWorker' + Settings.cron_jobs['iterations_update_status_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['iterations_update_status_worker']['cron'] ||= '5 0 * * *' + Settings.cron_jobs['iterations_update_status_worker']['job_class'] = 'IterationsUpdateStatusWorker' + Settings.cron_jobs['vulnerability_statistics_schedule_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['vulnerability_statistics_schedule_worker']['cron'] ||= '15 1 * * *' + Settings.cron_jobs['vulnerability_statistics_schedule_worker']['job_class'] = 'Vulnerabilities::Statistics::ScheduleWorker' end # @@ -598,6 +614,9 @@ Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.__send__(:build_gitlab_shell_ssh_path_prefix) Settings.gitlab_shell['git_timeout'] ||= 10800 +# Object storage +ObjectStoreSettings.new(Settings).parse! + # # Workhorse # @@ -735,12 +754,6 @@ Settings.webpack.dev_server['host'] ||= 'localhost' Settings.webpack.dev_server['port'] ||= 3808 # -# ActionCable settings -# -Settings['action_cable'] ||= Settingslogic.new({}) -Settings.action_cable['worker_pool_size'] ||= 4 - -# # Monitoring settings # Settings['monitoring'] ||= Settingslogic.new({}) diff --git a/config/initializers/action_cable.rb b/config/initializers/action_cable.rb index c549dd45ad9..5530e7d64a2 100644 --- a/config/initializers/action_cable.rb +++ b/config/initializers/action_cable.rb @@ -3,11 +3,11 @@ require 'action_cable/subscription_adapter/redis' Rails.application.configure do - # We only mount the ActionCable engine in tests where we run it in-app - # For other environments, we run it on a standalone Puma server - config.action_cable.mount_path = Rails.env.test? ? '/-/cable' : nil + # Mount the ActionCable engine when in-app mode is enabled + config.action_cable.mount_path = Gitlab::ActionCable::Config.in_app? ? '/-/cable' : nil + config.action_cable.url = Gitlab::Utils.append_path(Gitlab.config.gitlab.relative_url_root, '/-/cable') - config.action_cable.worker_pool_size = Gitlab.config.action_cable.worker_pool_size + config.action_cable.worker_pool_size = Gitlab::ActionCable::Config.worker_pool_size end # https://github.com/rails/rails/blob/bb5ac1623e8de08c1b7b62b1368758f0d3bb6379/actioncable/lib/action_cable/subscription_adapter/redis.rb#L18 diff --git a/config/initializers/action_dispatch_journey_formatter.rb b/config/initializers/action_dispatch_journey_formatter.rb index 93cf407c73c..108fb2e5012 100644 --- a/config/initializers/action_dispatch_journey_formatter.rb +++ b/config/initializers/action_dispatch_journey_formatter.rb @@ -9,8 +9,8 @@ module ActionDispatch module Path class Pattern def requirements_for_missing_keys_check - @requirements_for_missing_keys_check ||= requirements.each_with_object({}) do |(key, regex), hash| - hash[key] = /\A#{regex}\Z/ + @requirements_for_missing_keys_check ||= requirements.transform_values do |regex| + /\A#{regex}\Z/ end end end diff --git a/config/initializers/actionpack_generate_old_csrf_token.rb b/config/initializers/actionpack_generate_old_csrf_token.rb deleted file mode 100644 index 6367a1d4d59..00000000000 --- a/config/initializers/actionpack_generate_old_csrf_token.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RequestForgeryProtectionPatch - private - - # Patch to generate 6.0.3 tokens so that we do not have CSRF errors while - # rolling out 6.0.3.1. This enables GitLab to have a mix of 6.0.3 and - # 6.0.3.1 Rails servers - # - # 1. Deploy this patch with :global_csrf_token FF disabled. - # 2. Once all Rails servers are on 6.0.3.1, enable :global_csrf_token FF. - # 3. On GitLab 13.2, remove this patch - def masked_authenticity_token(session, form_options: {}) - action, method = form_options.values_at(:action, :method) - - raw_token = if per_form_csrf_tokens && action && method - action_path = normalize_action_path(action) - per_form_csrf_token(session, action_path, method) - else - if Feature.enabled?(:global_csrf_token) - global_csrf_token(session) - else - real_csrf_token(session) - end - end - - mask_token(raw_token) - end - end -end - -ActionController::Base.include Gitlab::RequestForgeryProtectionPatch diff --git a/config/initializers/active_record_schema_ignore_tables.rb b/config/initializers/active_record_schema_ignore_tables.rb index 661135f8ade..8ac565f239e 100644 --- a/config/initializers/active_record_schema_ignore_tables.rb +++ b/config/initializers/active_record_schema_ignore_tables.rb @@ -1,2 +1,5 @@ # Ignore table used temporarily in background migration ActiveRecord::SchemaDumper.ignore_tables = ["untracked_files_for_uploads"] + +# Ignore dynamically managed partitions in static application schema +ActiveRecord::SchemaDumper.ignore_tables += ["#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.*"] diff --git a/config/initializers/config_initializers_active_record_locking.rb b/config/initializers/config_initializers_active_record_locking.rb deleted file mode 100644 index 9f9908283c6..00000000000 --- a/config/initializers/config_initializers_active_record_locking.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -# ensure ActiveRecord's version has been required already -require 'active_record/locking/optimistic' - -# rubocop:disable Lint/RescueException -module ActiveRecord - module Locking - module Optimistic - private - - def _update_row(attribute_names, attempted_action = "update") - return super unless locking_enabled? - - begin - locking_column = self.class.locking_column - previous_lock_value = read_attribute_before_type_cast(locking_column) - attribute_names << locking_column - - self[locking_column] += 1 - - # Patched because when `lock_version` is read as `0`, it may actually be `NULL` in the DB. - possible_previous_lock_value = previous_lock_value.to_i == 0 ? [nil, 0] : previous_lock_value - - affected_rows = self.class.unscoped.where( - locking_column => possible_previous_lock_value, - self.class.primary_key => id_in_database - ).update_all( - attributes_with_values(attribute_names) - ) - - if affected_rows != 1 - raise ActiveRecord::StaleObjectError.new(self, attempted_action) - end - - affected_rows - - # If something went wrong, revert the locking_column value. - rescue Exception - self[locking_column] = previous_lock_value.to_i - raise - end - end - end - end -end diff --git a/config/initializers/doorkeeper_openid_connect.rb b/config/initializers/doorkeeper_openid_connect.rb index fd5a62c39c6..3523776c4f7 100644 --- a/config/initializers/doorkeeper_openid_connect.rb +++ b/config/initializers/doorkeeper_openid_connect.rb @@ -37,10 +37,10 @@ Doorkeeper::OpenidConnect.configure do # public email address (if present) # This allows existing solutions built for GitLab's old behavior to keep # working without modification. - o.claim(:email) do |user, scopes| + o.claim(:email, response: [:id_token, :user_info]) do |user, scopes| scopes.exists?(:email) ? user.email : user.public_email end - o.claim(:email_verified) do |user, scopes| + o.claim(:email_verified, response: [:id_token, :user_info]) do |user, scopes| if scopes.exists?(:email) user.primary_email_verified? elsif user.public_email? diff --git a/config/initializers/flipper.rb b/config/initializers/flipper.rb deleted file mode 100644 index 80cab7273e5..00000000000 --- a/config/initializers/flipper.rb +++ /dev/null @@ -1 +0,0 @@ -Feature.register_feature_groups diff --git a/config/initializers/grape_patch.rb b/config/initializers/grape_patch.rb new file mode 100644 index 00000000000..a9ac0840541 --- /dev/null +++ b/config/initializers/grape_patch.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true +# Monkey patch for Grape v1.4.0: https://github.com/ruby-grape/grape/pull/2088 + +require 'grape' + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Grape + module DSL + module InsideRoute + def stream(value = nil) + return if value.nil? && @stream.nil? + + header 'Content-Length', nil + header 'Transfer-Encoding', nil + header 'Cache-Control', 'no-cache' # Skips ETag generation (reading the response up front) + + if value.is_a?(String) + file_body = Grape::ServeStream::FileBody.new(value) + @stream = Grape::ServeStream::StreamResponse.new(file_body) + elsif value.respond_to?(:each) + @stream = Grape::ServeStream::StreamResponse.new(value) + elsif !value.is_a?(NilClass) + raise ArgumentError, 'Stream object must respond to :each.' + else + @stream + end + end + end + end +end +# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb index 01353ad4ec1..42c97e4aebd 100644 --- a/config/initializers/lograge.rb +++ b/config/initializers/lograge.rb @@ -15,6 +15,7 @@ unless Gitlab::Runtime.sidekiq? data[:db_duration_s] = Gitlab::Utils.ms_to_round_sec(data.delete(:db)) if data[:db] data[:view_duration_s] = Gitlab::Utils.ms_to_round_sec(data.delete(:view)) if data[:view] data[:duration_s] = Gitlab::Utils.ms_to_round_sec(data.delete(:duration)) if data[:duration] + data.merge!(::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_payload) data end diff --git a/config/initializers/multi_json.rb b/config/initializers/multi_json.rb new file mode 100644 index 00000000000..93a81d8320d --- /dev/null +++ b/config/initializers/multi_json.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +# Explicitly set the JSON adapter used by MultiJson +# Currently we want this to default to the existing json gem +MultiJson.use(:json_gem) diff --git a/config/initializers/oj.rb b/config/initializers/oj.rb new file mode 100644 index 00000000000..3fa26259fc6 --- /dev/null +++ b/config/initializers/oj.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +# Ensure Oj runs in json-gem compatibility mode by default +Oj.default_options = { mode: :rails } diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb new file mode 100644 index 00000000000..6c8a72d9bd5 --- /dev/null +++ b/config/initializers/postgres_partitioning.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Make sure we have loaded partitioned models here +# (even with eager loading disabled). + +begin + Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions +rescue ActiveRecord::ActiveRecordError, PG::Error + # ignore - happens when Rake tasks yet have to create a database, e.g. for testing +end diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 51b49bec864..b0778633199 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -68,6 +68,15 @@ class Rack::Attack end end + # Product analytics feature is in experimental stage. + # At this point we want to limit amount of events registered + # per application (aid stands for application id). + throttle('throttle_product_analytics_collector', limit: 100, period: 60) do |req| + if req.product_analytics_collector_request? + req.params['aid'] + end + end + throttle('throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req| if req.web_request? && Gitlab::Throttle.settings.throttle_authenticated_web_enabled @@ -128,6 +137,10 @@ class Rack::Attack path =~ %r{^/-/(health|liveness|readiness)} end + def product_analytics_collector_request? + path.start_with?('/-/collector/i') + end + def should_be_skipped? api_internal_request? || health_check_request? end diff --git a/config/initializers/rack_timeout.rb b/config/initializers/rack_timeout.rb index 5d5a5fcf980..e217398ee7d 100644 --- a/config/initializers/rack_timeout.rb +++ b/config/initializers/rack_timeout.rb @@ -10,8 +10,6 @@ # logged and we should fix the potential timeout issue in the code itself. if Gitlab::Runtime.puma? && !Rails.env.test? - require 'rack/timeout/base' - Rack::Timeout::Logger.level = Logger::ERROR Gitlab::Application.configure do |config| diff --git a/config/initializers/stackprof.rb b/config/initializers/stackprof.rb new file mode 100644 index 00000000000..5497ff9a459 --- /dev/null +++ b/config/initializers/stackprof.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +# trigger stackprof by sending a SIGUSR2 signal +# +# default settings: +# * collect raw samples +# * sample at 100hz (every 10k microseconds) +# * timeout profile after 30 seconds +# * write to $TMPDIR/stackprof.$PID.$RAND.profile + +if Gitlab::Utils.to_boolean(ENV['STACKPROF_ENABLED'].to_s) + Gitlab::Cluster::LifecycleEvents.on_worker_start do + require 'stackprof' + require 'tmpdir' + + Gitlab::AppJsonLogger.info "stackprof: listening on SIGUSR2 signal" + + # create a pipe in order to propagate signal out of the signal handler + # see also: https://cr.yp.to/docs/selfpipe.html + read, write = IO.pipe + + # create a separate thread that polls for signals on the pipe. + # + # this way we do not execute in signal handler context, which + # lifts restrictions and also serializes the calls in a thread-safe + # manner. + # + # it's very similar to a goroutine and channel design. + # + # another nice benefit of this method is that we can timeout the + # IO.select call, allowing the profile to automatically stop after + # a given interval (by default 30 seconds), avoiding unbounded memory + # growth from a profile that was started and never stopped. + t = Thread.new do + timeout_s = ENV['STACKPROF_TIMEOUT_S']&.to_i || 30 + current_timeout_s = nil + loop do + got_value = IO.select([read], nil, nil, current_timeout_s) + read.getbyte if got_value + + if StackProf.running? + stackprof_file_prefix = ENV['STACKPROF_FILE_PREFIX'] || Dir.tmpdir + stackprof_out_file = "#{stackprof_file_prefix}/stackprof.#{Process.pid}.#{SecureRandom.hex(6)}.profile" + + Gitlab::AppJsonLogger.info( + event: "stackprof", + message: "stopping profile", + output_filename: stackprof_out_file, + pid: Process.pid, + timeout_s: timeout_s, + timed_out: got_value.nil? + ) + + StackProf.stop + StackProf.results(stackprof_out_file) + current_timeout_s = nil + else + Gitlab::AppJsonLogger.info( + event: "stackprof", + message: "starting profile", + pid: Process.pid + ) + + StackProf.start( + mode: :cpu, + raw: Gitlab::Utils.to_boolean(ENV['STACKPROF_RAW'] || 'true'), + interval: ENV['STACKPROF_INTERVAL_US']&.to_i || 10_000 + ) + current_timeout_s = timeout_s + end + end + end + t.abort_on_exception = true + + # in the case of puma, this will override the existing SIGUSR2 signal handler + # that can be used to trigger a restart. + # + # puma cluster has two types of restarts: + # * SIGUSR1: phased restart + # * SIGUSR2: restart + # + # phased restart is not supported in our configuration, because we use + # preload_app. this means we will always perform a normal restart. + # additionally, phased restart is not supported when sending a SIGUSR2 + # directly to a puma worker (as opposed to the master process). + # + # the result is that the behaviour of SIGUSR1 and SIGUSR2 is identical in + # our configuration, and we can always use a SIGUSR1 to perform a restart. + # + # thus, it is acceptable for us to re-appropriate the SIGUSR2 signal, and + # override the puma behaviour. + # + # see also: + # * https://github.com/puma/puma/blob/master/docs/signals.md#puma-signals + # * https://github.com/phusion/unicorn/blob/master/SIGNALS + # * https://github.com/mperham/sidekiq/wiki/Signals + Signal.trap('SIGUSR2') do + write.write('.') + end + end +end diff --git a/config/initializers_before_autoloader/000_inflections.rb b/config/initializers_before_autoloader/000_inflections.rb index 1fabce9a57e..5c1a3e87fba 100644 --- a/config/initializers_before_autoloader/000_inflections.rb +++ b/config/initializers_before_autoloader/000_inflections.rb @@ -11,6 +11,7 @@ # ActiveSupport::Inflector.inflections do |inflect| inflect.uncountable %w( + custom_emoji award_emoji container_repository_registry design_registry diff --git a/config/karma.config.js b/config/karma.config.js index 97794225a3f..31fdd5bffd1 100644 --- a/config/karma.config.js +++ b/config/karma.config.js @@ -181,7 +181,7 @@ module.exports = function(config) { if (process.env.BABEL_ENV === 'coverage' || process.env.NODE_ENV === 'coverage') { karmaConfig.reporters.push('coverage-istanbul'); karmaConfig.coverageIstanbulReporter = { - reports: ['html', 'text-summary'], + reports: ['html', 'text-summary', 'cobertura'], dir: 'coverage-javascript/', subdir: '.', fixWebpackSourcePaths: true, diff --git a/config/locales/devise.en.yml b/config/locales/devise.en.yml index bd4c3ebc69e..ee75ffbb8e9 100644 --- a/config/locales/devise.en.yml +++ b/config/locales/devise.en.yml @@ -15,7 +15,7 @@ en: not_found_in_database: "Invalid %{authentication_keys} or password." timeout: "Your session expired. Please sign in again to continue." unauthenticated: "You need to sign in or sign up before continuing." - unconfirmed: "You have to confirm your email address before continuing." + unconfirmed: "You have to confirm your email address before continuing. Please check your email for the link we sent you, or click 'Resend confirmation email'." mailer: confirmation_instructions: subject: "Confirmation instructions" diff --git a/config/locales/en.yml b/config/locales/en.yml index ed0552ab452..7ff4e3bf7da 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -22,6 +22,8 @@ en: grafana_enabled: "Grafana integration enabled" user/user_detail: job_title: 'Job title' + user/user_detail: + bio: 'Bio' views: pagination: previous: "Prev" diff --git a/config/object_store_settings.rb b/config/object_store_settings.rb index d85ff394dcc..d8e1939a346 100644 --- a/config/object_store_settings.rb +++ b/config/object_store_settings.rb @@ -1,6 +1,12 @@ # Set default values for object_store settings class ObjectStoreSettings - def self.parse(object_store) + SUPPORTED_TYPES = %w(artifacts external_diffs lfs uploads packages dependency_proxy terraform_state).freeze + ALLOWED_OBJECT_STORE_OVERRIDES = %w(bucket enabled proxy_download).freeze + + attr_accessor :settings + + # Legacy parser + def self.legacy_parse(object_store) object_store ||= Settingslogic.new({}) object_store['enabled'] = false if object_store['enabled'].nil? object_store['remote_directory'] ||= nil @@ -12,4 +18,128 @@ class ObjectStoreSettings object_store['connection']&.deep_stringify_keys! object_store end + + def initialize(settings) + @settings = settings + end + + # This method converts the common object storage settings to + # the legacy, internal representation. + # + # For example, with the folowing YAML: + # + # object_store: + # enabled: true + # connection: + # provider: AWS + # aws_access_key_id: minio + # aws_secret_access_key: gdk-minio + # region: gdk + # endpoint: 'http://127.0.0.1:9000' + # path_style: true + # proxy_download: true + # objects: + # artifacts: + # bucket: artifacts + # proxy_download: false + # lfs: + # bucket: lfs-objects + # + # This method then will essentially call: + # + # Settings.artifacts['object_store'] = { + # "enabled" => true, + # "connection"=> { + # "provider" => "AWS", + # "aws_access_key_id" => "minio", + # "aws_secret_access_key" => "gdk-minio", + # "region" => "gdk", + # "endpoint" => "http://127.0.0.1:9000", + # "path_style" => true + # }, + # "direct_upload" => true, + # "background_upload" => false, + # "proxy_download" => false, + # "remote_directory" => "artifacts" + # } + # + # Settings.lfs['object_store'] = { + # "enabled" => true, + # "connection" => { + # "provider" => "AWS", + # "aws_access_key_id" => "minio", + # "aws_secret_access_key" => "gdk-minio", + # "region" => "gdk", + # "endpoint" => "http://127.0.0.1:9000", + # "path_style" => true + # }, + # "direct_upload" => true, + # "background_upload" => false, + # "proxy_download" => true, + # "remote_directory" => "lfs-objects" + # } + # + # Note that with the common config: + # 1. Only one object store credentials can now be used. This is + # necessary to limit configuration overhead when an object storage + # client (e.g. AWS S3) is used inside GitLab Workhorse. + # 2. However, a bucket has to be specified for each object + # type. Reusing buckets is not really supported, but we don't + # enforce that yet. + # 3. direct_upload and background_upload cannot be configured anymore. + def parse! + return unless use_consolidated_settings? + + main_config = settings['object_store'] + common_config = main_config.slice('enabled', 'connection', 'proxy_download') + # Convert connection settings to use string keys, to make Fog happy + common_config['connection']&.deep_stringify_keys! + # These are no longer configurable if common config is used + common_config['direct_upload'] = true + common_config['background_upload'] = false + + SUPPORTED_TYPES.each do |store_type| + overrides = main_config.dig('objects', store_type) || {} + target_config = common_config.merge(overrides.slice(*ALLOWED_OBJECT_STORE_OVERRIDES)) + section = settings.try(store_type) + + next unless section + + raise "Object storage for #{store_type} must have a bucket specified" if section['enabled'] && target_config['bucket'].blank? + + # Map bucket (external name) -> remote_directory (internal representation) + target_config['remote_directory'] = target_config.delete('bucket') + target_config['consolidated_settings'] = true + section['object_store'] = target_config + end + end + + private + + # We only can use the common object storage settings if: + # 1. The common settings are defined + # 2. The legacy settings are not defined + def use_consolidated_settings? + return false unless settings.dig('object_store', 'enabled') + return false unless settings.dig('object_store', 'connection').present? + + SUPPORTED_TYPES.each do |store| + # to_h is needed because something strange happens to + # Settingslogic#dig when stub_storage_settings is run in tests: + # + # (byebug) section.dig + # *** ArgumentError Exception: wrong number of arguments (given 0, expected 1+) + # (byebug) section.dig('object_store') + # *** ArgumentError Exception: wrong number of arguments (given 1, expected 0) + section = settings.try(store)&.to_h + + next unless section + + return false if section.dig('object_store', 'enabled') + # Omnibus defaults to an empty hash + return false if section.dig('object_store', 'connection').present? + end + + true + end end diff --git a/config/plugins/monaco_webpack.js b/config/plugins/monaco_webpack.js new file mode 100644 index 00000000000..7d283782453 --- /dev/null +++ b/config/plugins/monaco_webpack.js @@ -0,0 +1,17 @@ +const { languagesArr } = require('monaco-editor-webpack-plugin/out/languages'); + +// monaco-yaml library doesn't play so well with monaco-editor-webpack-plugin +// so the only way to include its workers is by patching the list of languages +// in monaco-editor-webpack-plugin and adding support for yaml workers. This is +// a known issue in the library and this workaround was suggested here: +// https://github.com/pengx17/monaco-yaml/issues/20 + +const yamlLang = languagesArr.find(t => t.label === 'yaml'); + +yamlLang.entry = [yamlLang.entry, '../../monaco-yaml/esm/monaco.contribution']; +yamlLang.worker = { + id: 'vs/language/yaml/yamlWorker', + entry: '../../monaco-yaml/esm/yaml.worker.js', +}; + +module.exports = require('monaco-editor-webpack-plugin'); diff --git a/config/prometheus/cluster_metrics.yml b/config/prometheus/cluster_metrics.yml index f2a41e4c337..1e396f4bbbd 100644 --- a/config/prometheus/cluster_metrics.yml +++ b/config/prometheus/cluster_metrics.yml @@ -1,63 +1,40 @@ +dashboard: 'Cluster health' +priority: 1 +panel_groups: - group: Cluster Health - priority: 1 - metrics: + priority: 10 + panels: - title: "CPU Usage" + type: "area-chart" y_label: "CPU (cores)" - required_metrics: ['container_cpu_usage_seconds_total'] weight: 1 - queries: - - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)' - label: Usage (cores) - unit: "cores" - appearance: - line: - width: 2 - area: - opacity: 0 - - query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})' - label: Requested (cores) - unit: "cores" - appearance: - line: - width: 2 - area: - opacity: 0 - - query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})' - label: Capacity (cores) - unit: "cores" - appearance: - line: - type: 'dashed' - width: 2 - area: - opacity: 0 - - title: "Memory usage" + metrics: + - id: cluster_health_cpu_usage + query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)' + unit: cores + label: Usage (cores) + - id: cluster_health_cpu_requested + query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})' + unit: cores + label: Requested (cores) + - id: cluster_health_cpu_capacity + query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})' + unit: cores + label: Capacity (cores) + - title: "Memory Usage" + type: "area-chart" y_label: "Memory (GiB)" - required_metrics: ['container_memory_usage_bytes'] weight: 1 - queries: - - query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30' - label: Usage (GiB) - unit: "GiB" - appearance: - line: - width: 2 - area: - opacity: 0 - - query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30' - label: Requested (GiB) - unit: "GiB" - appearance: - line: - width: 2 - area: - opacity: 0 - - query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30' - label: Capacity (GiB) - unit: "GiB" - appearance: - line: - type: 'dashed' - width: 2 - area: - opacity: 0 + metrics: + - id: cluster_health_memory_usage + query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30' + unit: GiB + label: Usage (GiB) + - id: cluster_health_memory_requested + query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30' + unit: GiB + label: Requested (GiB) + - id: cluster_health_memory_capacity + query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30' + unit: GiB + label: Capacity (GiB) diff --git a/config/prometheus/common_metrics.yml b/config/prometheus/common_metrics.yml index f0491df3db9..d9aaff12a4d 100644 --- a/config/prometheus/common_metrics.yml +++ b/config/prometheus/common_metrics.yml @@ -10,7 +10,9 @@ panel_groups: weight: 4 metrics: - id: system_metrics_kubernetes_container_memory_total - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) /1024/1024/1024' + # Remove the second metric (after OR) when we drop support for K8s 1.13 + # https://gitlab.com/gitlab-org/gitlab/-/issues/229279 + query_range: 'avg(sum(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) /1024/1024/1024 OR avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) /1024/1024/1024' label: Total (GB) unit: GB - title: "Core Usage (Total)" @@ -19,7 +21,9 @@ panel_groups: weight: 3 metrics: - id: system_metrics_kubernetes_container_cores_total - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job)' + # Remove the second metric (after OR) when we drop support for K8s 1.13 + # https://gitlab.com/gitlab-org/gitlab/-/issues/229279 + query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container!="POD",pod=~"^{{ci_environment_slug}}-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) OR avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job)' label: Total (cores) unit: "cores" - title: "Memory Usage (Pod average)" @@ -28,7 +32,9 @@ panel_groups: weight: 2 metrics: - id: system_metrics_kubernetes_container_memory_average - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024' + # Remove the second metric (after OR) when we drop support for K8s 1.13 + # https://gitlab.com/gitlab-org/gitlab/-/issues/229279 + query_range: 'avg(sum(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024 OR avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024' label: Pod average (MB) unit: MB - title: "Canary: Memory Usage (Pod Average)" @@ -37,7 +43,9 @@ panel_groups: weight: 2 metrics: - id: system_metrics_kubernetes_container_memory_average_canary - query_range: 'avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024' + # Remove the second metric (after OR) when we drop support for K8s 1.13 + # https://gitlab.com/gitlab-org/gitlab/-/issues/229279 + query_range: 'avg(sum(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container!="POD",pod=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024 OR avg(sum(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}) by (job)) without (job) / count(avg(container_memory_usage_bytes{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}) without (job)) /1024/1024' label: Pod average (MB) unit: MB track: canary @@ -47,7 +55,9 @@ panel_groups: weight: 1 metrics: - id: system_metrics_kubernetes_container_core_usage - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod_name))' + # Remove the second metric (after OR) when we drop support for K8s 1.13 + # https://gitlab.com/gitlab-org/gitlab/-/issues/229279 + query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container!="POD",pod=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod)) OR avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-([^c].*|c([^a]|a([^n]|n([^a]|a([^r]|r[^y])))).*|)-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod_name))' label: Pod average (cores) unit: "cores" - title: "Canary: Core Usage (Pod Average)" @@ -56,7 +66,9 @@ panel_groups: weight: 1 metrics: - id: system_metrics_kubernetes_container_core_usage_canary - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod_name))' + # Remove the second metric (after OR) when we drop support for K8s 1.13 + # https://gitlab.com/gitlab-org/gitlab/-/issues/229279 + query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{container!="POD",pod=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container!="POD",pod=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod)) OR avg(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}[15m])) by (job)) without (job) / count(sum(rate(container_cpu_usage_seconds_total{container_name!="POD",pod_name=~"^{{ci_environment_slug}}-canary-(.*)",namespace="{{kube_namespace}}"}[15m])) by (pod_name))' label: Pod average (cores) unit: "cores" track: canary diff --git a/config/prometheus/queries_cluster_metrics.yml b/config/prometheus/queries_cluster_metrics.yml new file mode 100644 index 00000000000..bec3ba22d83 --- /dev/null +++ b/config/prometheus/queries_cluster_metrics.yml @@ -0,0 +1,65 @@ +# most likely this file can be removed, but until we are sure and have capacity to tackle that I've +# only moved it and added https://gitlab.com/gitlab-org/gitlab/-/issues/225869 to track work need to clean up codebase. +- group: Cluster Health + priority: 1 + metrics: + - title: "CPU Usage" + y_label: "CPU (cores)" + required_metrics: ['container_cpu_usage_seconds_total'] + weight: 1 + queries: + - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)' + label: Usage (cores) + unit: "cores" + appearance: + line: + width: 2 + area: + opacity: 0 + - query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})' + label: Requested (cores) + unit: "cores" + appearance: + line: + width: 2 + area: + opacity: 0 + - query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})' + label: Capacity (cores) + unit: "cores" + appearance: + line: + type: 'dashed' + width: 2 + area: + opacity: 0 + - title: "Memory usage" + y_label: "Memory (GiB)" + required_metrics: ['container_memory_usage_bytes'] + weight: 1 + queries: + - query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30' + label: Usage (GiB) + unit: "GiB" + appearance: + line: + width: 2 + area: + opacity: 0 + - query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30' + label: Requested (GiB) + unit: "GiB" + appearance: + line: + width: 2 + area: + opacity: 0 + - query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30' + label: Capacity (GiB) + unit: "GiB" + appearance: + line: + type: 'dashed' + width: 2 + area: + opacity: 0 diff --git a/config/routes.rb b/config/routes.rb index 598a52cddb3..dd84bc859bb 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,6 @@ require 'sidekiq/web' require 'sidekiq/cron/web' +require 'product_analytics/collector_app' Rails.application.routes.draw do concern :access_requestable do @@ -58,6 +59,7 @@ Rails.application.routes.draw do # Search get 'search' => 'search#show' + get 'search/autocomplete' => 'search#autocomplete', as: :search_autocomplete get 'search/count' => 'search#count', as: :search_count # JSON Web Token @@ -75,6 +77,7 @@ Rails.application.routes.draw do get '/autocomplete/projects' => 'autocomplete#projects' get '/autocomplete/award_emojis' => 'autocomplete#award_emojis' get '/autocomplete/merge_request_target_branches' => 'autocomplete#merge_request_target_branches' + get '/autocomplete/deploy_keys_with_owners' => 'autocomplete#deploy_keys_with_owners' Gitlab.ee do get '/autocomplete/project_groups' => 'autocomplete#project_groups' @@ -174,6 +177,9 @@ Rails.application.routes.draw do # Used by third parties to verify CI_JOB_JWT, placeholder route # in case we decide to move away from doorkeeper-openid_connect get 'jwks' => 'doorkeeper/openid_connect/discovery#keys' + + # Product analytics collector + match '/collector/i', to: ProductAnalytics::CollectorApp.new, via: :all end # End of the /-/ scope. @@ -189,8 +195,6 @@ Rails.application.routes.draw do member do Gitlab.ee do get :metrics, format: :json - get :metrics_dashboard - get :'/prometheus/api/v1/*proxy_path', to: 'clusters#prometheus_proxy', as: :prometheus_api get :environments, format: :json end @@ -200,6 +204,8 @@ Rails.application.routes.draw do delete '/:application', to: 'clusters/applications#destroy', as: :uninstall_applications end + get :metrics_dashboard + get :'/prometheus/api/v1/*proxy_path', to: 'clusters#prometheus_proxy', as: :prometheus_api get :cluster_status, format: :json delete :clear_cache end @@ -242,6 +248,8 @@ Rails.application.routes.draw do post :preview_markdown end + draw :group + resources :projects, only: [:index, :new, :create] get '/projects/:id' => 'projects#resolve' @@ -258,10 +266,17 @@ Rails.application.routes.draw do draw :admin draw :profile draw :dashboard - draw :group draw :user draw :project + # Serve snippet routes under /-/snippets. + # To ensure an old unscoped routing is used for the UI we need to + # add prefix 'as' to the scope routing and place it below original routing. + # Issue https://gitlab.com/gitlab-org/gitlab/-/issues/210024 + scope '-', as: :scoped do + draw :snippets + end + root to: "root#index" get '*unmatched_route', to: 'application#route_not_found' diff --git a/config/routes/import.rb b/config/routes/import.rb index cd8278f6fd0..1dc27d489f0 100644 --- a/config/routes/import.rb +++ b/config/routes/import.rb @@ -24,14 +24,12 @@ namespace :import do resource :gitlab, only: [:create], controller: :gitlab do get :status get :callback - get :jobs get :realtime_changes end resource :bitbucket, only: [:create], controller: :bitbucket do get :status get :callback - get :jobs get :realtime_changes end @@ -39,7 +37,6 @@ namespace :import do post :configure get :status get :callback - get :jobs get :realtime_changes end @@ -55,7 +52,6 @@ namespace :import do resource :fogbugz, only: [:create, :new], controller: :fogbugz do get :status post :callback - get :jobs get :realtime_changes get :new_user_map, path: :user_map diff --git a/config/routes/issues.rb b/config/routes/issues.rb index 04a935c1016..eae1cacfcf7 100644 --- a/config/routes/issues.rb +++ b/config/routes/issues.rb @@ -17,6 +17,7 @@ resources :issues, concerns: :awardable, constraints: { id: /\d+/ } do end collection do + get :service_desk post :bulk_update post :import_csv post :export_csv diff --git a/config/routes/pipelines.rb b/config/routes/pipelines.rb index cc3c3400526..c7f9bf8791c 100644 --- a/config/routes/pipelines.rb +++ b/config/routes/pipelines.rb @@ -22,9 +22,13 @@ resources :pipelines, only: [:index, :new, :create, :show, :destroy] do get :test_reports_count end - member do - resources :stages, only: [], param: :name do - post :play_manual + resources :stages, only: [], param: :name, controller: 'pipelines/stages' do + post :play_manual + end + + resources :tests, only: [:show], param: :suite_name, controller: 'pipelines/tests' do + collection do + get :summary end end end diff --git a/config/routes/project.rb b/config/routes/project.rb index 78dcc189d5b..3bd72dbf87c 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -25,6 +25,8 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do # Use this scope for all new project routes. scope '-' do get 'archive/*id', constraints: { format: Gitlab::PathRegex.archive_formats_regex, id: /.+?/ }, to: 'repositories#archive', as: 'archive' + get 'metrics(/:dashboard_path)', constraints: { dashboard_path: /.+\.yml/ }, + to: 'metrics_dashboard#show', as: :metrics_dashboard, format: false resources :artifacts, only: [:index, :destroy] @@ -80,6 +82,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do resource :operations, only: [:show, :update] do member do post :reset_alerting_token + post :reset_pagerduty_token end end @@ -181,7 +184,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - resources :releases, only: [:index, :show, :edit], param: :tag, constraints: { tag: %r{[^/]+} } do + resources :releases, only: [:index, :new, :show, :edit], param: :tag, constraints: { tag: %r{[^/]+} } do member do get :downloads, path: 'downloads/*filepath', format: false scope module: :releases do @@ -257,7 +260,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do # This route is also defined in gitlab-workhorse. Make sure to update accordingly. get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', format: false - get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#proxy', as: :prometheus_api + get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#prometheus_proxy', as: :prometheus_api get '/sample_metrics', to: 'environments/sample_metrics#query' end @@ -313,6 +316,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end + get '/snippets/:snippet_id/raw/:ref/*path', + to: 'snippets/blobs#raw', + format: false, + as: :snippet_blob_raw, + constraints: { snippet_id: /\d+/ } + draw :issues draw :merge_requests draw :pipelines @@ -333,6 +342,12 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do # Look for scope '-' at the top of the file. # + # Service Desk + # + get '/service_desk' => 'service_desk#show', as: :service_desk + put '/service_desk' => 'service_desk#update', as: :service_desk_refresh + + # # Templates # get '/templates/:template_type/:key' => 'templates#show', @@ -363,6 +378,19 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end + # Serve snippet routes under /-/snippets. + # To ensure an old unscoped routing is used for the UI we need to + # add prefix 'as' to the scope routing and place it below original routing. + # Issue https://gitlab.com/gitlab-org/gitlab/-/issues/29572 + scope '-', as: :scoped do + resources :snippets, concerns: :awardable, constraints: { id: /\d+/ } do # rubocop: disable Cop/PutProjectRoutesUnderScope + member do + get :raw + post :mark_as_spam + end + end + end + namespace :prometheus do resources :alerts, constraints: { id: /\d+/ }, only: [:index, :create, :show, :update, :destroy] do # rubocop: disable Cop/PutProjectRoutesUnderScope post :notify, on: :collection @@ -379,6 +407,8 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do post 'alerts/notify', to: 'alerting/notifications#create' + post 'incidents/pagerduty', to: 'incident_management/pager_duty_incidents#create' + draw :legacy_builds resources :hooks, only: [:index, :create, :edit, :update, :destroy], constraints: { id: /\d+/ } do # rubocop: disable Cop/PutProjectRoutesUnderScope diff --git a/config/routes/snippets.rb b/config/routes/snippets.rb index ba6da3ac57e..1ea9a6431d8 100644 --- a/config/routes/snippets.rb +++ b/config/routes/snippets.rb @@ -17,5 +17,14 @@ resources :snippets, concerns: :awardable do end end +# Use this /-/ scope for all new snippet routes. +scope path: '-' do + get '/snippets/:snippet_id/raw/:ref/*path', + to: 'snippets/blobs#raw', + as: :snippet_blob_raw, + format: false, + constraints: { snippet_id: /\d+/ } +end + get '/s/:username', to: redirect('users/%{username}/snippets'), constraints: { username: /[a-zA-Z.0-9_\-]+(?<!\.atom)/ } diff --git a/config/routes/user.rb b/config/routes/user.rb index 9db3a71a270..3bf13908d39 100644 --- a/config/routes/user.rb +++ b/config/routes/user.rb @@ -56,7 +56,7 @@ end constraints(::Constraints::UserUrlConstrainer.new) do # Get all keys of user - get ':username.keys' => 'profiles/keys#get_keys', constraints: { username: Gitlab::PathRegex.root_namespace_route_regex } + get ':username.keys', controller: :users, action: :ssh_keys, constraints: { username: Gitlab::PathRegex.root_namespace_route_regex } scope(path: ':username', as: :user, diff --git a/config/routes/wiki.rb b/config/routes/wiki.rb index d439c99270e..49ad39e8369 100644 --- a/config/routes/wiki.rb +++ b/config/routes/wiki.rb @@ -3,13 +3,17 @@ scope(controller: :wikis) do get :git_access get :pages get :new - get '/', to: redirect('%{namespace_id}/%{project_id}/wikis/home') + get '/', to: redirect { |params, request| "#{request.path}/home" } post '/', to: 'wikis#create' + scope '-' do + resource :confluence, only: :show + end end scope(path: 'wikis/*id', as: :wiki, format: false) do get :edit get :history + get :diff post :preview_markdown get '/', action: :show put '/', action: :update diff --git a/config/settings.rb b/config/settings.rb index 99f1b85202e..c681fa32491 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -184,14 +184,15 @@ class Settings < Settingslogic URI.parse(url_without_path).host end - # Runs at a random time of day on a consistent day of the week based on + # Runs at a consistent random time of day on a day of the week based on # the instance UUID. This is to balance the load on the service receiving # these pings. The sidekiq job handles temporary http failures. def cron_for_usage_ping - hour = rand(24) - minute = rand(60) # Set a default UUID for the case when the UUID hasn't been initialized. uuid = Gitlab::CurrentSettings.uuid || 'uuid-not-set' + + minute = Digest::MD5.hexdigest(uuid + 'minute').to_i(16) % 60 + hour = Digest::MD5.hexdigest(uuid + 'hour').to_i(16) % 24 day_of_week = Digest::MD5.hexdigest(uuid).to_i(16) % 7 "#{minute} #{hour} * * #{day_of_week}" diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 0052910b56e..8dd60bcd65c 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -78,8 +78,6 @@ - 1 - - detect_repository_languages - 1 -- - elastic_batch_project_indexer - - 1 - - elastic_commit_indexer - 1 - - elastic_delete_project @@ -166,8 +164,6 @@ - 2 - - new_note - 2 -- - notifications - - 2 - - object_pool - 1 - - object_storage @@ -262,6 +258,8 @@ - 1 - - todos_destroyer - 1 +- - unassign_issuables + - 1 - - update_external_pull_requests - 3 - - update_highest_role @@ -274,6 +272,8 @@ - 1 - - upload_checksum - 1 +- - vulnerabilities_statistics_adjustment + - 1 - - vulnerability_exports_export - 1 - - vulnerability_exports_export_deletion diff --git a/config/webpack.config.js b/config/webpack.config.js index 557db58b1b9..8e51ce537c5 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -5,7 +5,7 @@ const webpack = require('webpack'); const VueLoaderPlugin = require('vue-loader/lib/plugin'); const StatsWriterPlugin = require('webpack-stats-plugin').StatsWriterPlugin; const CompressionPlugin = require('compression-webpack-plugin'); -const MonacoWebpackPlugin = require('monaco-editor-webpack-plugin'); +const MonacoWebpackPlugin = require('./plugins/monaco_webpack'); const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin; const CopyWebpackPlugin = require('copy-webpack-plugin'); const vendorDllHash = require('./helpers/vendor_dll_hash'); @@ -241,7 +241,7 @@ module.exports = { }, { test: /\.(eot|ttf|woff|woff2)$/, - include: /node_modules\/katex\/dist\/fonts/, + include: /node_modules\/(katex\/dist\/fonts|monaco-editor)/, loader: 'file-loader', options: { name: '[name].[contenthash:8].[ext]', |