diff options
Diffstat (limited to 'config/initializers')
| -rw-r--r-- | config/initializers/01_active_record_database_tasks_configuration_flag.rb | 37 | ||||
| -rw-r--r-- | config/initializers/1_settings.rb | 19 | ||||
| -rw-r--r-- | config/initializers/7_prometheus_metrics.rb | 101 | ||||
| -rw-r--r-- | config/initializers/action_cable.rb | 1 | ||||
| -rw-r--r-- | config/initializers/database_query_analyzers.rb | 16 | ||||
| -rw-r--r-- | config/initializers/gitlab_experiment.rb | 2 | ||||
| -rw-r--r-- | config/initializers/http_hostname_override.rb | 1 | ||||
| -rw-r--r-- | config/initializers/rails_host_authorization.rb | 2 | ||||
| -rw-r--r-- | config/initializers/sidekiq.rb | 4 |
9 files changed, 124 insertions, 59 deletions
diff --git a/config/initializers/01_active_record_database_tasks_configuration_flag.rb b/config/initializers/01_active_record_database_tasks_configuration_flag.rb new file mode 100644 index 00000000000..37374a41a30 --- /dev/null +++ b/config/initializers/01_active_record_database_tasks_configuration_flag.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +if Rails::VERSION::MAJOR >= 7 + raise "Remove `#{__FILE__}`. This is backport of `database_tasks:` Rails 7.x feature." +end + +# This backports `database_tasks:` feature to skip running migrations for some databases +# PR: https://github.com/rails/rails/pull/42794/files + +module DatabaseTasks + module ActiveRecordDatabaseConfigurations + def configs_for(env_name: nil, name: nil, include_replicas: false) + configs = super + + unless include_replicas + if name + configs = nil unless configs&.database_tasks? + else + configs = configs.select do |db_config| + db_config.database_tasks? + end + end + end + + configs + end + end + + module ActiveRecordDatabaseConfigurationsHashConfig + def database_tasks? # :nodoc: + !replica? && !!configuration_hash.fetch(:database_tasks, true) + end + end +end + +ActiveRecord::DatabaseConfigurations.prepend(DatabaseTasks::ActiveRecordDatabaseConfigurations) +ActiveRecord::DatabaseConfigurations::HashConfig.prepend(DatabaseTasks::ActiveRecordDatabaseConfigurationsHashConfig) diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 07d6e8e4882..25d142566f0 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -611,28 +611,40 @@ Settings.cron_jobs['ci_delete_unit_tests_worker']['job_class'] = 'Ci::DeleteUnit Settings.cron_jobs['batched_background_migrations_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['batched_background_migrations_worker']['cron'] ||= '* * * * *' Settings.cron_jobs['batched_background_migrations_worker']['job_class'] = 'Database::BatchedBackgroundMigrationWorker' +Settings.cron_jobs['batched_background_migration_worker_ci_database'] ||= Settingslogic.new({}) +Settings.cron_jobs['batched_background_migration_worker_ci_database']['cron'] ||= '* * * * *' +Settings.cron_jobs['batched_background_migration_worker_ci_database']['job_class'] = 'Database::BatchedBackgroundMigration::CiDatabaseWorker' Settings.cron_jobs['issues_reschedule_stuck_issue_rebalances'] ||= Settingslogic.new({}) Settings.cron_jobs['issues_reschedule_stuck_issue_rebalances']['cron'] ||= '*/15 * * * *' Settings.cron_jobs['issues_reschedule_stuck_issue_rebalances']['job_class'] = 'Issues::RescheduleStuckIssueRebalancesWorker' Settings.cron_jobs['clusters_integrations_check_prometheus_health_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['clusters_integrations_check_prometheus_health_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['clusters_integrations_check_prometheus_health_worker']['job_class'] = 'Clusters::Integrations::CheckPrometheusHealthWorker' +Settings.cron_jobs['projects_schedule_refresh_build_artifacts_size_statistics_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['projects_schedule_refresh_build_artifacts_size_statistics_worker']['cron'] ||= '2/17 * * * *' +Settings.cron_jobs['projects_schedule_refresh_build_artifacts_size_statistics_worker']['job_class'] = 'Projects::ScheduleRefreshBuildArtifactsSizeStatisticsWorker' Gitlab.ee do Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['cron'] ||= '0 1 * * *' Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker']['job_class'] = 'Analytics::DevopsAdoption::CreateAllSnapshotsWorker' + Settings.cron_jobs['analytics_cycle_analytics_incremental_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['analytics_cycle_analytics_incremental_worker']['cron'] ||= '*/10 * * * *' + Settings.cron_jobs['analytics_cycle_analytics_incremental_worker']['job_class'] = 'Analytics::CycleAnalytics::IncrementalWorker' + Settings.cron_jobs['analytics_cycle_analytics_consistency_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['analytics_cycle_analytics_consistency_worker']['cron'] ||= '*/30 * * * *' + Settings.cron_jobs['analytics_cycle_analytics_consistency_worker']['job_class'] = 'Analytics::CycleAnalytics::ConsistencyWorker' Settings.cron_jobs['active_user_count_threshold_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['active_user_count_threshold_worker']['cron'] ||= '0 12 * * *' Settings.cron_jobs['active_user_count_threshold_worker']['job_class'] = 'ActiveUserCountThresholdWorker' Settings.cron_jobs['adjourned_group_deletion_worker'] ||= Settingslogic.new({}) - Settings.cron_jobs['adjourned_group_deletion_worker']['cron'] ||= '0 3 * * *' + Settings.cron_jobs['adjourned_group_deletion_worker']['cron'] ||= '0 2 * * *' Settings.cron_jobs['adjourned_group_deletion_worker']['job_class'] = 'AdjournedGroupDeletionWorker' Settings.cron_jobs['clear_shared_runners_minutes_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['clear_shared_runners_minutes_worker']['cron'] ||= '0 0 1 * *' Settings.cron_jobs['clear_shared_runners_minutes_worker']['job_class'] = 'ClearSharedRunnersMinutesWorker' Settings.cron_jobs['adjourned_projects_deletion_cron_worker'] ||= Settingslogic.new({}) - Settings.cron_jobs['adjourned_projects_deletion_cron_worker']['cron'] ||= '0 4 * * *' + Settings.cron_jobs['adjourned_projects_deletion_cron_worker']['cron'] ||= '0 7 * * *' Settings.cron_jobs['adjourned_projects_deletion_cron_worker']['job_class'] = 'AdjournedProjectsDeletionCronWorker' Settings.cron_jobs['geo_verification_cron_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['geo_verification_cron_worker']['cron'] ||= '* * * * *' @@ -739,6 +751,9 @@ Gitlab.ee do Settings.cron_jobs['security_orchestration_policy_rule_schedule_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['security_orchestration_policy_rule_schedule_worker']['cron'] ||= '*/15 * * * *' Settings.cron_jobs['security_orchestration_policy_rule_schedule_worker']['job_class'] = 'Security::OrchestrationPolicyRuleScheduleWorker' + Settings.cron_jobs['security_findings_cleanup_worker'] ||= Settingslogic.new({}) + Settings.cron_jobs['security_findings_cleanup_worker']['cron'] ||= '0 */4 * * 6,0' + Settings.cron_jobs['security_findings_cleanup_worker']['job_class'] = 'Security::Findings::CleanupWorker' Settings.cron_jobs['app_sec_dast_profile_schedule_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['app_sec_dast_profile_schedule_worker']['cron'] ||= '7-59/15 * * * *' Settings.cron_jobs['app_sec_dast_profile_schedule_worker']['job_class'] = 'AppSec::Dast::ProfileScheduleWorker' diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index c55b074f9c8..e4d47d53815 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +PUMA_EXTERNAL_METRICS_SERVER = Gitlab::Utils.to_boolean(ENV['PUMA_EXTERNAL_METRICS_SERVER']) +require Rails.root.join('metrics_server', 'metrics_server') if PUMA_EXTERNAL_METRICS_SERVER + # Keep separate directories for separate processes def prometheus_default_multiproc_dir return unless Rails.env.development? || Rails.env.test? @@ -34,8 +37,6 @@ end ::Prometheus::Client.configure do |config| config.logger = Gitlab::AppLogger - config.initial_mmap_file_size = 4 * 1024 - config.multiprocess_files_dir = ENV['prometheus_multiproc_dir'] || prometheus_default_multiproc_dir config.pid_provider = ::Prometheus::PidProvider.method(:worker_id) @@ -46,6 +47,10 @@ Gitlab::Application.configure do |config| config.middleware.insert_after(Labkit::Middleware::Rack, Gitlab::Metrics::RequestsRackMiddleware) end +# Any actions beyond this check should only execute outside of tests, when running in an application +# context (i.e. not in the Rails console or rspec) and when users have enabled metrics. +return if Rails.env.test? || !Gitlab::Runtime.application? || !Gitlab::Metrics.prometheus_metrics_enabled? + if Gitlab::Runtime.sidekiq? && (!ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER_ID'] == '0') # The single worker outside of a sidekiq-cluster, or the first worker (sidekiq_0) # in a cluster of processes, is responsible for serving health checks. @@ -64,67 +69,75 @@ if Gitlab::Runtime.sidekiq? && (!ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER end end -if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled? - Gitlab::Cluster::LifecycleEvents.on_master_start do - if Gitlab::Runtime.puma? - Gitlab::Metrics::Samplers::PumaSampler.instance.start - end +Gitlab::Cluster::LifecycleEvents.on_master_start do + Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1) - Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1) + if Gitlab::Runtime.puma? + Gitlab::Metrics::Samplers::PumaSampler.instance.start - Gitlab::Ci::Parsers.instrument! - rescue IOError => e - Gitlab::ErrorTracking.track_exception(e) - Gitlab::Metrics.error_detected! + if PUMA_EXTERNAL_METRICS_SERVER && Settings.monitoring.web_exporter.enabled + MetricsServer.start_for_puma + else + Gitlab::Metrics::Exporter::WebExporter.instance.start + end end - Gitlab::Cluster::LifecycleEvents.on_worker_start do - defined?(::Prometheus::Client.reinitialize_on_pid_change) && ::Prometheus::Client.reinitialize_on_pid_change - logger = Gitlab::AppLogger - Gitlab::Metrics::Samplers::RubySampler.initialize_instance(logger: logger).start - Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance(logger: logger).start - Gitlab::Metrics::Samplers::ThreadsSampler.initialize_instance(logger: logger).start - - if Gitlab::Runtime.web_server? - Gitlab::Metrics::Samplers::ActionCableSampler.instance(logger: logger).start - end + Gitlab::Ci::Parsers.instrument! +rescue IOError => e + Gitlab::ErrorTracking.track_exception(e) + Gitlab::Metrics.error_detected! +end - if Gitlab.ee? && Gitlab::Runtime.sidekiq? - Gitlab::Metrics::Samplers::GlobalSearchSampler.instance(logger: logger).start +Gitlab::Cluster::LifecycleEvents.on_worker_start do + defined?(::Prometheus::Client.reinitialize_on_pid_change) && ::Prometheus::Client.reinitialize_on_pid_change + logger = Gitlab::AppLogger + Gitlab::Metrics::Samplers::RubySampler.initialize_instance(logger: logger).start + Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance(logger: logger).start + Gitlab::Metrics::Samplers::ThreadsSampler.initialize_instance(logger: logger).start + + if Gitlab::Runtime.puma? + # Since we are observing a metrics server from the Puma primary, we would inherit + # this supervision thread after forking into workers, so we need to explicitly stop it here. + if PUMA_EXTERNAL_METRICS_SERVER + ::MetricsServer::PumaProcessSupervisor.instance.stop + else + Gitlab::Metrics::Exporter::WebExporter.instance.stop end - Gitlab::Ci::Parsers.instrument! - rescue IOError => e - Gitlab::ErrorTracking.track_exception(e) - Gitlab::Metrics.error_detected! + Gitlab::Metrics::Samplers::ActionCableSampler.instance(logger: logger).start end -end -if Gitlab::Runtime.web_server? - Gitlab::Cluster::LifecycleEvents.on_master_start do - Gitlab::Metrics::Exporter::WebExporter.instance.start + if Gitlab.ee? && Gitlab::Runtime.sidekiq? + Gitlab::Metrics::Samplers::GlobalSearchSampler.instance(logger: logger).start end + Gitlab::Ci::Parsers.instrument! +rescue IOError => e + Gitlab::ErrorTracking.track_exception(e) + Gitlab::Metrics.error_detected! +end + +if Gitlab::Runtime.puma? Gitlab::Cluster::LifecycleEvents.on_before_graceful_shutdown do # We need to ensure that before we re-exec or shutdown server - # we do stop the exporter - Gitlab::Metrics::Exporter::WebExporter.instance.stop + # we also stop the metrics server + if PUMA_EXTERNAL_METRICS_SERVER + ::MetricsServer::PumaProcessSupervisor.instance.shutdown + else + Gitlab::Metrics::Exporter::WebExporter.instance.stop + end end Gitlab::Cluster::LifecycleEvents.on_before_master_restart do # We need to ensure that before we re-exec server - # we do stop the exporter + # we also stop the metrics server # # We do it again, for being extra safe, # but it should not be needed - Gitlab::Metrics::Exporter::WebExporter.instance.stop - end - - Gitlab::Cluster::LifecycleEvents.on_worker_start do - # The `#close_on_exec=` takes effect only on `execve` - # but this does not happen for Ruby fork - # - # This does stop server, as it is running on master. - Gitlab::Metrics::Exporter::WebExporter.instance.stop + if PUMA_EXTERNAL_METRICS_SERVER + ::MetricsServer::PumaProcessSupervisor.instance.shutdown + else + Gitlab::Metrics::Exporter::WebExporter.instance.stop + end end end diff --git a/config/initializers/action_cable.rb b/config/initializers/action_cable.rb index 0d2073586be..3f9ceb7cfa7 100644 --- a/config/initializers/action_cable.rb +++ b/config/initializers/action_cable.rb @@ -10,6 +10,7 @@ Rails.application.configure do end ActionCable::SubscriptionAdapter::Base.prepend(Gitlab::Patch::ActionCableSubscriptionAdapterIdentifier) +ActionCable::SubscriptionAdapter::Redis::Listener.prepend(Gitlab::Patch::ActionCableRedisListener) # https://github.com/rails/rails/blob/bb5ac1623e8de08c1b7b62b1368758f0d3bb6379/actioncable/lib/action_cable/subscription_adapter/redis.rb#L18 ActionCable::SubscriptionAdapter::Redis.redis_connector = lambda do |config| diff --git a/config/initializers/database_query_analyzers.rb b/config/initializers/database_query_analyzers.rb index 8a2fe1d8388..d1010e054af 100644 --- a/config/initializers/database_query_analyzers.rb +++ b/config/initializers/database_query_analyzers.rb @@ -1,15 +1,13 @@ # frozen_string_literal: true # Currently we register validator only for `dev` or `test` environment -if Gitlab.dev_or_test_env? || Gitlab::Utils.to_boolean(ENV['GITLAB_ENABLE_QUERY_ANALYZERS'], default: false) - Gitlab::Database::QueryAnalyzer.instance.hook! - Gitlab::Database::QueryAnalyzer.instance.all_analyzers.append(::Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics) +Gitlab::Database::QueryAnalyzer.instance.hook! +Gitlab::Database::QueryAnalyzer.instance.all_analyzers.append(::Gitlab::Database::QueryAnalyzers::GitlabSchemasMetrics) - if Rails.env.test? || Gitlab::Utils.to_boolean(ENV['ENABLE_CROSS_DATABASE_MODIFICATION_DETECTION'], default: false) - Gitlab::Database::QueryAnalyzer.instance.all_analyzers.append(::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification) - end +if Rails.env.test? || Gitlab::Utils.to_boolean(ENV['ENABLE_CROSS_DATABASE_MODIFICATION_DETECTION'], default: false) + Gitlab::Database::QueryAnalyzer.instance.all_analyzers.append(::Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification) +end - Gitlab::Application.configure do |config| - config.middleware.use(Gitlab::Middleware::QueryAnalyzer) - end +Gitlab::Application.configure do |config| + config.middleware.use(Gitlab::Middleware::QueryAnalyzer) end diff --git a/config/initializers/gitlab_experiment.rb b/config/initializers/gitlab_experiment.rb index fdb21d90c28..9a6ef325c9e 100644 --- a/config/initializers/gitlab_experiment.rb +++ b/config/initializers/gitlab_experiment.rb @@ -49,7 +49,7 @@ Gitlab::Experiment.configure do |config| # valid_domains = %w[about.gitlab.com docs.gitlab.com gitlab.com gdk.test localhost] config.redirect_url_validator = lambda do |url| - Gitlab.dev_env_or_com? && (url = URI.parse(url)) && valid_domains.include?(url.host) + Gitlab.com? && (url = URI.parse(url)) && valid_domains.include?(url.host) rescue URI::InvalidURIError false end diff --git a/config/initializers/http_hostname_override.rb b/config/initializers/http_hostname_override.rb index 5d2739c1f58..3d840cd3251 100644 --- a/config/initializers/http_hostname_override.rb +++ b/config/initializers/http_hostname_override.rb @@ -34,6 +34,7 @@ end class Net::HTTP attr_accessor :hostname_override + SSL_IVNAMES << :@hostname_override SSL_ATTRIBUTES << :hostname_override diff --git a/config/initializers/rails_host_authorization.rb b/config/initializers/rails_host_authorization.rb index 22bb6fb7061..6fe4679433c 100644 --- a/config/initializers/rails_host_authorization.rb +++ b/config/initializers/rails_host_authorization.rb @@ -8,7 +8,7 @@ if Gitlab.config.gitlab.allowed_hosts.present? end if Rails.env.development? - Rails.application.config.hosts += [Gitlab.config.gitlab.host, 'unix', 'host.docker.internal'] + Rails.application.config.hosts += [Gitlab.config.gitlab.host, 'unix', 'host.docker.internal', 'docker.for.mac.localhost'] if ENV['RAILS_HOSTS'] additional_hosts = ENV['RAILS_HOSTS'].split(',').select(&:presence) diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index baf252a5d10..25984c45318 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -30,8 +30,6 @@ Sidekiq.configure_server do |config| config.options[:strict] = false config.options[:queues] = Gitlab::SidekiqConfig.expand_queues(config.options[:queues]) - Sidekiq.logger.info "Listening on queues #{config.options[:queues].uniq.sort}" - if enable_json_logs config.log_formatter = Gitlab::SidekiqLogging::JSONFormatter.new config.options[:job_logger] = Gitlab::SidekiqLogging::StructuredLogger @@ -41,6 +39,8 @@ Sidekiq.configure_server do |config| config.error_handlers.reject! { |handler| handler.is_a?(Sidekiq::ExceptionHandler::Logger) } end + Sidekiq.logger.info "Listening on queues #{config.options[:queues].uniq.sort}" + config.redis = queues_config_hash config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator( |