diff options
Diffstat (limited to 'config')
35 files changed, 899 insertions, 114 deletions
diff --git a/config/application.rb b/config/application.rb index c5ef6a2c60d..de386506233 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,11 +1,18 @@ -require File.expand_path('boot', __dir__) +require_relative 'boot' -require 'rails/all' +# Based on https://github.com/rails/rails/blob/v5.2.3/railties/lib/rails/all.rb +# Only load the railties we need instead of loading everything +require 'active_record/railtie' +require 'action_controller/railtie' +require 'action_view/railtie' +require 'action_mailer/railtie' +require 'rails/test_unit/railtie' -Bundler.require(:default, Rails.env) +Bundler.require(*Rails.groups) module Gitlab class Application < Rails::Application + require_dependency Rails.root.join('lib/gitlab') require_dependency Rails.root.join('lib/gitlab/redis/wrapper') require_dependency Rails.root.join('lib/gitlab/redis/cache') require_dependency Rails.root.join('lib/gitlab/redis/queues') @@ -24,6 +31,8 @@ module Gitlab # Application configuration should go into files in config/initializers # -- all .rb files in that directory are automatically loaded. + config.active_record.sqlite3.represent_boolean_as_integer = true + # Sidekiq uses eager loading, but directories not in the standard Rails # directories must be added to the eager load paths: # https://github.com/mperham/sidekiq/wiki/FAQ#why-doesnt-sidekiq-autoload-my-rails-application-code @@ -85,13 +94,6 @@ module Gitlab # Configure the default encoding used in templates for Ruby 1.9. config.encoding = "utf-8" - # ActionCable mount point. - # The default Rails' mount point is `/cable` which may conflict with existing - # namespaces/users. - # https://github.com/rails/rails/blob/5-0-stable/actioncable/lib/action_cable.rb#L38 - # Please change this value when configuring ActionCable for real usage. - config.action_cable.mount_path = "/-/cable" - # Configure sensitive parameters which will be filtered from the log file. # # Parameters filtered: @@ -271,5 +273,10 @@ module Gitlab Gitlab::Routing.add_helpers(project_url_helpers) Gitlab::Routing.add_helpers(MilestonesRoutingHelper) end + + # This makes generated cookies to be compatible with Rails 5.1 and older + # We can remove this when we're confident that there are no issues with the Rails 5.2 upgrade + # and we won't need to rollback to older versions + config.action_dispatch.use_authenticated_cookie_encryption = false end end diff --git a/config/brakeman.ignore b/config/brakeman.ignore new file mode 100644 index 00000000000..0e4fef65781 --- /dev/null +++ b/config/brakeman.ignore @@ -0,0 +1,24 @@ +{ + "ignored_warnings": [ + { + "warning_type": "Cross-Site Request Forgery", + "warning_code": 7, + "fingerprint": "dc562678129557cdb8b187217da304044547a3605f05fe678093dcb4b4d8bbe4", + "message": "'protect_from_forgery' should be called in Oauth::GeoAuthController", + "file": "app/controllers/oauth/geo_auth_controller.rb", + "line": 1, + "link": "http://brakemanscanner.org/docs/warning_types/cross-site_request_forgery/", + "code": null, + "render_path": null, + "location": { + "type": "controller", + "controller": "Oauth::GeoAuthController" + }, + "user_input": null, + "confidence": "High", + "note": "" + } + ], + "updated": "2017-01-20 02:06:54 +0000", + "brakeman_version": "3.4.1" +} diff --git a/config/database_geo.yml.postgresql b/config/database_geo.yml.postgresql new file mode 100644 index 00000000000..2918879f7ed --- /dev/null +++ b/config/database_geo.yml.postgresql @@ -0,0 +1,51 @@ +# +# PRODUCTION +# +production: + adapter: postgresql + encoding: unicode + database: gitlabhq_geo_production + pool: 10 + username: git + password: "secure password" + host: localhost + fdw: true + +# +# Development specific +# +development: + adapter: postgresql + encoding: unicode + database: gitlabhq_geo_development + pool: 5 + username: postgres + password: "secure password" + host: localhost + fdw: true + +# +# Staging specific +# +staging: + adapter: postgresql + encoding: unicode + database: gitlabhq_geo_staging + pool: 10 + username: git + password: "secure password" + host: localhost + fdw: true + +# Warning: The database defined as "test" will be erased and +# re-generated from your development database when you run "rake". +# Do not set this db to the same as development or production. +test: &test + adapter: postgresql + encoding: unicode + database: gitlabhq_geo_test + pool: 5 + username: postgres + password: + host: localhost + fdw: true diff --git a/config/environments/development.rb b/config/environments/development.rb index ac9b02b08d5..3881f1be152 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -7,6 +7,7 @@ Rails.application.configure do config.cache_classes = false # Show full error reports and disable caching + config.active_record.verbose_query_logs = true config.consider_all_requests_local = true config.action_controller.perform_caching = false diff --git a/config/environments/test.rb b/config/environments/test.rb index e7166882eea..153d16e4e55 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -23,6 +23,7 @@ Rails.application.configure do config.public_file_server.headers = { 'Cache-Control' => 'public, max-age=3600' } # Show full error reports and disable caching + config.active_record.verbose_query_logs = true config.consider_all_requests_local = true config.action_controller.perform_caching = false @@ -40,7 +41,7 @@ Rails.application.configure do # Print deprecation notices to the stderr config.active_support.deprecation = :stderr - config.eager_load = false + config.eager_load = true config.cache_store = :null_store diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index c82d9b5ceef..334c241bcaa 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -664,6 +664,9 @@ production: &base # Port where the client side certificate is requested by the webserver (NGINX/Apache) # client_certificate_required_port: 3444 + # Browser session with smartcard sign-in is required for Git access + # required_for_git_access: false + ## Kerberos settings kerberos: # Allow the HTTP Negotiate authentication method for Git clients diff --git a/config/initializers/0_license.rb b/config/initializers/0_license.rb new file mode 100644 index 00000000000..f750022dfdf --- /dev/null +++ b/config/initializers/0_license.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +Gitlab.ee do + begin + public_key_file = File.read(Rails.root.join(".license_encryption_key.pub")) + public_key = OpenSSL::PKey::RSA.new(public_key_file) + Gitlab::License.encryption_key = public_key + rescue + warn "WARNING: No valid license encryption key provided." + end + + # Needed to run migration + if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.data_source_exists?('licenses') + message = LicenseHelper.license_message(signed_in: true, is_admin: true, in_html: false) + if ::License.block_changes? && message.present? + warn "WARNING: #{message}" + end + end +end diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index bf187e9a282..3a121addc98 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -55,7 +55,7 @@ if Settings.ldap['enabled'] || Rails.env.test? server['tls_options'] ||= {} if server['ssl_version'] || server['ca_file'] - Rails.logger.warn 'DEPRECATED: LDAP options `ssl_version` and `ca_file` should be nested within `tls_options`' + Rails.logger.warn 'DEPRECATED: LDAP options `ssl_version` and `ca_file` should be nested within `tls_options`' # rubocop:disable Gitlab/RailsLogger end if server['ssl_version'] @@ -76,6 +76,7 @@ Gitlab.ee do Settings['smartcard'] ||= Settingslogic.new({}) Settings.smartcard['enabled'] = false if Settings.smartcard['enabled'].nil? Settings.smartcard['client_certificate_required_port'] = 3444 if Settings.smartcard['client_certificate_required_port'].nil? + Settings.smartcard['required_for_git_access'] = false if Settings.smartcard['required_for_git_access'].nil? end Settings['omniauth'] ||= Settingslogic.new({}) diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index 741c8ef1ca0..53c3eac3c74 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -17,7 +17,7 @@ def prometheus_default_multiproc_dir end Prometheus::Client.configure do |config| - config.logger = Rails.logger + config.logger = Rails.logger # rubocop:disable Gitlab/RailsLogger config.initial_mmap_file_size = 4 * 1024 diff --git a/config/initializers/active_record_data_types.rb b/config/initializers/active_record_data_types.rb index e95157bfde5..151bce4d130 100644 --- a/config/initializers/active_record_data_types.rb +++ b/config/initializers/active_record_data_types.rb @@ -22,7 +22,7 @@ if Gitlab::Database.postgresql? # # When schema dumping, `timestamptz` columns will be output as # `t.datetime_with_timezone`. - def initialize_type_map(mapping) + def initialize_type_map(mapping = type_map) super mapping mapping.register_type 'timestamptz' do |_, _, sql_type| @@ -51,7 +51,7 @@ elsif Gitlab::Database.mysql? # # When schema dumping, `timestamp` columns will be output as # `t.datetime_with_timezone`. - def initialize_type_map(mapping) + def initialize_type_map(mapping = type_map) super mapping mapping.register_type(/timestamp/i) do |sql_type| diff --git a/config/initializers/active_record_lifecycle.rb b/config/initializers/active_record_lifecycle.rb index 7fa37121efc..61f1d299960 100644 --- a/config/initializers/active_record_lifecycle.rb +++ b/config/initializers/active_record_lifecycle.rb @@ -7,7 +7,7 @@ if defined?(ActiveRecord::Base) && !Sidekiq.server? ActiveSupport.on_load(:active_record) do ActiveRecord::Base.establish_connection - Rails.logger.debug("ActiveRecord connection established") + Rails.logger.debug("ActiveRecord connection established") # rubocop:disable Gitlab/RailsLogger end end end @@ -18,6 +18,6 @@ if defined?(ActiveRecord::Base) # as there's no need for the master process to hold a connection ActiveRecord::Base.connection.disconnect! - Rails.logger.debug("ActiveRecord connection disconnected") + Rails.logger.debug("ActiveRecord connection disconnected") # rubocop:disable Gitlab/RailsLogger end end diff --git a/config/initializers/active_record_preloader.rb b/config/initializers/active_record_preloader.rb index 3b16014f302..a293909149e 100644 --- a/config/initializers/active_record_preloader.rb +++ b/config/initializers/active_record_preloader.rb @@ -1,9 +1,22 @@ module ActiveRecord module Associations class Preloader + class NullPreloader + def self.new(klass, owners, reflection, preload_scope) + self + end + + def self.run(preloader) + end + + def self.preloaded_records + [] + end + end + module NoCommitPreloader - def preloader_for(reflection, owners, rhs_klass) - return NullPreloader if rhs_klass == ::Commit + def preloader_for(reflection, owners) + return NullPreloader if owners.first.association(reflection.name).klass == ::Commit super end diff --git a/config/initializers/active_record_query_cache.rb b/config/initializers/active_record_query_cache.rb new file mode 100644 index 00000000000..61505a1edd3 --- /dev/null +++ b/config/initializers/active_record_query_cache.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +ActiveRecord::ConnectionAdapters::ConnectionPool.prepend Gitlab::Patch::ActiveRecordQueryCache diff --git a/config/initializers/active_record_verbose_query_logs.rb b/config/initializers/active_record_verbose_query_logs.rb deleted file mode 100644 index 1c5fbc8e830..00000000000 --- a/config/initializers/active_record_verbose_query_logs.rb +++ /dev/null @@ -1,56 +0,0 @@ -# frozen_string_literal: true - -# This is backport of https://github.com/rails/rails/pull/26815/files -# Enabled by default for every non-production environment - -module ActiveRecord - class LogSubscriber - module VerboseQueryLogs - def debug(progname = nil, &block) - return unless super - - log_query_source - end - - def log_query_source - source_line, line_number = extract_callstack(caller_locations) - - if source_line - if defined?(::Rails.root) - app_root = "#{::Rails.root}/".freeze - source_line = source_line.sub(app_root, "") - end - - logger.debug(" ↳ #{source_line}:#{line_number}") - end - end - - def extract_callstack(callstack) - line = callstack.find do |frame| - frame.absolute_path && !ignored_callstack(frame.absolute_path) - end - - offending_line = line || callstack.first - [ - offending_line.path, - offending_line.lineno, - offending_line.label - ] - end - - LOG_SUBSCRIBER_FILE = ActiveRecord::LogSubscriber.method(:logger).source_location.first - RAILS_GEM_ROOT = File.expand_path("../../../..", LOG_SUBSCRIBER_FILE) + "/" - APP_CONFIG_ROOT = File.expand_path("..", __dir__) + "/" - - def ignored_callstack(path) - path.start_with?(APP_CONFIG_ROOT, RAILS_GEM_ROOT, RbConfig::CONFIG["rubylibdir"]) - end - end - - if Rails.version.start_with?("5.2") - raise "Remove this monkey patch: #{__FILE__}" - else - prepend(VerboseQueryLogs) unless Rails.env.production? - end - end -end diff --git a/config/initializers/ar_speed_up_migration_checking.rb b/config/initializers/ar_speed_up_migration_checking.rb index aae774daa35..c4ffcc54cb2 100644 --- a/config/initializers/ar_speed_up_migration_checking.rb +++ b/config/initializers/ar_speed_up_migration_checking.rb @@ -2,16 +2,14 @@ if Rails.env.test? require 'active_record/migration' module ActiveRecord - class Migrator - class << self - alias_method :migrations_unmemoized, :migrations + class MigrationContext + alias_method :migrations_unmemoized, :migrations - # This method is called a large number of times per rspec example, and - # it reads + parses `db/migrate/*` each time. Memoizing it can save 0.5 - # seconds per spec. - def migrations(paths) - (@migrations ||= migrations_unmemoized(paths)).dup - end + # This method is called a large number of times per rspec example, and + # it reads + parses `db/migrate/*` each time. Memoizing it can save 0.5 + # seconds per spec. + def migrations + @migrations ||= migrations_unmemoized end end end diff --git a/config/initializers/config_initializers_active_record_locking.rb b/config/initializers/config_initializers_active_record_locking.rb index 608d63223a3..915247826e9 100644 --- a/config/initializers/config_initializers_active_record_locking.rb +++ b/config/initializers/config_initializers_active_record_locking.rb @@ -22,10 +22,11 @@ module ActiveRecord # Patched because when `lock_version` is read as `0`, it may actually be `NULL` in the DB. possible_previous_lock_value = previous_lock_value.to_i == 0 ? [nil, 0] : previous_lock_value - affected_rows = self.class.unscoped._update_record( - arel_attributes_with_values(attribute_names), - self.class.primary_key => id_in_database, - locking_column => possible_previous_lock_value + affected_rows = self.class.unscoped.where( + locking_column => possible_previous_lock_value, + self.class.primary_key => id_in_database + ).update_all( + attributes_with_values_for_update(attribute_names) ) if affected_rows != 1 diff --git a/config/initializers/console_message.rb b/config/initializers/console_message.rb index 05eb395028d..04c109aa844 100644 --- a/config/initializers/console_message.rb +++ b/config/initializers/console_message.rb @@ -2,9 +2,18 @@ if defined?(Rails::Console) # note that this will not print out when using `spring` justify = 15 - puts "-------------------------------------------------------------------------------------" + + puts '-' * 80 puts " GitLab:".ljust(justify) + "#{Gitlab::VERSION} (#{Gitlab.revision})" puts " GitLab Shell:".ljust(justify) + "#{Gitlab::VersionInfo.parse(Gitlab::Shell.new.version)}" puts " #{Gitlab::Database.human_adapter_name}:".ljust(justify) + Gitlab::Database.version - puts "-------------------------------------------------------------------------------------" + + Gitlab.ee do + if Gitlab::Geo.enabled? + puts " Geo enabled:".ljust(justify) + 'yes' + puts " Geo server:".ljust(justify) + EE::GeoHelper.current_node_human_status + end + end + + puts '-' * 80 end diff --git a/config/initializers/deprecations.rb b/config/initializers/deprecations.rb index 14616e726d9..0d096e34eb7 100644 --- a/config/initializers/deprecations.rb +++ b/config/initializers/deprecations.rb @@ -2,7 +2,7 @@ if Rails.env.development? || ENV['GITLAB_LEGACY_PATH_LOG_MESSAGE'] deprecator = ActiveSupport::Deprecation.new('11.0', 'GitLab') deprecator.behavior = -> (message, callstack) { - Rails.logger.warn("#{message}: #{callstack[1..20].join}") + Rails.logger.warn("#{message}: #{callstack[1..20].join}") # rubocop:disable Gitlab/RailsLogger } ActiveSupport::Deprecation.deprecate_methods(Gitlab::GitalyClient::StorageSettings, :legacy_disk_path, deprecator: deprecator) diff --git a/config/initializers/elastic_client_setup.rb b/config/initializers/elastic_client_setup.rb new file mode 100644 index 00000000000..2ecb7956007 --- /dev/null +++ b/config/initializers/elastic_client_setup.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# Be sure to restart your server when you modify this file. + +require 'gitlab/current_settings' + +Gitlab.ee do + Elasticsearch::Model::Response::Records.prepend GemExtensions::Elasticsearch::Model::Response::Records + Elasticsearch::Model::Adapter::Multiple::Records.prepend GemExtensions::Elasticsearch::Model::Adapter::Multiple::Records + Elasticsearch::Model::Indexing::InstanceMethods.prepend GemExtensions::Elasticsearch::Model::Indexing::InstanceMethods + + module Elasticsearch + module Model + module Client + # This mutex is only used to synchronize *creation* of a new client, so + # all including classes can share the same client instance + CLIENT_MUTEX = Mutex.new + + cattr_accessor :cached_client + cattr_accessor :cached_config + + module ClassMethods + # Override the default ::Elasticsearch::Model::Client implementation to + # return a client configured from application settings. All including + # classes will use the same instance, which is refreshed automatically + # if the settings change. + # + # _client is present to match the arity of the overridden method, where + # it is also not used. + # + # @return [Elasticsearch::Transport::Client] + def client(_client = nil) + store = ::Elasticsearch::Model::Client + + store::CLIENT_MUTEX.synchronize do + config = Gitlab::CurrentSettings.elasticsearch_config + + if store.cached_client.nil? || config != store.cached_config + store.cached_client = ::Gitlab::Elastic::Client.build(config) + store.cached_config = config + end + end + + store.cached_client + end + end + end + end + end +end diff --git a/config/initializers/geo.rb b/config/initializers/geo.rb new file mode 100644 index 00000000000..4cc9fbf49b2 --- /dev/null +++ b/config/initializers/geo.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +Gitlab.ee do + if File.exist?(Rails.root.join('config/database_geo.yml')) + Rails.application.configure do + config.geo_database = config_for(:database_geo) + end + end + + begin + if Gitlab::Geo.connected? && Gitlab::Geo.primary? + Gitlab::Geo.current_node&.update_clone_url! + end + rescue => e + warn "WARNING: Unable to check/update clone_url_prefix for Geo: #{e}" + end +end diff --git a/config/initializers/health_check.rb b/config/initializers/health_check.rb index 959daa93f78..9f466dc39de 100644 --- a/config/initializers/health_check.rb +++ b/config/initializers/health_check.rb @@ -1,4 +1,10 @@ HealthCheck.setup do |config| config.standard_checks = %w(database migrations cache) config.full_checks = %w(database migrations cache) + + Gitlab.ee do + config.add_custom_check('geo') do + Gitlab::Geo::HealthCheck.new.perform_checks + end + end end diff --git a/config/initializers/load_balancing.rb b/config/initializers/load_balancing.rb new file mode 100644 index 00000000000..029c0ff4277 --- /dev/null +++ b/config/initializers/load_balancing.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# We need to run this initializer after migrations are done so it doesn't fail on CI + +Gitlab.ee do + if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.data_source_exists?('licenses') + if Gitlab::Database::LoadBalancing.enable? + Gitlab::Database.disable_prepared_statements + + Gitlab::Application.configure do |config| + config.middleware.use(Gitlab::Database::LoadBalancing::RackMiddleware) + end + + Gitlab::Database::LoadBalancing.configure_proxy + + # This needs to be executed after fork of clustered processes + Gitlab::Cluster::LifecycleEvents.on_worker_start do + # Service discovery must be started after configuring the proxy, as service + # discovery depends on this. + Gitlab::Database::LoadBalancing.start_service_discovery + end + + end + end +end diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb index 1ad93e14f7e..fbec28186eb 100644 --- a/config/initializers/lograge.rb +++ b/config/initializers/lograge.rb @@ -18,7 +18,7 @@ unless Sidekiq.server? .map { |k, v| { key: k, value: v } } payload = { - time: event.time.utc.iso8601(3), + time: Time.now.utc.iso8601(3), params: params, remote_ip: event.payload[:remote_ip], user_id: event.payload[:user_id], diff --git a/config/initializers/mysql_ignore_postgresql_options.rb b/config/initializers/mysql_ignore_postgresql_options.rb index 9a569be7674..e6a7d9bef52 100644 --- a/config/initializers/mysql_ignore_postgresql_options.rb +++ b/config/initializers/mysql_ignore_postgresql_options.rb @@ -15,7 +15,6 @@ if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter) module ConnectionAdapters class Mysql2Adapter < AbstractMysqlAdapter alias_method :__gitlab_add_index, :add_index - alias_method :__gitlab_add_index_sql, :add_index_sql alias_method :__gitlab_add_index_options, :add_index_options def add_index(table_name, column_name, options = {}) @@ -24,12 +23,6 @@ if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter) end end - def add_index_sql(table_name, column_name, options = {}) - unless options[:opclasses] - __gitlab_add_index_sql(table_name, column_name, options) - end - end - def add_index_options(table_name, column_name, options = {}) if options[:using] && options[:using] == :gin options = options.dup diff --git a/config/initializers/postgresql_cte.rb b/config/initializers/postgresql_cte.rb index 56689bc8e74..68d53c4edbf 100644 --- a/config/initializers/postgresql_cte.rb +++ b/config/initializers/postgresql_cte.rb @@ -94,8 +94,8 @@ module ActiveRecord end end - def build_arel - arel = super() + def build_arel(aliases) + arel = super build_with(arel) if @values[:with] diff --git a/config/initializers/rack_attack_logging.rb b/config/initializers/rack_attack_logging.rb index 338e968cc6c..7eb34bd69e5 100644 --- a/config/initializers/rack_attack_logging.rb +++ b/config/initializers/rack_attack_logging.rb @@ -12,7 +12,7 @@ ActiveSupport::Notifications.subscribe('rack.attack') do |name, start, finish, r fullpath: req.fullpath } - if req.env['rack.attack.matched'] != 'throttle_unauthenticated' + if %w(throttle_authenticated_api throttle_authenticated_web).include? req.env['rack.attack.matched'] user_id = req.env['rack.attack.match_discriminator'] user = User.find_by(id: user_id) diff --git a/config/initializers/rack_timeout.rb b/config/initializers/rack_timeout.rb index 58f46b55725..246cf3482a4 100644 --- a/config/initializers/rack_timeout.rb +++ b/config/initializers/rack_timeout.rb @@ -14,8 +14,8 @@ if defined?(::Puma) && !Rails.env.test? Gitlab::Application.configure do |config| config.middleware.insert_before(Rack::Runtime, Rack::Timeout, - service_timeout: 60, - wait_timeout: 90) + service_timeout: ENV.fetch('GITLAB_RAILS_RACK_TIMEOUT', 60).to_i, + wait_timeout: ENV.fetch('GITLAB_RAILS_WAIT_TIMEOUT', 90).to_i) end observer = Gitlab::Cluster::RackTimeoutObserver.new diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 7b69cf11288..166fb9b6916 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -72,7 +72,7 @@ Sidekiq.configure_server do |config| cron_jobs[k]['class'] = cron_jobs[k].delete('job_class') else cron_jobs.delete(k) - Rails.logger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.") + Rails.logger.error("Invalid cron_jobs config key: '#{k}'. Check your gitlab config file.") # rubocop:disable Gitlab/RailsLogger end end Sidekiq::Cron::Job.load_from_hash! cron_jobs @@ -83,7 +83,20 @@ Sidekiq.configure_server do |config| Rails.application.config.database_configuration[Rails.env] db_config['pool'] = Sidekiq.options[:concurrency] ActiveRecord::Base.establish_connection(db_config) - Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{ActiveRecord::Base.connection.pool.instance_variable_get('@size')}") + Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{ActiveRecord::Base.connection.pool.instance_variable_get('@size')}") # rubocop:disable Gitlab/RailsLogger + + Gitlab.ee do + Gitlab::Mirror.configure_cron_job! + + Gitlab::Geo.configure_cron_jobs! + + if Gitlab::Geo.geo_database_configured? + Rails.configuration.geo_database['pool'] = Sidekiq.options[:concurrency] + Geo::TrackingBase.establish_connection(Rails.configuration.geo_database) + + Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{Geo::TrackingBase.connection_pool.size} (Geo tracking database)") # rubocop:disable Gitlab/RailsLogger + end + end # Avoid autoload issue such as 'Mail::Parsers::AddressStruct' # https://github.com/mikel/mail/issues/912#issuecomment-214850355 diff --git a/config/initializers/sidekiq_cluster.rb b/config/initializers/sidekiq_cluster.rb new file mode 100644 index 00000000000..baa7495aa29 --- /dev/null +++ b/config/initializers/sidekiq_cluster.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +if ENV['ENABLE_SIDEKIQ_CLUSTER'] && Gitlab.ee? + Thread.new do + Thread.current.abort_on_exception = true + + parent = Process.ppid + + loop do + sleep(5) + + # In cluster mode it's possible that the master process is SIGKILL'd. In + # this case the parent PID changes and we need to terminate ourselves. + if Process.ppid != parent + Process.kill(:TERM, Process.pid) + break + end + end + end +end diff --git a/config/prometheus/cluster_metrics.yml b/config/prometheus/cluster_metrics.yml new file mode 100644 index 00000000000..3df76b0974f --- /dev/null +++ b/config/prometheus/cluster_metrics.yml @@ -0,0 +1,63 @@ +- group: Cluster Health + priority: 1 + metrics: + - title: "CPU Usage" + y_label: "CPU" + required_metrics: ['container_cpu_usage_seconds_total'] + weight: 1 + queries: + - query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)' + label: Usage + unit: "cores" + appearance: + line: + width: 2 + area: + opacity: 0 + - query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})' + label: Requested + unit: "cores" + appearance: + line: + width: 2 + area: + opacity: 0 + - query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})' + label: Capacity + unit: "cores" + appearance: + line: + type: 'dashed' + width: 2 + area: + opacity: 0 + - title: "Memory usage" + y_label: "Memory" + required_metrics: ['container_memory_usage_bytes'] + weight: 1 + queries: + - query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30' + label: Usage + unit: "GiB" + appearance: + line: + width: 2 + area: + opacity: 0 + - query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30' + label: Requested + unit: "GiB" + appearance: + line: + width: 2 + area: + opacity: 0 + - query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30' + label: Capacity + unit: "GiB" + appearance: + line: + type: 'dashed' + width: 2 + area: + opacity: 0 diff --git a/config/pseudonymizer.yml b/config/pseudonymizer.yml new file mode 100644 index 00000000000..1d85ac1db45 --- /dev/null +++ b/config/pseudonymizer.yml @@ -0,0 +1,475 @@ +tables: + approvals: + whitelist: + - id + - merge_request_id + - user_id + - created_at + - updated_at + approver_groups: + whitelist: + - id + - target_type + - group_id + - created_at + - updated_at + board_assignees: + whitelist: + - id + - board_id + - assignee_id + board_labels: + whitelist: + - id + - board_id + - label_id + boards: + whitelist: + - id + - project_id + - created_at + - updated_at + - milestone_id + - group_id + - weight + epic_issues: + whitelist: + - id + - epic_id + - issue_id + - relative_position + epic_metrics: + whitelist: + - id + - epic_id + - created_at + - updated_at + epics: + whitelist: + - id + - milestone_id + - group_id + - author_id + - assignee_id + - iid + - updated_by_id + - last_edited_by_id + - lock_version + - start_date + - end_date + - last_edited_at + - created_at + - updated_at + - title + - description + issue_assignees: + whitelist: + - user_id + - issue_id + issue_links: + whitelist: + - id + - source_id + - target_id + - created_at + - updated_at + issue_metrics: + whitelist: + - id + - issue_id + - first_mentioned_in_commit_at + - first_associated_with_milestone_at + - first_added_to_board_at + - created_at + - updated_at + issues: + whitelist: + - id + - title + - author_id + - project_id + - created_at + - confidential + - updated_at + - description + - milestone_id + - state + - updated_by_id + - weight + - due_date + - moved_to_id + - lock_version + - time_estimate + - last_edited_at + - last_edited_by_id + - discussion_locked + - closed_at + label_links: + whitelist: + - id + - label_id + - target_id + - target_type + - created_at + - updated_at + label_priorities: + whitelist: + - id + - project_id + - label_id + - priority + - created_at + - updated_at + labels: + whitelist: + - id + - title + - color + - project_id + - created_at + - updated_at + - template + - type + - group_id + licenses: + whitelist: + - id + - created_at + - updated_at + merge_request_diffs: + whitelist: + - id + - state + - merge_request_id + - created_at + - updated_at + - base_commit_sha + - real_size + - head_commit_sha + - start_commit_sha + - commits_count + merge_request_metrics: + whitelist: + - id + - merge_request_id + - latest_build_started_at + - latest_build_finished_at + - first_deployed_to_production_at + - merged_at + - created_at + - updated_at + - pipeline_id + - merged_by_id + - latest_closed_by_id + - latest_closed_at + merge_requests: + whitelist: + - id + - target_branch + - source_branch + - source_project_id + - author_id + - assignee_id + - created_at + - updated_at + - milestone_id + - state + - merge_status + - target_project_id + - updated_by_id + - merge_error + - merge_params + - merge_when_pipeline_succeeds + - merge_user_id + - approvals_before_merge + - lock_version + - time_estimate + - squash + - last_edited_at + - last_edited_by_id + - head_pipeline_id + - discussion_locked + - latest_merge_request_diff_id + - allow_maintainer_to_push + merge_requests_closing_issues: + whitelist: + - id + - merge_request_id + - issue_id + - created_at + - updated_at + milestones: + whitelist: + - id + - project_id + - due_date + - created_at + - updated_at + - state + - start_date + - group_id + namespace_statistics: + whitelist: + - id + - namespace_id + - shared_runners_seconds + - shared_runners_seconds_last_reset + namespaces: + whitelist: + - id + - name + - path + - owner_id + - created_at + - updated_at + - type + - avatar + - membership_lock + - share_with_group_lock + - visibility_level + - request_access_enabled + - ldap_sync_status + - ldap_sync_error + - ldap_sync_last_update_at + - ldap_sync_last_successful_update_at + - ldap_sync_last_sync_at + - lfs_enabled + - parent_id + - shared_runners_minutes_limit + - repository_size_limit + - require_two_factor_authentication + - two_factor_grace_period + - plan_id + - project_creation_level + members: + whitelist: + - id + - access_level + - source_id + - source_type + - user_id + - notification_level + - type + - created_by_id + - invite_email + - invite_accepted_at + - requested_at + - expires_at + - ldap + - override + notification_settings: + whitelist: + - id + - user_id + - source_id + - source_type + - level + - created_at + - updated_at + - new_note + - new_issue + - reopen_issue + - close_issue + - reassign_issue + - new_merge_request + - reopen_merge_request + - close_merge_request + - reassign_merge_request + - merge_merge_request + - failed_pipeline + - success_pipeline + project_authorizations: + whitelist: + - user_id + - project_id + - access_level + project_auto_devops: + whitelist: + - id + - project_id + - created_at + - updated_at + - enabled + project_custom_attributes: + whitelist: + - id + - created_at + - updated_at + - project_id + - key + - value + project_features: + whitelist: + - id + - project_id + - merge_requests_access_level + - issues_access_level + - wiki_access_level + - snippets_access_level + - builds_access_level + - created_at + - updated_at + - repository_access_level + project_group_links: + whitelist: + - id + - project_id + - group_id + - created_at + - updated_at + - group_access + - expires_at + project_import_data: + whitelist: + - id + - project_id + project_mirror_data: + whitelist: + - id + - project_id + - retry_count + - last_update_started_at + - last_update_scheduled_at + - next_execution_timestamp + project_repository_states: + whitelist: + - id + - project_id + - repository_verification_checksum + - wiki_verification_checksum + - last_repository_verification_failure + - last_wiki_verification_failure + project_statistics: + whitelist: + - id + - project_id + - namespace_id + - commit_count + - storage_size + - repository_size + - lfs_objects_size + - build_artifacts_size + - shared_runners_seconds + - shared_runners_seconds_last_reset + projects: + whitelist: + - id + - name + - path + - description + - created_at + - updated_at + - creator_id + - namespace_id + - last_activity_at + - import_url + - visibility_level + - archived + - avatar + - merge_requests_template + - star_count + - merge_requests_rebase_enabled + - import_type + - import_source + - approvals_before_merge + - reset_approvals_on_push + - merge_requests_ff_only_enabled + - issues_template + - mirror + - mirror_user_id + - shared_runners_enabled + - build_coverage_regex + - build_allow_git_fetch + - build_timeout + - mirror_trigger_builds + - pending_delete + - public_builds + - last_repository_check_failed + - last_repository_check_at + - container_registry_enabled + - only_allow_merge_if_pipeline_succeeds + - has_external_issue_tracker + - repository_storage + - repository_read_only + - request_access_enabled + - has_external_wiki + - ci_config_path + - lfs_enabled + - only_allow_merge_if_all_discussions_are_resolved + - repository_size_limit + - printing_merge_request_link_enabled + - auto_cancel_pending_pipelines + - service_desk_enabled + - delete_error + - last_repository_updated_at + - disable_overriding_approvers_per_merge_request + - storage_version + - resolve_outdated_diff_discussions + - remote_mirror_available_overridden + - only_mirror_protected_branches + - pull_mirror_available_overridden + - mirror_overwrites_diverged_branches + - external_authorization_classification_label + subscriptions: + whitelist: + - id + - user_id + - subscribable_id + - subscribable_type + - subscribed + - created_at + - updated_at + - project_id + users: + whitelist: + - id + - remember_created_at + - sign_in_count + - current_sign_in_at + - last_sign_in_at + - current_sign_in_ip + - last_sign_in_ip + - created_at + - updated_at + - admin + - projects_limit + - failed_attempts + - locked_at + - can_create_group + - can_create_team + - state + - color_scheme_id + - password_expires_at + - created_by_id + - last_credential_check_at + - avatar + - confirmed_at + - confirmation_sent_at + - unconfirmed_email + - hide_no_ssh_key + - website_url + - admin_email_unsubscribed_at + - notification_email + - hide_no_password + - password_automatically_set + - location + - public_email + - dashboard + - project_view + - consumed_timestep + - layout + - hide_project_limit + - note + - otp_grace_period_started_at + - external + - organization + - auditor + - require_two_factor_authentication_from_group + - two_factor_grace_period + - ghost + - last_activity_on + - notified_of_own_activity + - bot_type + - preferred_language + - theme_id + diff --git a/config/routes.rb b/config/routes.rb index a42fc037227..d208df62b4a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -82,7 +82,11 @@ Rails.application.routes.draw do resources :issues, only: [:index, :create, :update] end - resources :issues, module: :boards, only: [:index, :update] + resources :issues, module: :boards, only: [:index, :update] do + collection do + put :bulk_move, format: :json + end + end Gitlab.ee do resources :users, module: :boards, only: [:index] @@ -106,6 +110,22 @@ Rails.application.routes.draw do draw :jira_connect end + Gitlab.ee do + constraints(::Constraints::FeatureConstrainer.new(:analytics)) do + draw :analytics + end + end + + if ENV['GITLAB_CHAOS_SECRET'] || Rails.env.development? + resource :chaos, only: [] do + get :leakmem + get :cpu_spin + get :db_spin + get :sleep + get :kill + end + end + if ENV['GITLAB_ENABLE_CHAOS_ENDPOINTS'] get '/chaos/leakmem' => 'chaos#leakmem' get '/chaos/cpuspin' => 'chaos#cpuspin' diff --git a/config/routes/api.rb b/config/routes/api.rb index 3ba9176d943..d55bbdf6776 100644 --- a/config/routes/api.rb +++ b/config/routes/api.rb @@ -1,7 +1,5 @@ -constraints(::Constraints::FeatureConstrainer.new(:graphql, default_enabled: true)) do - post '/api/graphql', to: 'graphql#execute' - mount GraphiQL::Rails::Engine, at: '/-/graphql-explorer', graphql_path: '/api/graphql' -end +post '/api/graphql', to: 'graphql#execute' +mount GraphiQL::Rails::Engine, at: '/-/graphql-explorer', graphql_path: '/api/graphql' -::API::API.logger Rails.logger +::API::API.logger Rails.logger # rubocop:disable Gitlab/RailsLogger mount ::API::API => '/' diff --git a/config/settings.rb b/config/settings.rb index 6df2132332c..da459afcce2 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -62,6 +62,31 @@ class Settings < Settingslogic (base_url(gitlab) + [gitlab.relative_url_root]).join('') end + def kerberos_protocol + kerberos.https ? "https" : "http" + end + + def kerberos_port + kerberos.use_dedicated_port ? kerberos.port : gitlab.port + end + + # Curl expects username/password for authentication. However when using GSS-Negotiate not credentials should be needed. + # By inserting in the Kerberos dedicated URL ":@", we give to curl an empty username and password and GSS auth goes ahead + # Known bug reported in http://sourceforge.net/p/curl/bugs/440/ and http://curl.haxx.se/docs/knownbugs.html + def build_gitlab_kerberos_url + [ + kerberos_protocol, + "://:@", + gitlab.host, + ":#{kerberos_port}", + gitlab.relative_url_root + ].join('') + end + + def alternative_gitlab_kerberos_url? + kerberos.enabled && (build_gitlab_kerberos_url != build_gitlab_url) + end + # check that values in `current` (string or integer) is a contant in `modul`. def verify_constant_array(modul, current, default) values = default || [] diff --git a/config/webpack.config.js b/config/webpack.config.js index 19b48845305..cd793743eb7 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -322,7 +322,10 @@ module.exports = { }), new webpack.DefinePlugin({ + // This one is used to define window.gon.ee and other things properly in tests: 'process.env.IS_GITLAB_EE': JSON.stringify(IS_EE), + // This one is used to check against "EE" properly in application code + IS_EE: IS_EE ? 'window.gon && window.gon.ee' : JSON.stringify(false), }), ].filter(Boolean), @@ -341,6 +344,8 @@ module.exports = { devtool: NO_SOURCEMAPS ? false : devtool, - // sqljs requires fs - node: { fs: 'empty' }, + node: { + fs: 'empty', // sqljs requires fs + setImmediate: false, + }, }; |