diff options
Diffstat (limited to 'config')
74 files changed, 2547 insertions, 258 deletions
diff --git a/config/README.md b/config/README.md index 0a5ea2424e0..2778d0d4f02 100644 --- a/config/README.md +++ b/config/README.md @@ -19,4 +19,132 @@ an ERB file and then loads the resulting YML as its configuration. This file is called `resque.yml` for historical reasons. We are **NOT** using Resque at the moment. It is used to specify Redis configuration -values instead. +values when a single database instance of Redis is desired. + +# Advanced Redis configuration files + +In more advanced configurations of Redis key-value storage, it is desirable +to separate the keys by lifecycle and intended use to ease provisioning and +management of scalable Redis clusters. + +These settings provide routing and other configuration data (such as sentinel, +persistence policies, and other Redis customization) for connections +to Redis single instances, Redis sentinel, and Redis clusters. + +If desired, the routing URL provided by these settings can be used with: +1. Unix Socket + 1. named socket for each Redis instance desired. + 2. `database number` for each Redis instance desired. +2. TCP Socket + 1. `host name` or IP for each Redis instance desired + 2. TCP port number for each Redis instance desired + 3. `database number` for each Redis instance desired + +## Example URL attribute formats for GitLab Redis `.yml` configuration files +* Unix Socket, default Redis database (0) + * `url: unix:/path/to/redis.sock` + * `url: unix:/path/to/redis.sock?db=` +* Unix Socket, Redis database 44 + * `url: unix:/path/to/redis.sock?db=44` + * `url: unix:/path/to/redis.sock?extra=foo&db=44` +* TCP Socket for Redis on localhost, port 6379, database 33 + * `url: redis://:mynewpassword@localhost:6379/33` +* TCP Socket for Redis on remote host `myserver`, port 6379, database 33 + * `url: redis://:mynewpassword@myserver:6379/33` + +## redis.cache.yml + +If configured, `redis.cache.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for `Rails.cache` and other volatile non-persistent data which enhances +the performance of GitLab. +Settings here can be overridden by the environment variable +`GITLAB_REDIS_CACHE_CONFIG_FILE` which provides +an alternate location for configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `cache` is: +1. URL from a configuration file pointed to by the +`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable +2. URL from `redis.cache.yml` +3. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. URL from `resque.yml` +5. `redis://localhost:6380` + +The order of precedence for all other configuration settings for `cache` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable +2. the configuration file `redis.cache.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` + +## redis.queues.yml + +If configured, `redis.queues.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for clients of `::Gitlab::Redis::Queues`. +These queues are intended to be the foundation +of reliable inter-process communication between modules, whether on the same +host node, or within a cluster. The primary clients of the queues are +SideKiq, Mailroom, CI Runner, Workhorse, and push services. Settings here can +be overridden by the environment variable +`GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for +configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `queues` is: +1. URL from a configuration file pointed to by the +`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable +2. URL from `redis.queues.yml` +3. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. URL from `resque.yml` +5. `redis://localhost:6381` + +The order of precedence for all other configuration settings for `queues` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable +2. the configuration file `redis.queues.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` + +## redis.shared_state.yml + +If configured, `redis.shared_state.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for clients of `::Gitlab::Redis::SharedState` such as session state, +and rate limiting. +Settings here can be overridden by the environment variable +`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides +an alternate location for configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `shared_state` is: +1. URL from a configuration file pointed to by the +`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable +2. URL from `redis.shared_state.yml` +3. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. URL from `resque.yml` +5. `redis://localhost:6382` + +The order of precedence for all other configuration settings for `shared_state` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable +2. the configuration file `redis.shared_state.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` + diff --git a/config/application.rb b/config/application.rb index f2ecc4ce77c..f69dab4de39 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,7 +6,9 @@ Bundler.require(:default, Rails.env) module Gitlab class Application < Rails::Application - require_dependency Rails.root.join('lib/gitlab/redis') + require_dependency Rails.root.join('lib/gitlab/redis/cache') + require_dependency Rails.root.join('lib/gitlab/redis/queues') + require_dependency Rails.root.join('lib/gitlab/redis/shared_state') require_dependency Rails.root.join('lib/gitlab/request_context') # Settings in config/environments/* take precedence over those specified here. @@ -21,13 +23,13 @@ module Gitlab # https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/engine.rb#L687 # This is a nice reference article on autoloading/eager loading: # http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload - config.eager_load_paths.push(*%W(#{config.root}/lib - #{config.root}/app/models/ci + config.eager_load_paths.push(*%W[#{config.root}/lib #{config.root}/app/models/hooks #{config.root}/app/models/members #{config.root}/app/models/project_services #{config.root}/app/workers/concerns - #{config.root}/app/services/concerns)) + #{config.root}/app/services/concerns + #{config.root}/app/finders/concerns]) config.generators.templates.push("#{config.root}/generator_templates") @@ -40,6 +42,9 @@ module Gitlab # config.i18n.default_locale = :de config.i18n.enforce_available_locales = false + # Translation for AR attrs is not working well for POROs like WikiPage + config.gettext_i18n_rails.use_for_active_record_attributes = false + # Configure the default encoding used in templates for Ruby 1.9. config.encoding = "utf-8" @@ -63,6 +68,7 @@ module Gitlab hook import_url incoming_email_token + rss_token key otp_attempt password @@ -102,8 +108,12 @@ module Gitlab config.assets.precompile << "katex.css" config.assets.precompile << "katex.js" config.assets.precompile << "xterm/xterm.css" + config.assets.precompile << "performance_bar.css" config.assets.precompile << "lib/ace.js" config.assets.precompile << "vendor/assets/fonts/*" + config.assets.precompile << "test.css" + config.assets.precompile << "new_nav.css" + config.assets.precompile << "new_sidebar.css" # Version of your assets, change this if you want to expire all your assets config.assets.version = '1.0' @@ -134,15 +144,15 @@ module Gitlab end end - # Use Redis caching across all environments - redis_config_hash = Gitlab::Redis.params - redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE - redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever + # Use caching across all environments + caching_config_hash = Gitlab::Redis::Cache.params + caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE + caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever if Sidekiq.server? # threaded context - redis_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 - redis_config_hash[:pool_timeout] = 1 + caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 + caching_config_hash[:pool_timeout] = 1 end - config.cache_store = :redis_store, redis_config_hash + config.cache_store = :redis_store, caching_config_hash config.active_record.raise_in_transactional_callbacks = true @@ -155,5 +165,27 @@ module Gitlab config.generators do |g| g.factory_girl false end + + config.after_initialize do + Rails.application.reload_routes! + + project_url_helpers = Module.new do + extend ActiveSupport::Concern + + Gitlab::Application.routes.named_routes.helper_names.each do |name| + next unless name.include?('namespace_project') + + define_method(name.sub('namespace_project', 'project')) do |project, *args| + send(name, project&.namespace, project, *args) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + # We add the MilestonesRoutingHelper because we know that this does not + # conflict with the methods defined in `project_url_helpers`, and we want + # these methods available in the same places. + Gitlab::Routing.add_helpers(project_url_helpers) + Gitlab::Routing.add_helpers(MilestonesRoutingHelper) + end end end diff --git a/config/database.yml.mysql b/config/database.yml.mysql index db1b712d3bc..eb71d3f5fe1 100644 --- a/config/database.yml.mysql +++ b/config/database.yml.mysql @@ -42,3 +42,4 @@ test: &test password: # host: localhost # socket: /tmp/mysql.sock + prepared_statements: false diff --git a/config/database.yml.postgresql b/config/database.yml.postgresql index c517a4c0cb8..4b30982fe82 100644 --- a/config/database.yml.postgresql +++ b/config/database.yml.postgresql @@ -46,3 +46,4 @@ test: &test username: postgres password: # host: localhost + prepared_statements: false diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml index 59c7050a14d..ca5b941aebf 100644 --- a/config/dependency_decisions.yml +++ b/config/dependency_decisions.yml @@ -398,3 +398,9 @@ :why: https://github.com/remy/undefsafe/blob/master/LICENSE :versions: [] :when: 2017-04-10 06:30:00.002555000 Z +- - :approve + - thunky + - :who: Mike Greiling + :why: https://github.com/mafintosh/thunky/blob/master/README.md#license + :versions: [] + :when: 2017-08-07 05:56:09.907045000 Z diff --git a/config/environments/production.rb b/config/environments/production.rb index a9d8ac4b6d4..c5cbfcf64cf 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -16,7 +16,7 @@ Rails.application.configure do # config.assets.css_compressor = :sass # Don't fallback to assets pipeline if a precompiled asset is missed - config.assets.compile = true + config.assets.compile = false # Generate digests for assets URLs config.assets.digest = true @@ -50,7 +50,7 @@ Rails.application.configure do # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) # Enable serving of images, stylesheets, and JavaScripts from an asset server - # config.action_controller.asset_host = "http://assets.example.com" + config.action_controller.asset_host = ENV['GITLAB_CDN_HOST'] if ENV['GITLAB_CDN_HOST'].present? # Precompile additional assets (application.js, application.css, and all non-JS/CSS are already added) # config.assets.precompile += %w( search.js ) diff --git a/config/environments/test.rb b/config/environments/test.rb index c3b788c038e..278144b8943 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -43,4 +43,9 @@ Rails.application.configure do config.cache_store = :null_store config.active_job.queue_adapter = :test + + if ENV['CI'] && !ENV['RAILS_ENABLE_TEST_LOG'] + config.logger = ActiveSupport::TaggedLogging.new(Logger.new(nil)) + config.log_level = :fatal + end end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index c2eaf263937..25285525846 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -169,7 +169,7 @@ production: &base ## Gravatar ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html gravatar: - # gravatar urls: possible placeholders: %{hash} %{size} %{email} + # gravatar urls: possible placeholders: %{hash} %{size} %{email} %{username} # plain_url: "http://..." # default: http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon # ssl_url: "https://..." # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon @@ -181,8 +181,8 @@ production: &base stuck_ci_jobs_worker: cron: "0 * * * *" # Execute scheduled triggers - trigger_schedule_worker: - cron: "0 */12 * * *" + pipeline_schedule_worker: + cron: "19 * * * *" # Remove expired build artifacts expire_build_artifacts_worker: cron: "50 * * * *" @@ -228,7 +228,8 @@ production: &base # ========================== ## LDAP settings - # You can inspect a sample of the LDAP users with login access by running: + # You can test connections and inspect a sample of the LDAP users with login + # access by running: # bundle exec rake gitlab:ldap:check RAILS_ENV=production ldap: enabled: false @@ -251,13 +252,45 @@ production: &base # Example: 'Paris' or 'Acme, Ltd.' label: 'LDAP' + # Example: 'ldap.mydomain.com' host: '_your_ldap_server' - port: 389 - uid: 'sAMAccountName' - method: 'plain' # "tls" or "ssl" or "plain" + # This port is an example, it is sometimes different but it is always an integer and not a string + port: 389 # usually 636 for SSL + uid: 'sAMAccountName' # This should be the attribute, not the value that maps to uid. + + # Examples: 'america\\momo' or 'CN=Gitlab Git,CN=Users,DC=mydomain,DC=com' bind_dn: '_the_full_dn_of_the_user_you_will_bind_with' password: '_the_password_of_the_bind_user' + # Encryption method. The "method" key is deprecated in favor of + # "encryption". + # + # Examples: "start_tls" or "simple_tls" or "plain" + # + # Deprecated values: "tls" was replaced with "start_tls" and "ssl" was + # replaced with "simple_tls". + # + encryption: 'plain' + + # Enables SSL certificate verification if encryption method is + # "start_tls" or "simple_tls". (Defaults to false for backward- + # compatibility) + verify_certificates: false + + # Specifies the path to a file containing a PEM-format CA certificate, + # e.g. if you need to use an internal CA. + # + # Example: '/etc/ca.pem' + # + ca_file: '' + + # Specifies the SSL version for OpenSSL to use, if the OpenSSL default + # is not appropriate. + # + # Example: 'TLSv1_1' + # + ssl_version: '' + # Set a timeout, in seconds, for LDAP queries. This helps avoid blocking # a request if the LDAP server becomes unresponsive. # A value of 0 means there is no timeout. @@ -286,17 +319,20 @@ production: &base # Base where we can search for users # - # Ex. ou=People,dc=gitlab,dc=example + # Ex. 'ou=People,dc=gitlab,dc=example' or 'DC=mydomain,DC=com' # base: '' # Filter LDAP users # - # Format: RFC 4515 http://tools.ietf.org/search/rfc4515 + # Format: RFC 4515 https://tools.ietf.org/search/rfc4515 # Ex. (employeeType=developer) # # Note: GitLab does not support omniauth-ldap's custom filter syntax. # + # Example for getting only specific users: + # '(&(objectclass=user)(|(samaccountname=momo)(samaccountname=toto)))' + # user_filter: '' # LDAP attributes that GitLab will use to create an account for the LDAP user. @@ -337,6 +373,10 @@ production: &base # showing GitLab's sign-in page (default: show the GitLab sign-in page) # auto_sign_in_with_provider: saml + # Sync user's email address from the specified Omniauth provider every time the user logs + # in (default: nil). And consequently make this field read-only. + # sync_email_from_provider: cas3 + # CAUTION! # This allows users to login without having a user account first. Define the allowed providers # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. @@ -379,13 +419,13 @@ production: &base # service_validate_url: '/cas/p3/serviceValidate', # logout_url: '/cas/logout'} } # - { name: 'authentiq', - # # for client credentials (client ID and secret), go to https://www.authentiq.com/ + # # for client credentials (client ID and secret), go to https://www.authentiq.com/developers # app_id: 'YOUR_CLIENT_ID', # app_secret: 'YOUR_CLIENT_SECRET', # args: { # scope: 'aq:name email~rs address aq:push' - # # redirect_uri parameter is optional except when 'gitlab.host' in this file is set to 'localhost' - # # redirect_uri: 'YOUR_REDIRECT_URI' + # # callback_url parameter is optional except when 'gitlab.host' in this file is set to 'localhost' + # # callback_url: 'YOUR_CALLBACK_URL' # } # } # - { name: 'github', @@ -446,10 +486,10 @@ production: &base # Gitaly settings gitaly: - # This setting controls whether GitLab uses Gitaly (new component - # introduced in 9.0). Eventually Gitaly use will become mandatory and - # this option will disappear. - enabled: false + # Default Gitaly authentication token. Can be overriden per storage. Can + # be left blank when Gitaly is running locally on a Unix socket, which + # is the normal way to deploy Gitaly. + token: # # 4. Advanced settings @@ -465,6 +505,12 @@ production: &base default: path: /home/git/repositories/ gitaly_address: unix:/home/git/gitlab/tmp/sockets/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port) + # gitaly_token: 'special token' # Optional: override global gitaly.token for this storage. + failure_count_threshold: 10 # number of failures before stopping attempts + failure_wait_time: 30 # Seconds after an access failure before allowing access again + failure_reset_time: 1800 # Time in seconds to expire failures + storage_timeout: 5 # Time in seconds to wait before aborting a storage access attempt + ## Backup settings backup: @@ -502,6 +548,9 @@ production: &base upload_pack: true receive_pack: true + # Git import/fetch timeout + # git_timeout: 800 + # If you use non-standard ssh port you need to specify it # ssh_port: 22 @@ -532,6 +581,21 @@ production: &base # host: localhost # port: 3808 + ## Monitoring + # Built in monitoring settings + monitoring: + # Time between sampling of unicorn socket metrics, in seconds + # unicorn_sampler_interval: 10 + # IP whitelist to access monitoring endpoints + ip_whitelist: + - 127.0.0.0/8 + + # Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics + sidekiq_exporter: + # enabled: true + # address: localhost + # port: 3807 + # # 5. Extra customization # ========================== @@ -585,8 +649,16 @@ test: default: path: tmp/tests/repositories/ gitaly_address: unix:tmp/tests/gitaly/gitaly.socket + failure_count_threshold: 999999 + failure_wait_time: 0 + storage_timeout: 30 + broken: + path: tmp/tests/non-existent-repositories + gitaly_address: unix:tmp/tests/gitaly/gitaly.socket + gitaly: enabled: true + token: secret backup: path: tmp/tests/backups gitlab_shell: @@ -602,6 +674,52 @@ test: title: "JIRA" url: https://sample_company.atlassian.net project_key: PROJECT + + omniauth: + enabled: true + allow_single_sign_on: true + external_providers: [] + + providers: + - { name: 'cas3', + label: 'cas3', + args: { url: 'https://sso.example.com', + disable_ssl_verification: false, + login_url: '/cas/login', + service_validate_url: '/cas/p3/serviceValidate', + logout_url: '/cas/logout'} } + - { name: 'github', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + url: "https://github.com/", + verify_ssl: false, + args: { scope: 'user:email' } } + - { name: 'bitbucket', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET' } + - { name: 'gitlab', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { scope: 'api' } } + - { name: 'google_oauth2', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { access_type: 'offline', approval_prompt: '' } } + - { name: 'facebook', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET' } + - { name: 'twitter', + app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET' } + - { name: 'auth0', + args: { + client_id: 'YOUR_AUTH0_CLIENT_ID', + client_secret: 'YOUR_AUTH0_CLIENT_SECRET', + namespace: 'YOUR_AUTH0_DOMAIN' } } + - { name: 'authentiq', + app_id: 'YOUR_CLIENT_ID', + app_secret: 'YOUR_CLIENT_SECRET', + args: { scope: 'aq:name email~rs address aq:push' } } ldap: enabled: false servers: @@ -610,7 +728,7 @@ test: host: 127.0.0.1 port: 3890 uid: 'uid' - method: 'plain' # "tls" or "ssl" or "plain" + encryption: 'plain' # "start_tls" or "simple_tls" or "plain" base: 'dc=example,dc=com' user_filter: '' group_base: 'ou=groups,dc=example,dc=com' diff --git a/config/initializers/acts_as_taggable.rb b/config/initializers/0_acts_as_taggable.rb index c564c0cab11..50dc47673ab 100644 --- a/config/initializers/acts_as_taggable.rb +++ b/config/initializers/0_acts_as_taggable.rb @@ -3,3 +3,7 @@ ActsAsTaggableOn.strict_case_match = true # tags_counter enables caching count of tags which results in an update whenever a tag is added or removed # since the count is not used anywhere its better performance wise to disable this cache ActsAsTaggableOn.tags_counter = false + +# validate that counter cache is disabled +raise "Counter cache is not disabled" if + ActsAsTaggableOn::Tagging.reflections["tag"].options[:counter_cache] diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 7a8f00f11b2..38ade18bdc0 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -1,3 +1,5 @@ +# rubocop:disable GitlabSecurity/PublicSend + require_dependency Rails.root.join('lib/gitlab') # Load Gitlab as soon as possible class Settings < Settingslogic @@ -71,7 +73,7 @@ class Settings < Settingslogic # check that `current` (string or integer) is a contant in `modul`. def verify_constant(modul, current, default) - constant = modul.constants.find{ |name| modul.const_get(name) == current } + constant = modul.constants.find { |name| modul.const_get(name) == current } value = constant.nil? ? default : modul.const_get(constant) if current.is_a? String value = modul.const_get(current.upcase) rescue default @@ -145,6 +147,24 @@ if Settings.ldap['enabled'] || Rails.env.test? server['attributes'] = {} if server['attributes'].nil? server['provider_name'] ||= "ldap#{key}".downcase server['provider_class'] = OmniAuth::Utils.camelize(server['provider_name']) + + # For backwards compatibility + server['encryption'] ||= server['method'] + server['encryption'] = 'simple_tls' if server['encryption'] == 'ssl' + server['encryption'] = 'start_tls' if server['encryption'] == 'tls' + + # Certificates are not verified for backwards compatibility. + # This default should be flipped to true in 9.5. + if server['verify_certificates'].nil? + server['verify_certificates'] = false + + message = <<-MSG.strip_heredoc + LDAP SSL certificate verification is disabled for backwards-compatibility. + Please add the "verify_certificates" option to gitlab.yml for each LDAP + server. Certificate verification will be enabled by default in GitLab 9.5. + MSG + Rails.logger.warn(message) + end end end @@ -156,6 +176,7 @@ Settings.omniauth['external_providers'] = [] if Settings.omniauth['external_prov Settings.omniauth['block_auto_created_users'] = true if Settings.omniauth['block_auto_created_users'].nil? Settings.omniauth['auto_link_ldap_user'] = false if Settings.omniauth['auto_link_ldap_user'].nil? Settings.omniauth['auto_link_saml_user'] = false if Settings.omniauth['auto_link_saml_user'].nil? +Settings.omniauth['sync_email_from_provider'] ||= nil Settings.omniauth['providers'] ||= [] Settings.omniauth['cas3'] ||= Settingslogic.new({}) @@ -204,7 +225,7 @@ Settings.gitlab['default_can_create_group'] = true if Settings.gitlab['default_c Settings.gitlab['host'] ||= ENV['GITLAB_HOST'] || 'localhost' Settings.gitlab['ssh_host'] ||= Settings.gitlab.host Settings.gitlab['https'] = false if Settings.gitlab['https'].nil? -Settings.gitlab['port'] ||= Settings.gitlab.https ? 443 : 80 +Settings.gitlab['port'] ||= ENV['GITLAB_PORT'] || (Settings.gitlab.https ? 443 : 80) Settings.gitlab['relative_url_root'] ||= ENV['RAILS_RELATIVE_URL_ROOT'] || '' Settings.gitlab['protocol'] ||= Settings.gitlab.https ? "https" : "http" Settings.gitlab['email_enabled'] ||= true if Settings.gitlab['email_enabled'].nil? @@ -222,7 +243,7 @@ rescue ArgumentError # no user configured end Settings.gitlab['time_zone'] ||= nil Settings.gitlab['signup_enabled'] ||= true if Settings.gitlab['signup_enabled'].nil? -Settings.gitlab['signin_enabled'] ||= true if Settings.gitlab['signin_enabled'].nil? +Settings.gitlab['password_authentication_enabled'] ||= true if Settings.gitlab['password_authentication_enabled'].nil? Settings.gitlab['restricted_visibility_levels'] = Settings.__send__(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], []) Settings.gitlab['username_changing_enabled'] = true if Settings.gitlab['username_changing_enabled'].nil? Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing))(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil? @@ -241,6 +262,7 @@ Settings.gitlab['domain_whitelist'] ||= [] Settings.gitlab['import_sources'] ||= %w[github bitbucket gitlab google_code fogbugz git gitlab_project gitea] Settings.gitlab['trusted_proxies'] ||= [] Settings.gitlab['no_todos_messages'] ||= YAML.load_file(Rails.root.join('config', 'no_todos_messages.yml')) +Settings.gitlab['usage_ping_enabled'] = true if Settings.gitlab['usage_ping_enabled'].nil? # # CI @@ -323,9 +345,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_jobs_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_jobs_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['stuck_ci_jobs_worker']['job_class'] = 'StuckCiJobsWorker' -Settings.cron_jobs['trigger_schedule_worker'] ||= Settingslogic.new({}) -Settings.cron_jobs['trigger_schedule_worker']['cron'] ||= '0 */12 * * *' -Settings.cron_jobs['trigger_schedule_worker']['job_class'] = 'TriggerScheduleWorker' +Settings.cron_jobs['pipeline_schedule_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['pipeline_schedule_worker']['cron'] ||= '19 * * * *' +Settings.cron_jobs['pipeline_schedule_worker']['job_class'] = 'PipelineScheduleWorker' Settings.cron_jobs['expire_build_artifacts_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *' Settings.cron_jobs['expire_build_artifacts_worker']['job_class'] = 'ExpireBuildArtifactsWorker' @@ -367,11 +389,18 @@ Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_random_weekly_time) Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker' -# Every day at 00:30 Settings.cron_jobs['schedule_update_user_activity_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['schedule_update_user_activity_worker']['cron'] ||= '30 0 * * *' Settings.cron_jobs['schedule_update_user_activity_worker']['job_class'] = 'ScheduleUpdateUserActivityWorker' +Settings.cron_jobs['remove_old_web_hook_logs_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['remove_old_web_hook_logs_worker']['cron'] ||= '40 0 * * *' +Settings.cron_jobs['remove_old_web_hook_logs_worker']['job_class'] = 'RemoveOldWebHookLogsWorker' + +Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *' +Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker' + # # GitLab Shell # @@ -386,6 +415,7 @@ Settings.gitlab_shell['ssh_port'] ||= 22 Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.__send__(:build_gitlab_shell_ssh_path_prefix) +Settings.gitlab_shell['git_timeout'] ||= 800 # # Workhorse @@ -409,6 +439,17 @@ end Settings.repositories.storages.values.each do |storage| # Expand relative paths storage['path'] = Settings.absolute(storage['path']) + # Set failure defaults + storage['failure_count_threshold'] ||= 10 + storage['failure_wait_time'] ||= 30 + storage['failure_reset_time'] ||= 1800 + storage['storage_timeout'] ||= 5 + # Set turn strings into numbers + storage['failure_count_threshold'] = storage['failure_count_threshold'].to_i + storage['failure_wait_time'] = storage['failure_wait_time'].to_i + storage['failure_reset_time'] = storage['failure_reset_time'].to_i + # We might want to have a timeout shorter than 1 second. + storage['storage_timeout'] = storage['storage_timeout'].to_f end # @@ -435,10 +476,6 @@ Settings.backup['pg_schema'] = nil Settings.backup['path'] = Settings.absolute(Settings.backup['path'] || "tmp/backups/") Settings.backup['archive_permissions'] ||= 0600 Settings.backup['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'connection' => nil }) -# Convert upload connection settings to use symbol keys, to make Fog happy -if Settings.backup['upload']['connection'] - Settings.backup['upload']['connection'] = Hash[Settings.backup['upload']['connection'].map { |k, v| [k.to_sym, v] }] -end Settings.backup['upload']['multipart_chunk_size'] ||= 104857600 Settings.backup['upload']['encryption'] ||= nil Settings.backup['upload']['storage_class'] ||= nil @@ -477,7 +514,6 @@ Settings.rack_attack.git_basic_auth['bantime'] ||= 1.hour # Gitaly # Settings['gitaly'] ||= Settingslogic.new({}) -Settings.gitaly['enabled'] ||= false # # Webpack settings @@ -489,6 +525,17 @@ Settings.webpack.dev_server['host'] ||= 'localhost' Settings.webpack.dev_server['port'] ||= 3808 # +# Monitoring settings +# +Settings['monitoring'] ||= Settingslogic.new({}) +Settings.monitoring['ip_whitelist'] ||= ['127.0.0.1/8'] +Settings.monitoring['unicorn_sampler_interval'] ||= 10 +Settings.monitoring['sidekiq_exporter'] ||= Settingslogic.new({}) +Settings.monitoring.sidekiq_exporter['enabled'] ||= false +Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost' +Settings.monitoring.sidekiq_exporter['port'] ||= 3807 + +# # Testing settings # if Rails.env.test? diff --git a/config/initializers/5_backend.rb b/config/initializers/5_backend.rb index 2bd159ca7f1..482613dacc9 100644 --- a/config/initializers/5_backend.rb +++ b/config/initializers/5_backend.rb @@ -1,6 +1,8 @@ -required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required) -current_version = Gitlab::VersionInfo.parse(Gitlab::Shell.new.version) +unless Rails.env.test? + required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required) + current_version = Gitlab::VersionInfo.parse(Gitlab::Shell.new.version) -unless current_version.valid? && required_version <= current_version - warn "WARNING: This version of GitLab depends on gitlab-shell #{required_version}, but you're running #{current_version}. Please update gitlab-shell." + unless current_version.valid? && required_version <= current_version + warn "WARNING: This version of GitLab depends on gitlab-shell #{required_version}, but you're running #{current_version}. Please update gitlab-shell." + end end diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb index 9e24f42d284..f8e67ce04c9 100644 --- a/config/initializers/6_validations.rb +++ b/config/initializers/6_validations.rb @@ -7,6 +7,13 @@ def find_parent_path(name, path) Gitlab.config.repositories.storages.detect do |n, rs| name != n && Pathname.new(rs['path']).realpath == parent end +rescue Errno::EIO, Errno::ENOENT => e + warning = "WARNING: couldn't verify #{path} (#{name}). "\ + "If this is an external storage, it might be offline." + message = "#{warning}\n#{e.message}" + Rails.logger.error("#{message}\n\t" + e.backtrace.join("\n\t")) + + nil end def storage_validation_error(message) @@ -29,6 +36,15 @@ def validate_storages_config if !repository_storage.is_a?(Hash) || repository_storage['path'].nil? storage_validation_error("#{name} is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example") end + + %w(failure_count_threshold failure_reset_time storage_timeout).each do |setting| + # Falling back to the defaults is fine! + next if repository_storage[setting].nil? + + unless repository_storage[setting].to_f > 0 + storage_validation_error("`#{setting}` for storage `#{name}` needs to be greater than 0") + end + end end end diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb new file mode 100644 index 00000000000..54c797e0714 --- /dev/null +++ b/config/initializers/7_prometheus_metrics.rb @@ -0,0 +1,18 @@ +require 'prometheus/client' + +Prometheus::Client.configure do |config| + config.logger = Rails.logger + + config.initial_mmap_file_size = 4 * 1024 + config.multiprocess_files_dir = ENV['prometheus_multiproc_dir'] + + if Rails.env.development? || Rails.env.test? + config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir') + end +end + +Sidekiq.configure_server do |config| + config.on(:startup) do + Gitlab::Metrics::SidekiqMetricsExporter.instance.start + end +end diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index ae2ca258df1..af4967521b8 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -1,3 +1,8 @@ -# Make sure we initialize a Redis connection pool before Sidekiq starts -# multi-threaded execution. -Gitlab::Redis.with { nil } +# Make sure we initialize a Redis connection pool before multi-threaded +# execution starts by +# 1. Sidekiq +# 2. Rails.cache +# 3. HTTP clients +Gitlab::Redis::Cache.with { nil } +Gitlab::Redis::Queues.with { nil } +Gitlab::Redis::SharedState.with { nil } diff --git a/config/initializers/8_gitaly.rb b/config/initializers/8_gitaly.rb index 42ec7240b0f..f4f116e67f7 100644 --- a/config/initializers/8_gitaly.rb +++ b/config/initializers/8_gitaly.rb @@ -1,6 +1,6 @@ require 'uri' -# Make sure we initialize our Gitaly channels before Sidekiq starts multi-threaded execution. -if Gitlab.config.gitaly.enabled || Rails.env.test? - Gitlab::GitalyClient.configure_channels +Gitlab.config.repositories.storages.keys.each do |storage| + # Force validation of each address + Gitlab::GitalyClient.address(storage) end diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb index 5e0eefdb154..2aeb94d47cd 100644 --- a/config/initializers/8_metrics.rb +++ b/config/initializers/8_metrics.rb @@ -113,9 +113,22 @@ def instrument_classes(instrumentation) # This is a Rails scope so we have to instrument it manually. instrumentation.instrument_method(Project, :visible_to_user) + + # Needed for https://gitlab.com/gitlab-org/gitlab-ce/issues/34509 + instrumentation.instrument_method(MarkupHelper, :link_to_gfm) + + # Needed for https://gitlab.com/gitlab-org/gitlab-ce/issues/30224#note_32306159 + instrumentation.instrument_instance_method(MergeRequestDiff, :load_commits) end # rubocop:enable Metrics/AbcSize +Gitlab::Metrics::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start + +Gitlab::Application.configure do |config| + # 0 should be Sentry to catch errors in this middleware + config.middleware.insert(1, Gitlab::Metrics::RequestsRackMiddleware) +end + if Gitlab::Metrics.enabled? require 'pathname' require 'influxdb' @@ -151,8 +164,8 @@ if Gitlab::Metrics.enabled? ActiveRecord::Querying.public_instance_methods(false).map(&:to_s) ) - Gitlab::Metrics::Instrumentation. - instrument_class_hierarchy(ActiveRecord::Base) do |klass, method| + Gitlab::Metrics::Instrumentation + .instrument_class_hierarchy(ActiveRecord::Base) do |klass, method| # Instrumenting the ApplicationSetting class can lead to an infinite # loop. Since the data is cached any way we don't really need to # instrument it. @@ -164,6 +177,10 @@ if Gitlab::Metrics.enabled? loc && loc[0].start_with?(models) && method.source =~ regex end end + + # Ability is in app/models, is not an ActiveRecord model, but should still + # be instrumented. + Gitlab::Metrics::Instrumentation.instrument_methods(Ability) end Gitlab::Metrics::Instrumentation.configure do |config| @@ -172,7 +189,7 @@ if Gitlab::Metrics.enabled? GC::Profiler.enable - Gitlab::Metrics::Sampler.new.start + Gitlab::Metrics::InfluxSampler.initialize_instance.start module TrackNewRedisConnections def connect(*args) diff --git a/config/initializers/active_record_array_type_casting.rb b/config/initializers/active_record_array_type_casting.rb new file mode 100644 index 00000000000..d94d592add6 --- /dev/null +++ b/config/initializers/active_record_array_type_casting.rb @@ -0,0 +1,20 @@ +module ActiveRecord + class PredicateBuilder + class ArrayHandler + module TypeCasting + def call(attribute, value) + # This is necessary because by default ActiveRecord does not respect + # custom type definitions (like our `ShaAttribute`) when providing an + # array in `where`, like in `where(commit_sha: [sha1, sha2, sha3])`. + model = attribute.relation&.engine + type = model.user_provided_columns[attribute.name] if model + value = value.map { |value| type.type_cast_for_database(value) } if type + + super(attribute, value) + end + end + + prepend TypeCasting + end + end +end diff --git a/config/initializers/active_record_data_types.rb b/config/initializers/active_record_data_types.rb new file mode 100644 index 00000000000..fef591c397d --- /dev/null +++ b/config/initializers/active_record_data_types.rb @@ -0,0 +1,81 @@ +# ActiveRecord custom data type for storing datetimes with timezone information. +# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11229 + +if Gitlab::Database.postgresql? + require 'active_record/connection_adapters/postgresql_adapter' + + module ActiveRecord::ConnectionAdapters::PostgreSQL::OID + # Add the class `DateTimeWithTimeZone` so we can map `timestamptz` to it. + class DateTimeWithTimeZone < DateTime + def type + :datetime_with_timezone + end + end + end + + module RegisterDateTimeWithTimeZone + # Run original `initialize_type_map` and then register `timestamptz` as a + # `DateTimeWithTimeZone`. + # + # Apparently it does not matter that the original `initialize_type_map` + # aliases `timestamptz` to `timestamp`. + # + # When schema dumping, `timestamptz` columns will be output as + # `t.datetime_with_timezone`. + def initialize_type_map(mapping) + super mapping + + mapping.register_type 'timestamptz' do |_, _, sql_type| + precision = extract_precision(sql_type) + ActiveRecord::ConnectionAdapters::PostgreSQLAdapter::OID::DateTimeWithTimeZone.new(precision: precision) + end + end + end + + class ActiveRecord::ConnectionAdapters::PostgreSQLAdapter + prepend RegisterDateTimeWithTimeZone + + # Add column type `datetime_with_timezone` so we can do this in + # migrations: + # + # add_column(:users, :datetime_with_timezone) + # + NATIVE_DATABASE_TYPES[:datetime_with_timezone] = { name: 'timestamptz' } + end +elsif Gitlab::Database.mysql? + require 'active_record/connection_adapters/mysql2_adapter' + + module RegisterDateTimeWithTimeZone + # Run original `initialize_type_map` and then register `timestamp` as a + # `MysqlDateTimeWithTimeZone`. + # + # When schema dumping, `timestamp` columns will be output as + # `t.datetime_with_timezone`. + def initialize_type_map(mapping) + super mapping + + mapping.register_type(%r(timestamp)i) do |sql_type| + precision = extract_precision(sql_type) + ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter::MysqlDateTimeWithTimeZone.new(precision: precision) + end + end + end + + class ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter + prepend RegisterDateTimeWithTimeZone + + # Add the class `DateTimeWithTimeZone` so we can map `timestamp` to it. + class MysqlDateTimeWithTimeZone < MysqlDateTime + def type + :datetime_with_timezone + end + end + + # Add column type `datetime_with_timezone` so we can do this in + # migrations: + # + # add_column(:users, :datetime_with_timezone) + # + NATIVE_DATABASE_TYPES[:datetime_with_timezone] = { name: 'timestamp' } + end +end diff --git a/config/initializers/ar_monkey_patch.rb b/config/initializers/active_record_locking.rb index 6979f4641b0..150aaa2a8c2 100644 --- a/config/initializers/ar_monkey_patch.rb +++ b/config/initializers/active_record_locking.rb @@ -18,7 +18,7 @@ module ActiveRecord lock_col = self.class.locking_column - previous_lock_value = send(lock_col).to_i + previous_lock_value = send(lock_col).to_i # rubocop:disable GitlabSecurity/PublicSend # This line is added as a patch previous_lock_value = nil if previous_lock_value == '0' || previous_lock_value == 0 @@ -33,7 +33,7 @@ module ActiveRecord affected_rows = relation.where( self.class.primary_key => id, - lock_col => previous_lock_value, + lock_col => previous_lock_value ).update_all( attributes_for_update(attribute_names).map do |name| [name, _read_attribute(name)] @@ -48,7 +48,7 @@ module ActiveRecord # If something went wrong, revert the version. rescue Exception - send(lock_col + '=', previous_lock_value) + send(lock_col + '=', previous_lock_value) # rubocop:disable GitlabSecurity/PublicSend raise end end diff --git a/config/initializers/active_record_mysql_timestamp.rb b/config/initializers/active_record_mysql_timestamp.rb new file mode 100644 index 00000000000..af74c4ff6fb --- /dev/null +++ b/config/initializers/active_record_mysql_timestamp.rb @@ -0,0 +1,30 @@ +# Make sure that MySQL won't try to use CURRENT_TIMESTAMP when the timestamp +# column is NOT NULL. See https://gitlab.com/gitlab-org/gitlab-ce/issues/36405 +# And also: https://bugs.mysql.com/bug.php?id=75098 +# This patch was based on: +# https://github.com/rails/rails/blob/15ef55efb591e5379486ccf53dd3e13f416564f6/activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb#L34-L36 + +if Gitlab::Database.mysql? + require 'active_record/connection_adapters/abstract/schema_creation' + + module MySQLTimestampFix + def add_column_options!(sql, options) + # By default, TIMESTAMP columns are NOT NULL, cannot contain NULL values, + # and assigning NULL assigns the current timestamp. To permit a TIMESTAMP + # column to contain NULL, explicitly declare it with the NULL attribute. + # See http://dev.mysql.com/doc/refman/5.7/en/timestamp-initialization.html + if sql.end_with?('timestamp') && !options[:primary_key] + if options[:null] != false + sql << ' NULL' + elsif options[:column].default.nil? + sql << ' DEFAULT 0' + end + end + + super + end + end + + ActiveRecord::ConnectionAdapters::AbstractAdapter::SchemaCreation + .prepend(MySQLTimestampFix) +end diff --git a/config/initializers/active_record_preloader.rb b/config/initializers/active_record_preloader.rb new file mode 100644 index 00000000000..3b16014f302 --- /dev/null +++ b/config/initializers/active_record_preloader.rb @@ -0,0 +1,15 @@ +module ActiveRecord + module Associations + class Preloader + module NoCommitPreloader + def preloader_for(reflection, owners, rhs_klass) + return NullPreloader if rhs_klass == ::Commit + + super + end + end + + prepend NoCommitPreloader + end + end +end diff --git a/config/initializers/active_record_query_trace.rb b/config/initializers/active_record_query_trace.rb deleted file mode 100644 index 4b3c2803b3b..00000000000 --- a/config/initializers/active_record_query_trace.rb +++ /dev/null @@ -1,5 +0,0 @@ -if ENV['ENABLE_QUERY_TRACE'] - require 'active_record_query_trace' - - ActiveRecordQueryTrace.enabled = 'true' -end diff --git a/config/initializers/active_record_table_definition.rb b/config/initializers/active_record_table_definition.rb new file mode 100644 index 00000000000..8e3a1c7a62f --- /dev/null +++ b/config/initializers/active_record_table_definition.rb @@ -0,0 +1,34 @@ +# ActiveRecord custom method definitions with timezone information. +# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11229 + +require 'active_record/connection_adapters/abstract/schema_definitions' + +module ActiveRecord + module ConnectionAdapters + class TableDefinition + # Appends columns `created_at` and `updated_at` to a table. + # + # It is used in table creation like: + # create_table 'users' do |t| + # t.timestamps_with_timezone + # end + def timestamps_with_timezone(**options) + options[:null] = false if options[:null].nil? + + [:created_at, :updated_at].each do |column_name| + column(column_name, :datetime_with_timezone, options) + end + end + + # Adds specified column with appropriate timestamp type + # + # It is used in table creation like: + # create_table 'users' do |t| + # t.datetime_with_timezone :did_something_at + # end + def datetime_with_timezone(column_name, **options) + column(column_name, :datetime_with_timezone, options) + end + end + end +end diff --git a/config/initializers/ar_speed_up_migration_checking.rb b/config/initializers/ar_speed_up_migration_checking.rb index 1fe5defc01d..aae774daa35 100644 --- a/config/initializers/ar_speed_up_migration_checking.rb +++ b/config/initializers/ar_speed_up_migration_checking.rb @@ -10,7 +10,7 @@ if Rails.env.test? # it reads + parses `db/migrate/*` each time. Memoizing it can save 0.5 # seconds per spec. def migrations(paths) - @migrations ||= migrations_unmemoized(paths) + (@migrations ||= migrations_unmemoized(paths)).dup end end end diff --git a/config/initializers/bootstrap_form.rb b/config/initializers/bootstrap_form.rb new file mode 100644 index 00000000000..11171b38a85 --- /dev/null +++ b/config/initializers/bootstrap_form.rb @@ -0,0 +1,7 @@ +module BootstrapFormBuilderCustomization + def label_class + "label-light" + end +end + +BootstrapForm::FormBuilder.prepend(BootstrapFormBuilderCustomization) diff --git a/config/initializers/carrierwave.rb b/config/initializers/carrierwave.rb index 1933afcbfb1..cd7df44351a 100644 --- a/config/initializers/carrierwave.rb +++ b/config/initializers/carrierwave.rb @@ -6,6 +6,8 @@ if File.exist?(aws_file) AWS_CONFIG = YAML.load(File.read(aws_file))[Rails.env] CarrierWave.configure do |config| + config.fog_provider = 'fog/aws' + config.fog_credentials = { provider: 'AWS', # required aws_access_key_id: AWS_CONFIG['access_key_id'], # required diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb index 3b1317030bc..3aed2136f1b 100644 --- a/config/initializers/devise.rb +++ b/config/initializers/devise.rb @@ -192,7 +192,7 @@ Devise.setup do |config| # # The :"*/*" and "*/*" formats below is required to match Internet # Explorer requests. - # config.navigational_formats = [:"*/*", "*/*", :html] + config.navigational_formats = [:"*/*", "*/*", :html, :zip] # The default HTTP method used to sign out a resource. Default is :delete. config.sign_out_via = :delete @@ -206,11 +206,11 @@ Devise.setup do |config| # If you want to use other strategies, that are not supported by Devise, or # change the failure app, you can configure them inside the config.warden block. # - config.warden do |manager| - manager.failure_app = Gitlab::DeviseFailure - # manager.intercept_401 = false - # manager.default_strategies(scope: :user).unshift :some_external_strategy - end + # config.warden do |manager| + # manager.failure_app = Gitlab::DeviseFailure + # manager.intercept_401 = false + # manager.default_strategies(scope: :user).unshift :some_external_strategy + # end if Gitlab::LDAP::Config.enabled? Gitlab::LDAP::Config.providers.each do |provider| diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb index a5636765774..40e635bf2cf 100644 --- a/config/initializers/doorkeeper.rb +++ b/config/initializers/doorkeeper.rb @@ -87,16 +87,14 @@ Doorkeeper.configure do # "password" => Resource Owner Password Credentials Grant Flow # "client_credentials" => Client Credentials Grant Flow # - # If not specified, Doorkeeper enables all the four grant flows. - # - grant_flows %w(authorization_code password client_credentials) + grant_flows %w(authorization_code implicit password client_credentials) # Under some circumstances you might want to have applications auto-approved, # so that the user skips the authorization step. # For example if dealing with trusted a application. - # skip_authorization do |resource_owner, client| - # client.superapp? or resource_owner.admin? - # end + skip_authorization do |resource_owner, client| + client.application.trusted? + end # WWW-Authenticate Realm (default "Doorkeeper"). # realm "Doorkeeper" diff --git a/config/initializers/doorkeeper_openid_connect.rb b/config/initializers/doorkeeper_openid_connect.rb index 700ca25b884..c58f425b19b 100644 --- a/config/initializers/doorkeeper_openid_connect.rb +++ b/config/initializers/doorkeeper_openid_connect.rb @@ -29,8 +29,8 @@ Doorkeeper::OpenidConnect.configure do o.claim(:email) { |user| user.public_email } o.claim(:email_verified) { |user| true if user.public_email? } o.claim(:website) { |user| user.full_website_url if user.website_url? } - o.claim(:profile) { |user| Rails.application.routes.url_helpers.user_url user } - o.claim(:picture) { |user| user.avatar_url } + o.claim(:profile) { |user| Gitlab::Routing.url_helpers.user_url user } + o.claim(:picture) { |user| user.avatar_url(only_path: false) } end end end diff --git a/config/initializers/fast_gettext.rb b/config/initializers/fast_gettext.rb new file mode 100644 index 00000000000..eb589ecdb52 --- /dev/null +++ b/config/initializers/fast_gettext.rb @@ -0,0 +1,6 @@ +FastGettext.add_text_domain 'gitlab', path: File.join(Rails.root, 'locale'), type: :po +FastGettext.default_text_domain = 'gitlab' +FastGettext.default_available_locales = Gitlab::I18n.available_locales +FastGettext.default_locale = :en + +I18n.available_locales = Gitlab::I18n.available_locales diff --git a/config/initializers/flipper.rb b/config/initializers/flipper.rb new file mode 100644 index 00000000000..bfab8c77a4b --- /dev/null +++ b/config/initializers/flipper.rb @@ -0,0 +1,8 @@ +require 'flipper/middleware/memoizer' + +unless Rails.env.test? + Rails.application.config.middleware.use Flipper::Middleware::Memoizer, + lambda { Feature.flipper } + + Feature.register_feature_groups +end diff --git a/config/initializers/forbid_sidekiq_in_transactions.rb b/config/initializers/forbid_sidekiq_in_transactions.rb new file mode 100644 index 00000000000..a78711fe599 --- /dev/null +++ b/config/initializers/forbid_sidekiq_in_transactions.rb @@ -0,0 +1,49 @@ +module Sidekiq + module Worker + mattr_accessor :skip_transaction_check + self.skip_transaction_check = false + + def self.skipping_transaction_check(&block) + skip_transaction_check = self.skip_transaction_check + self.skip_transaction_check = true + yield + ensure + self.skip_transaction_check = skip_transaction_check + end + + module ClassMethods + module NoSchedulingFromTransactions + NESTING = ::Rails.env.test? ? 1 : 0 + + %i(perform_async perform_at perform_in).each do |name| + define_method(name) do |*args| + return super(*args) if Sidekiq::Worker.skip_transaction_check + return super(*args) unless ActiveRecord::Base.connection.open_transactions > NESTING + + raise <<-MSG.strip_heredoc + `#{self}.#{name}` cannot be called inside a transaction as this can lead to + race conditions when the worker runs before the transaction is committed and + tries to access a model that has not been saved yet. + + Use an `after_commit` hook, or include `AfterCommitQueue` and use a `run_after_commit` block instead. + MSG + end + end + end + + prepend NoSchedulingFromTransactions + end + end +end + +module ActiveRecord + class Base + module SkipTransactionCheckAfterCommit + def committed!(*) + Sidekiq::Worker.skipping_transaction_check { super } + end + end + + prepend SkipTransactionCheckAfterCommit + end +end diff --git a/config/initializers/gettext_rails_i18n_patch.rb b/config/initializers/gettext_rails_i18n_patch.rb new file mode 100644 index 00000000000..377e5104f9d --- /dev/null +++ b/config/initializers/gettext_rails_i18n_patch.rb @@ -0,0 +1,41 @@ +require 'gettext_i18n_rails/haml_parser' +require 'gettext_i18n_rails_js/parser/javascript' + +VUE_TRANSLATE_REGEX = /((%[\w.-]+)(?:\s))?{{ (N|n|s)?__\((.*)\) }}/ + +module GettextI18nRails + class HamlParser + singleton_class.send(:alias_method, :old_convert_to_code, :convert_to_code) + + # We need to convert text in Mustache format + # to a format that can be parsed by Gettext scripts. + # If we found a content like "{{ __('Stage') }}" + # in a HAML file we convert it to "= _('Stage')", that way + # it can be processed by the "rake gettext:find" script. + # + # Overwrites: https://github.com/grosser/gettext_i18n_rails/blob/8396387a431e0f8ead72fc1cd425cad2fa4992f2/lib/gettext_i18n_rails/haml_parser.rb#L9 + def self.convert_to_code(text) + text.gsub!(VUE_TRANSLATE_REGEX, "\\2= \\3_(\\4)") + + old_convert_to_code(text) + end + end +end + +module GettextI18nRailsJs + module Parser + module Javascript + # This is required to tell the `rake gettext:find` script to use the Javascript + # parser for *.vue files. + # + # Overwrites: https://github.com/webhippie/gettext_i18n_rails_js/blob/46c58db6d2053a4f5f36a0eb024ea706ff5707cb/lib/gettext_i18n_rails_js/parser/javascript.rb#L36 + def target?(file) + [ + ".js", + ".jsx", + ".vue" + ].include? ::File.extname(file) + end + end + end +end diff --git a/config/initializers/grape_route_helpers_fix.rb b/config/initializers/grape_route_helpers_fix.rb new file mode 100644 index 00000000000..d3cf9e453d0 --- /dev/null +++ b/config/initializers/grape_route_helpers_fix.rb @@ -0,0 +1,35 @@ +if defined?(GrapeRouteHelpers) + module GrapeRouteHelpers + class DecoratedRoute + # GrapeRouteHelpers gem tries to parse the versions + # from a string, not supporting Grape `version` array definition. + # + # Without the following fix, we get this on route helpers generation: + # + # => undefined method `scan' for ["v3", "v4"] + # + # 2.0.0 implementation of this method: + # + # ``` + # def route_versions + # version_pattern = /[^\[",\]\s]+/ + # if route_version + # route_version.scan(version_pattern) + # else + # [nil] + # end + # end + # ``` + def route_versions + return [nil] if route_version.nil? || route_version.empty? + + if route_version.is_a?(String) + version_pattern = /[^\[",\]\s]+/ + route_version.scan(version_pattern) + else + route_version + end + end + end + end +end diff --git a/config/initializers/hamlit.rb b/config/initializers/hamlit.rb index 7b545d8c06c..51dbffeda05 100644 --- a/config/initializers/hamlit.rb +++ b/config/initializers/hamlit.rb @@ -3,7 +3,7 @@ module Hamlit def call(template) Engine.new( generator: Temple::Generators::RailsOutputBuffer, - attr_quote: '"', + attr_quote: '"' ).call(template.source) end end @@ -11,7 +11,7 @@ end ActionView::Template.register_template_handler( :haml, - Hamlit::TemplateHandler.new, + Hamlit::TemplateHandler.new ) Hamlit::Filters.remove_filter('coffee') diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb new file mode 100644 index 00000000000..21fe8d72459 --- /dev/null +++ b/config/initializers/lograge.rb @@ -0,0 +1,24 @@ +# Only use Lograge for Rails +unless Sidekiq.server? + filename = File.join(Rails.root, 'log', "#{Rails.env}_json.log") + + Rails.application.configure do + config.lograge.enabled = true + # Store the lograge JSON files in a separate file + config.lograge.keep_original_rails_log = true + # Don't use the Logstash formatter since this requires logstash-event, an + # unmaintained gem that monkey patches `Time` + config.lograge.formatter = Lograge::Formatters::Json.new + config.lograge.logger = ActiveSupport::Logger.new(filename) + # Add request parameters to log output + config.lograge.custom_options = lambda do |event| + { + time: event.time.utc.iso8601(3), + params: event.payload[:params].except(*%w(controller action format)), + remote_ip: event.payload[:remote_ip], + user_id: event.payload[:user_id], + username: event.payload[:username] + } + end + end +end diff --git a/config/initializers/mysql_set_length_for_binary_indexes.rb b/config/initializers/mysql_set_length_for_binary_indexes.rb new file mode 100644 index 00000000000..de0bc5322aa --- /dev/null +++ b/config/initializers/mysql_set_length_for_binary_indexes.rb @@ -0,0 +1,21 @@ +# This patches ActiveRecord so indexes for binary columns created using the +# MySQL adapter apply a length of 20. Otherwise MySQL can't create an index on +# binary columns. + +module MysqlSetLengthForBinaryIndex + def add_index(table_name, column_names, options = {}) + Array(column_names).each do |column_name| + column = ActiveRecord::Base.connection.columns(table_name).find { |c| c.name == column_name } + + if column&.type == :binary + options[:length] = 20 + end + end + + super(table_name, column_names, options) + end +end + +if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter) + ActiveRecord::ConnectionAdapters::Mysql2Adapter.send(:prepend, MysqlSetLengthForBinaryIndex) +end diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb index f7fa6d1c2de..56c279ffcf4 100644 --- a/config/initializers/omniauth.rb +++ b/config/initializers/omniauth.rb @@ -1,13 +1,16 @@ if Gitlab::LDAP::Config.enabled? module OmniAuth::Strategies - server = Gitlab.config.ldap.servers.values.first - klass = server['provider_class'] - const_set(klass, Class.new(LDAP)) unless klass == 'LDAP' + Gitlab::LDAP::Config.available_servers.each do |server| + # do not redeclare LDAP + next if server['provider_name'] == 'ldap' + const_set(server['provider_class'], Class.new(LDAP)) + end end OmniauthCallbacksController.class_eval do - server = Gitlab.config.ldap.servers.values.first - alias_method server['provider_name'], :ldap + Gitlab::LDAP::Config.available_servers.each do |server| + alias_method server['provider_name'], :ldap + end end end @@ -16,7 +19,7 @@ OmniAuth.config.allowed_request_methods = [:post] # In case of auto sign-in, the GET method is used (users don't get to click on a button) OmniAuth.config.allowed_request_methods << :get if Gitlab.config.omniauth.auto_sign_in_with_provider.present? OmniAuth.config.before_request_phase do |env| - OmniAuth::RequestForgeryProtection.call(env) + Gitlab::RequestForgeryProtection.call(env) end if Gitlab.config.omniauth.enabled diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb new file mode 100644 index 00000000000..a54d53cbbe2 --- /dev/null +++ b/config/initializers/peek.rb @@ -0,0 +1,28 @@ +Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Cache.params) } + +Peek.into Peek::Views::Host +Peek.into Peek::Views::PerformanceBar +if Gitlab::Database.mysql? + require 'peek-mysql2' + PEEK_DB_CLIENT = ::Mysql2::Client + PEEK_DB_VIEW = Peek::Views::Mysql2 +else + require 'peek-pg' + PEEK_DB_CLIENT = ::PG::Connection + PEEK_DB_VIEW = Peek::Views::PG +end +Peek.into PEEK_DB_VIEW +Peek.into Peek::Views::Redis +Peek.into Peek::Views::Sidekiq +Peek.into Peek::Views::Rblineprof +Peek.into Peek::Views::GC + +# rubocop:disable Style/ClassAndModuleCamelCase +class PEEK_DB_CLIENT + class << self + attr_accessor :query_details + end + self.query_details = Concurrent::Array.new +end + +PEEK_DB_VIEW.prepend ::Gitlab::PerformanceBar::PeekQueryTracker diff --git a/config/initializers/postgresql_cte.rb b/config/initializers/postgresql_cte.rb new file mode 100644 index 00000000000..7f0df8949db --- /dev/null +++ b/config/initializers/postgresql_cte.rb @@ -0,0 +1,132 @@ +# Adds support for WITH statements when using PostgreSQL. The code here is taken +# from https://github.com/shmay/ctes_in_my_pg which at the time of writing has +# not been pushed to RubyGems. The license of this repository is as follows: +# +# The MIT License (MIT) +# +# Copyright (c) 2012 Dan McClain +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +module ActiveRecord + class Relation + class Merger # :nodoc: + def normal_values + NORMAL_VALUES + [:with] + end + end + end +end + +module ActiveRecord::Querying + delegate :with, to: :all +end + +module ActiveRecord + class Relation + # WithChain objects act as placeholder for queries in which #with does not have any parameter. + # In this case, #with must be chained with #recursive to return a new relation. + class WithChain + def initialize(scope) + @scope = scope + end + + # Returns a new relation expressing WITH RECURSIVE + def recursive(*args) + @scope.with_values += args + @scope.recursive_value = true + @scope + end + end + + def with_values + @values[:with] || [] + end + + def with_values=(values) + raise ImmutableRelation if @loaded + @values[:with] = values + end + + def recursive_value=(value) + raise ImmutableRelation if @loaded + @values[:recursive] = value + end + + def recursive_value + @values[:recursive] + end + + def with(opts = :chain, *rest) + if opts == :chain + WithChain.new(spawn) + elsif opts.blank? + self + else + spawn.with!(opts, *rest) + end + end + + def with!(opts = :chain, *rest) # :nodoc: + if opts == :chain + WithChain.new(self) + else + self.with_values += [opts] + rest + self + end + end + + def build_arel + arel = super() + + build_with(arel) if @values[:with] + + arel + end + + def build_with(arel) + with_statements = with_values.flat_map do |with_value| + case with_value + when String + with_value + when Hash + with_value.map do |name, expression| + case expression + when String + select = Arel::Nodes::SqlLiteral.new "(#{expression})" + when ActiveRecord::Relation, Arel::SelectManager + select = Arel::Nodes::SqlLiteral.new "(#{expression.to_sql})" + end + Arel::Nodes::As.new Arel::Nodes::SqlLiteral.new("\"#{name}\""), select + end + when Arel::Nodes::As + with_value + end + end + + unless with_statements.empty? + if recursive_value + arel.with :recursive, with_statements + else + arel.with with_statements + end + end + end + end +end diff --git a/config/initializers/relative_naming_ci_namespace.rb b/config/initializers/relative_naming_ci_namespace.rb index 59abe1b9b91..d9d3034150f 100644 --- a/config/initializers/relative_naming_ci_namespace.rb +++ b/config/initializers/relative_naming_ci_namespace.rb @@ -4,10 +4,10 @@ # - [project.namespace, project, build] # # instead of: -# - namespace_project_build_path(project.namespace, project, build) +# - project_job_path(project, build) # # Without that, Ci:: namespace is used for resolving routes: -# - namespace_project_ci_build_path(project.namespace, project, build) +# - project_ci_build_path(project, build) module Ci def self.use_relative_model_naming? diff --git a/config/initializers/rspec_profiling.rb b/config/initializers/rspec_profiling.rb index b909cc5b9a4..16b9d5b15e5 100644 --- a/config/initializers/rspec_profiling.rb +++ b/config/initializers/rspec_profiling.rb @@ -32,14 +32,14 @@ end if Rails.env.test? RspecProfiling.configure do |config| - if ENV['RSPEC_PROFILING_POSTGRES_URL'] + if ENV['RSPEC_PROFILING_POSTGRES_URL'].present? RspecProfiling::Collectors::PSQL.prepend(RspecProfilingExt::PSQL) config.collector = RspecProfiling::Collectors::PSQL end - end - if ENV.has_key?('CI') && ENV['GITLAB_DATABASE'] == 'postgresql' - RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git) - RspecProfiling::Run.prepend(RspecProfilingExt::Run) + if ENV.key?('CI') + RspecProfiling::VCS::Git.prepend(RspecProfilingExt::Git) + RspecProfiling::Run.prepend(RspecProfilingExt::Run) + end end end diff --git a/config/initializers/rugged_use_gitlab_git_attributes.rb b/config/initializers/rugged_use_gitlab_git_attributes.rb new file mode 100644 index 00000000000..7d652799786 --- /dev/null +++ b/config/initializers/rugged_use_gitlab_git_attributes.rb @@ -0,0 +1,25 @@ +# We don't want to ever call Rugged::Repository#fetch_attributes, because it has +# a lot of I/O overhead: +# <https://gitlab.com/gitlab-org/gitlab_git/commit/340e111e040ae847b614d35b4d3173ec48329015> +# +# While we don't do this from within the GitLab source itself, the Linguist gem +# has a dependency on Rugged and uses the gitattributes file when calculating +# repository-wide language statistics: +# <https://github.com/github/linguist/blob/v4.7.0/lib/linguist/lazy_blob.rb#L33-L36> +# +# The options passed by Linguist are those assumed by Gitlab::Git::Attributes +# anyway, and there is no great efficiency gain from just fetching the listed +# attributes with our implementation, so we ignore the additional arguments. +# +module Rugged + class Repository + module UseGitlabGitAttributes + def fetch_attributes(name, *) + @attributes ||= Gitlab::Git::Attributes.new(path) + @attributes.attributes(name) + end + end + + prepend UseGitlabGitAttributes + end +end diff --git a/config/initializers/server_uptime.rb b/config/initializers/server_uptime.rb new file mode 100644 index 00000000000..46bf242e143 --- /dev/null +++ b/config/initializers/server_uptime.rb @@ -0,0 +1 @@ +Rails.application.config.booted_at = Time.now diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb index 70be2617cab..e8213ac8ba4 100644 --- a/config/initializers/session_store.rb +++ b/config/initializers/session_store.rb @@ -10,16 +10,22 @@ rescue Settings.gitlab['session_expire_delay'] ||= 10080 end +cookie_key = if Rails.env.development? + "_gitlab_session_#{Digest::SHA256.hexdigest(Rails.root.to_s)}" + else + "_gitlab_session" + end + if Rails.env.test? Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" else - redis_config = Gitlab::Redis.params - redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE + sessions_config = Gitlab::Redis::SharedState.params + sessions_config[:namespace] = Gitlab::Redis::SharedState::SESSION_NAMESPACE Gitlab::Application.config.session_store( :redis_store, # Using the cookie_store would enable session replay attacks. - servers: redis_config, - key: '_gitlab_session', + servers: sessions_config, + key: cookie_key, secure: Gitlab.config.gitlab.https, httponly: true, expires_in: Settings.gitlab['session_expire_delay'] * 60, diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index ecd73956488..a1cc9655319 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -1,12 +1,12 @@ -# Custom Redis configuration -redis_config_hash = Gitlab::Redis.params -redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE +# Custom Queues configuration +queues_config_hash = Gitlab::Redis::Queues.params +queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE # Default is to retry 25 times with exponential backoff. That's too much. Sidekiq.default_worker_options = { retry: 3 } Sidekiq.configure_server do |config| - config.redis = redis_config_hash + config.redis = queues_config_hash config.server_middleware do |chain| chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] @@ -54,7 +54,7 @@ Sidekiq.configure_server do |config| end Sidekiq.configure_client do |config| - config.redis = redis_config_hash + config.redis = queues_config_hash config.client_middleware do |chain| chain.add Gitlab::SidekiqStatus::ClientMiddleware @@ -74,5 +74,5 @@ begin end end end -rescue Redis::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED +rescue Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED end diff --git a/config/initializers/static_files.rb b/config/initializers/static_files.rb index 74aba6c5d06..943e01f1496 100644 --- a/config/initializers/static_files.rb +++ b/config/initializers/static_files.rb @@ -1,15 +1,15 @@ app = Rails.application if app.config.serve_static_files - # The `ActionDispatch::Static` middleware intercepts requests for static files - # by checking if they exist in the `/public` directory. + # The `ActionDispatch::Static` middleware intercepts requests for static files + # by checking if they exist in the `/public` directory. # We're replacing it with our `Gitlab::Middleware::Static` that does the same, # except ignoring `/uploads`, letting those go through to the GitLab Rails app. app.config.middleware.swap( - ActionDispatch::Static, - Gitlab::Middleware::Static, - app.paths["public"].first, + ActionDispatch::Static, + Gitlab::Middleware::Static, + app.paths["public"].first, app.config.static_cache_control ) @@ -23,21 +23,21 @@ if app.config.serve_static_files host: dev_server.host, port: dev_server.port, manifest_host: dev_server.host, - manifest_port: dev_server.port, + manifest_port: dev_server.port } if Rails.env.development? settings.merge!( host: Gitlab.config.gitlab.host, port: Gitlab.config.gitlab.port, - https: Gitlab.config.gitlab.https, + https: Gitlab.config.gitlab.https ) app.config.middleware.insert_before( Gitlab::Middleware::Static, Gitlab::Middleware::WebpackProxy, proxy_path: app.config.webpack.public_path, proxy_host: dev_server.host, - proxy_port: dev_server.port, + proxy_port: dev_server.port ) end diff --git a/config/initializers/trusted_proxies.rb b/config/initializers/trusted_proxies.rb index fc4f02453d7..0c32528311e 100644 --- a/config/initializers/trusted_proxies.rb +++ b/config/initializers/trusted_proxies.rb @@ -2,7 +2,7 @@ # as the ActionDispatch::Request object. This is necessary for libraries # like rack_attack where they don't use ActionDispatch, and we want them # to block/throttle requests on private networks. -# Rack Attack specific issue: https://github.com/kickstarter/rack-attack/issues/145 +# Rack Attack specific issue: https://github.com/kickstarter/rack-attack/issues/145 module Rack class Request def trusted_proxy?(ip) diff --git a/config/karma.config.js b/config/karma.config.js index eb082dd28bf..2f571978e08 100644 --- a/config/karma.config.js +++ b/config/karma.config.js @@ -13,13 +13,26 @@ if (webpackConfig.plugins) { }); } +webpackConfig.devtool = 'cheap-inline-source-map'; + // Karma configuration module.exports = function(config) { var progressReporter = process.env.CI ? 'mocha' : 'progress'; var karmaConfig = { basePath: ROOT_PATH, - browsers: ['PhantomJS'], + browsers: ['ChromeHeadlessCustom'], + customLaunchers: { + ChromeHeadlessCustom: { + base: 'ChromeHeadless', + displayName: 'Chrome', + flags: [ + // chrome cannot run in sandboxed mode inside a docker container unless it is run with + // escalated kernel privileges (e.g. docker run --cap-add=CAP_SYS_ADMIN) + '--no-sandbox', + ], + } + }, frameworks: ['jasmine'], files: [ { pattern: 'spec/javascripts/test_bundle.js', watched: false }, @@ -41,6 +54,25 @@ module.exports = function(config) { subdir: '.', fixWebpackSourcePaths: true }; + karmaConfig.browserNoActivityTimeout = 60000; // 60 seconds + } + + if (process.env.DEBUG) { + karmaConfig.logLevel = config.LOG_DEBUG; + process.env.CHROME_LOG_FILE = process.env.CHROME_LOG_FILE || 'chrome_debug.log'; + } + + if (process.env.CHROME_LOG_FILE) { + karmaConfig.customLaunchers.ChromeHeadlessCustom.flags.push('--enable-logging', '--v=1'); + } + + if (process.env.DEBUG) { + karmaConfig.logLevel = config.LOG_DEBUG; + process.env.CHROME_LOG_FILE = process.env.CHROME_LOG_FILE || 'chrome_debug.log'; + } + + if (process.env.CHROME_LOG_FILE) { + karmaConfig.customLaunchers.ChromeHeadlessCustom.flags.push('--enable-logging', '--v=1'); } config.set(karmaConfig); diff --git a/config/locales/de.yml b/config/locales/de.yml new file mode 100644 index 00000000000..38c3711c6c7 --- /dev/null +++ b/config/locales/de.yml @@ -0,0 +1,256 @@ +--- +de: + activerecord: + errors: + messages: + record_invalid: 'Gültigkeitsprüfung ist fehlgeschlagen: %{errors}' + restrict_dependent_destroy: + has_one: Datensatz kann nicht gelöscht werden, da ein abhängiger %{record}-Datensatz + existiert. + has_many: Datensatz kann nicht gelöscht werden, da abhängige %{record} existieren. + date: + abbr_day_names: + - So + - Mo + - Di + - Mi + - Do + - Fr + - Sa + abbr_month_names: + - + - Jan + - Feb + - Mär + - Apr + - Mai + - Jun + - Jul + - Aug + - Sep + - Okt + - Nov + - Dez + day_names: + - Sonntag + - Montag + - Dienstag + - Mittwoch + - Donnerstag + - Freitag + - Samstag + formats: + default: "%d.%m.%Y" + long: "%e. %B %Y" + short: "%e. %b" + month_names: + - + - Januar + - Februar + - März + - April + - Mai + - Juni + - Juli + - August + - September + - Oktober + - November + - Dezember + order: + - :day + - :month + - :year + datetime: + # used in a custom scope that has been created to fix https://gitlab.com/gitlab-org/gitlab-ce/issues/32747 + time_ago_in_words: + half_a_minute: vor einer halben Minute + less_than_x_seconds: + one: vor weniger als einer Sekunde + other: "vor weniger als %{count} Sekunden" + x_seconds: + one: vor einer Sekunde + other: "vor %{count} Sekunden" + less_than_x_minutes: + one: vor weniger als einer Minute + other: vor weniger als %{count} Minuten + x_minutes: + one: vor einer Minute + other: "vor %{count} Minuten" + about_x_hours: + one: vor etwa einer Stunde + other: "vor etwa %{count} Stunden" + x_days: + one: vor einem Tag + other: "vor %{count} Tagen" + about_x_months: + one: vor etwa einem Monat + other: "vor etwa %{count} Monaten" + x_months: + one: vor einem Monat + other: "vor %{count} Monaten" + about_x_years: + one: vor etwa einem Jahr + other: "vor etwa %{count} Jahren" + over_x_years: + one: vor mehr als einem Jahr + other: "vor mehr als %{count} Jahren" + almost_x_years: + one: vor fast einem Jahr + other: "vor fast %{count} Jahren" + # Used in distance_of_time_in_words(), distance_of_time_in_words_to_now(), time_ago_in_words() + distance_in_words: + about_x_hours: + one: etwa eine Stunde + other: etwa %{count} Stunden + about_x_months: + one: etwa ein Monat + other: etwa %{count} Monate + about_x_years: + one: etwa ein Jahr + other: etwa %{count} Jahre + almost_x_years: + one: fast ein Jahr + other: fast %{count} Jahre + half_a_minute: eine halbe Minute + less_than_x_minutes: + one: weniger als eine Minute + other: weniger als %{count} Minuten + less_than_x_seconds: + one: weniger als eine Sekunde + other: weniger als %{count} Sekunden + over_x_years: + one: mehr als ein Jahr + other: mehr als %{count} Jahre + x_days: + one: ein Tag + other: "%{count} Tage" + x_minutes: + one: eine Minute + other: "%{count} Minuten" + x_months: + one: ein Monat + other: "%{count} Monate" + x_seconds: + one: eine Sekunde + other: "%{count} Sekunden" + prompts: + day: Tag + hour: Stunden + minute: Minute + month: Monat + second: Sekunde + year: Jahr + errors: + format: "%{attribute} %{message}" + messages: + accepted: muss akzeptiert werden + blank: muss ausgefüllt werden + present: darf nicht ausgefüllt werden + confirmation: stimmt nicht mit %{attribute} überein + empty: muss ausgefüllt werden + equal_to: muss genau %{count} sein + even: muss gerade sein + exclusion: ist nicht verfügbar + greater_than: muss größer als %{count} sein + greater_than_or_equal_to: muss größer oder gleich %{count} sein + inclusion: ist kein gültiger Wert + invalid: ist nicht gültig + less_than: muss kleiner als %{count} sein + less_than_or_equal_to: muss kleiner oder gleich %{count} sein + model_invalid: 'Gültigkeitsprüfung ist fehlgeschlagen: %{errors}' + not_a_number: ist keine Zahl + not_an_integer: muss ganzzahlig sein + odd: muss ungerade sein + required: muss ausgefüllt werden + taken: ist bereits vergeben + too_long: + one: ist zu lang (mehr als 1 Zeichen) + other: ist zu lang (mehr als %{count} Zeichen) + too_short: + one: ist zu kurz (weniger als 1 Zeichen) + other: ist zu kurz (weniger als %{count} Zeichen) + wrong_length: + one: hat die falsche Länge (muss genau 1 Zeichen haben) + other: hat die falsche Länge (muss genau %{count} Zeichen haben) + other_than: darf nicht gleich %{count} sein + template: + body: 'Bitte überprüfen Sie die folgenden Felder:' + header: + one: 'Konnte %{model} nicht speichern: ein Fehler.' + other: 'Konnte %{model} nicht speichern: %{count} Fehler.' + helpers: + select: + prompt: Bitte wählen + submit: + create: "%{model} erstellen" + submit: "%{model} speichern" + update: "%{model} aktualisieren" + number: + currency: + format: + delimiter: "." + format: "%n %u" + precision: 2 + separator: "," + significant: false + strip_insignificant_zeros: false + unit: "€" + format: + delimiter: "." + precision: 2 + separator: "," + significant: false + strip_insignificant_zeros: false + human: + decimal_units: + format: "%n %u" + units: + billion: + one: Milliarde + other: Milliarden + million: + one: Million + other: Millionen + quadrillion: + one: Billiarde + other: Billiarden + thousand: Tausend + trillion: + one: Billion + other: Billionen + unit: '' + format: + delimiter: '' + precision: 3 + significant: true + strip_insignificant_zeros: true + storage_units: + format: "%n %u" + units: + byte: + one: Byte + other: Bytes + gb: GB + kb: KB + mb: MB + tb: TB + percentage: + format: + delimiter: '' + format: "%n %" + precision: + format: + delimiter: '' + support: + array: + last_word_connector: " und " + two_words_connector: " und " + words_connector: ", " + time: + am: vormittags + formats: + default: "%A, %d. %B %Y, %H:%M Uhr" + long: "%A, %d. %B %Y, %H:%M Uhr" + short: "%d. %B, %H:%M Uhr" + pm: nachmittags diff --git a/config/locales/en.yml b/config/locales/en.yml index 12a59be79f0..8932db138d9 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -2,14 +2,251 @@ # See https://github.com/svenfuchs/rails-i18n/tree/master/rails%2Flocale for starting points. en: - hello: "Hello world" + views: + pagination: + previous: "Prev" + next: "Next" + date: + abbr_day_names: + - Sun + - Mon + - Tue + - Wed + - Thu + - Fri + - Sat + abbr_month_names: + - + - Jan + - Feb + - Mar + - Apr + - May + - Jun + - Jul + - Aug + - Sep + - Oct + - Nov + - Dec + day_names: + - Sunday + - Monday + - Tuesday + - Wednesday + - Thursday + - Friday + - Saturday + formats: + default: "%Y-%m-%d" + long: "%B %d, %Y" + short: "%b %d" + month_names: + - + - January + - February + - March + - April + - May + - June + - July + - August + - September + - October + - November + - December + order: + - :year + - :month + - :day + datetime: + time_ago_in_words: + half_a_minute: "half a minute ago" + less_than_x_seconds: + one: "less than 1 second ago" + other: "less than %{count} seconds ago" + x_seconds: + one: "1 second ago" + other: "%{count} seconds ago" + less_than_x_minutes: + one: "less than a minute ago" + other: "less than %{count} minutes ago" + x_minutes: + one: "1 minute ago" + other: "%{count} minutes ago" + about_x_hours: + one: "about 1 hour ago" + other: "about %{count} hours ago" + x_days: + one: "1 day ago" + other: "%{count} days ago" + about_x_months: + one: "about 1 month ago" + other: "about %{count} months ago" + x_months: + one: "1 month ago" + other: "%{count} months ago" + about_x_years: + one: "about 1 year ago" + other: "about %{count} years ago" + over_x_years: + one: "over 1 year ago" + other: "over %{count} years ago" + almost_x_years: + one: "almost 1 year ago" + other: "almost %{count} years ago" + distance_in_words: + about_x_hours: + one: about 1 hour + other: about %{count} hours + about_x_months: + one: about 1 month + other: about %{count} months + about_x_years: + one: about 1 year + other: about %{count} years + almost_x_years: + one: almost 1 year + other: almost %{count} years + half_a_minute: half a minute + less_than_x_minutes: + one: less than a minute + other: less than %{count} minutes + less_than_x_seconds: + one: less than 1 second + other: less than %{count} seconds + over_x_years: + one: over 1 year + other: over %{count} years + x_days: + one: 1 day + other: "%{count} days" + x_minutes: + one: 1 minute + other: "%{count} minutes" + x_months: + one: 1 month + other: "%{count} months" + x_years: + one: 1 year + other: "%{count} years" + x_seconds: + one: 1 second + other: "%{count} seconds" + prompts: + day: Day + hour: Hour + minute: Minute + month: Month + second: Seconds + year: Year errors: + format: "%{attribute} %{message}" messages: label_already_exists_at_group_level: "already exists at group level for %{group}. Please choose another one." wrong_size: "is the wrong size (should be %{file_size})" size_too_small: "is too small (should be at least %{file_size})" size_too_big: "is too big (should be at most %{file_size})" - views: - pagination: - previous: "Prev" - next: "Next" + accepted: must be accepted + blank: can't be blank + present: must be blank + confirmation: doesn't match %{attribute} + empty: can't be empty + equal_to: must be equal to %{count} + even: must be even + exclusion: is reserved + greater_than: must be greater than %{count} + greater_than_or_equal_to: must be greater than or equal to %{count} + inclusion: is not included in the list + invalid: is invalid + less_than: must be less than %{count} + less_than_or_equal_to: must be less than or equal to %{count} + model_invalid: "Validation failed: %{errors}" + not_a_number: is not a number + not_an_integer: must be an integer + odd: must be odd + required: must exist + taken: has already been taken + too_long: + one: is too long (maximum is 1 character) + other: is too long (maximum is %{count} characters) + too_short: + one: is too short (minimum is 1 character) + other: is too short (minimum is %{count} characters) + wrong_length: + one: is the wrong length (should be 1 character) + other: is the wrong length (should be %{count} characters) + other_than: must be other than %{count} + template: + body: 'There were problems with the following fields:' + header: + one: 1 error prohibited this %{model} from being saved + other: "%{count} errors prohibited this %{model} from being saved" + helpers: + select: + prompt: Please select + submit: + create: Create %{model} + submit: Save %{model} + update: Update %{model} + number: + currency: + format: + delimiter: "," + format: "%u%n" + precision: 2 + separator: "." + significant: false + strip_insignificant_zeros: false + unit: "$" + format: + delimiter: "," + precision: 3 + separator: "." + significant: false + strip_insignificant_zeros: false + human: + decimal_units: + format: "%n %u" + units: + billion: Billion + million: Million + quadrillion: Quadrillion + thousand: Thousand + trillion: Trillion + unit: '' + format: + delimiter: '' + precision: 3 + significant: true + strip_insignificant_zeros: true + storage_units: + format: "%n %u" + units: + byte: + one: Byte + other: Bytes + gb: GB + kb: KB + mb: MB + tb: TB + percentage: + format: + delimiter: '' + format: "%n%" + precision: + format: + delimiter: '' + support: + array: + last_word_connector: ", and " + two_words_connector: " and " + words_connector: ", " + time: + am: am + formats: + default: "%a, %d %b %Y %H:%M:%S %z" + long: "%B %d, %Y %H:%M" + short: "%d %b %H:%M" + timeago_tooltip: "%b %-d, %Y %-l:%M%P" + pm: pm diff --git a/config/locales/es.yml b/config/locales/es.yml new file mode 100644 index 00000000000..fdc52b4ae11 --- /dev/null +++ b/config/locales/es.yml @@ -0,0 +1,255 @@ +--- +es: + activerecord: + errors: + messages: + record_invalid: "La validación falló: %{errors}" + restrict_dependent_destroy: + has_one: No se puede eliminar el registro porque existe un %{record} dependiente + has_many: No se puede eliminar el registro porque existen %{record} dependientes + date: + abbr_day_names: + - dom + - lun + - mar + - mié + - jue + - vie + - sáb + abbr_month_names: + - + - ene + - feb + - mar + - abr + - may + - jun + - jul + - ago + - sep + - oct + - nov + - dic + day_names: + - domingo + - lunes + - martes + - miércoles + - jueves + - viernes + - sábado + formats: + default: "%d/%m/%Y" + long: "%d de %B de %Y" + short: "%d de %b" + month_names: + - + - enero + - febrero + - marzo + - abril + - mayo + - junio + - julio + - agosto + - septiembre + - octubre + - noviembre + - diciembre + order: + - :day + - :month + - :year + datetime: + # used in a custom scope that has been created to fix https://gitlab.com/gitlab-org/gitlab-ce/issues/32747 + time_ago_in_words: + half_a_minute: "hace medio minuto" + less_than_x_seconds: + one: "hace menos de 1 segundo" + other: "hace menos de %{count} segundos" + x_seconds: + one: "hace 1 segundo" + other: "hace %{count} segundos" + less_than_x_minutes: + one: "hace menos de un minuto" + other: "hace menos de %{count} minutos" + x_minutes: + one: "hace 1 minuto" + other: "hace %{count} minutos" + about_x_hours: + one: "hace alrededor de 1 hora" + other: "hace alrededor de %{count} horas" + x_days: + one: "hace un dÃa" + other: "hace %{count} dÃas" + about_x_months: + one: "hace alrededor de 1 mes" + other: "hace alrededor de %{count} meses" + x_months: + one: "hace 1 mes" + other: "hace %{count} meses" + about_x_years: + one: "hace alrededor de 1 año" + other: "hace alrededor de %{count} años" + over_x_years: + one: "hace más de 1 año" + other: "hace %{count} años" + almost_x_years: + one: "hace casi 1 año" + other: "hace casi %{count} años" + # Used in distance_of_time_in_words(), distance_of_time_in_words_to_now(), time_ago_in_words() + distance_in_words: + about_x_hours: + one: alrededor de 1 hora + other: alrededor de %{count} horas + about_x_months: + one: alrededor de 1 mes + other: alrededor de %{count} meses + about_x_years: + one: alrededor de 1 año + other: alrededor de %{count} años + almost_x_years: + one: casi 1 año + other: casi %{count} años + half_a_minute: medio minuto + less_than_x_minutes: + one: menos de 1 minuto + other: menos de %{count} minutos + less_than_x_seconds: + one: menos de 1 segundo + other: menos de %{count} segundos + over_x_years: + one: más de 1 año + other: más de %{count} años + x_days: + one: 1 dÃa + other: "%{count} dÃas" + x_minutes: + one: 1 minuto + other: "%{count} minutos" + x_months: + one: 1 mes + other: "%{count} meses" + x_years: + one: 1 año + other: "%{count} años" + x_seconds: + one: 1 segundo + other: "%{count} segundos" + prompts: + day: DÃa + hour: Hora + minute: Minutos + month: Mes + second: Segundos + year: Año + errors: + format: "%{attribute} %{message}" + messages: + accepted: debe ser aceptado + blank: no puede estar en blanco + present: debe estar en blanco + confirmation: no coincide + empty: no puede estar vacÃo + equal_to: debe ser igual a %{count} + even: debe ser par + exclusion: está reservado + greater_than: debe ser mayor que %{count} + greater_than_or_equal_to: debe ser mayor que o igual a %{count} + inclusion: no está incluido en la lista + invalid: no es válido + less_than: debe ser menor que %{count} + less_than_or_equal_to: debe ser menor que o igual a %{count} + model_invalid: "La validación falló: %{errors}" + not_a_number: no es un número + not_an_integer: debe ser un entero + odd: debe ser impar + required: debe existir + taken: ya está en uso + too_long: + one: "es demasiado largo (1 carácter máximo)" + other: "es demasiado largo (%{count} caracteres máximo)" + too_short: + one: "es demasiado corto (1 carácter mÃnimo)" + other: "es demasiado corto (%{count} caracteres mÃnimo)" + wrong_length: + one: "no tiene la longitud correcta (1 carácter exactos)" + other: "no tiene la longitud correcta (%{count} caracteres exactos)" + other_than: debe ser distinto de %{count} + template: + body: 'Se encontraron problemas con los siguientes campos:' + header: + one: No se pudo guardar este/a %{model} porque se encontró 1 error + other: No se pudo guardar este/a %{model} porque se encontraron %{count} errores + helpers: + select: + prompt: Por favor seleccione + submit: + create: Crear %{model} + submit: Guardar %{model} + update: Actualizar %{model} + number: + currency: + format: + delimiter: "." + format: "%n %u" + precision: 2 + separator: "," + significant: false + strip_insignificant_zeros: false + unit: "€" + format: + delimiter: "." + precision: 3 + separator: "," + significant: false + strip_insignificant_zeros: false + human: + decimal_units: + format: "%n %u" + units: + billion: mil millones + million: + one: millón + other: millones + quadrillion: mil billones + thousand: mil + trillion: + one: billón + other: billones + unit: '' + format: + delimiter: '' + precision: 1 + significant: true + strip_insignificant_zeros: true + storage_units: + format: "%n %u" + units: + byte: + one: Byte + other: Bytes + gb: GB + kb: KB + mb: MB + tb: TB + percentage: + format: + delimiter: '' + format: "%n %" + precision: + format: + delimiter: '' + support: + array: + last_word_connector: " y " + two_words_connector: " y " + words_connector: ", " + time: + am: am + formats: + default: "%A, %d de %B de %Y %H:%M:%S %z" + long: "%d de %B de %Y %H:%M" + short: "%d de %b %H:%M" + timeago_tooltip: "%d de %B de %Y %H:%M" + pm: pm diff --git a/config/mail_room.yml b/config/mail_room.yml index 88d93d4bc6b..c3a5be8d38c 100644 --- a/config/mail_room.yml +++ b/config/mail_room.yml @@ -21,7 +21,7 @@ :delivery_method: sidekiq :delivery_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %> + :namespace: <%= Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE %> :queue: email_receiver :worker: EmailReceiverWorker <% if config[:sentinels] %> @@ -36,7 +36,7 @@ :arbitration_method: redis :arbitration_options: :redis_url: <%= config[:redis_url].to_json %> - :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %> + :namespace: <%= Gitlab::Redis::Queues::MAILROOM_NAMESPACE %> <% if config[:sentinels] %> :sentinels: <% config[:sentinels].each do |sentinel| %> diff --git a/config/prometheus/additional_metrics.yml b/config/prometheus/additional_metrics.yml new file mode 100644 index 00000000000..0642a0b2fe9 --- /dev/null +++ b/config/prometheus/additional_metrics.yml @@ -0,0 +1,133 @@ +- group: Response metrics (NGINX Ingress) + priority: 10 + metrics: + - title: "Throughput" + y_label: "Requests / Sec" + required_metrics: + - nginx_upstream_requests_total + weight: 1 + queries: + - query_range: 'sum(rate(nginx_upstream_requests_total{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m]))' + label: Total + unit: req / sec + - title: "Latency" + y_label: "Latency (ms)" + required_metrics: + - nginx_upstream_response_msecs_avg + weight: 1 + queries: + - query_range: 'avg(nginx_upstream_response_msecs_avg{upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"})' + label: Average + unit: ms + - title: "HTTP Error Rate" + y_label: "HTTP 500 Errors / Sec" + required_metrics: + - nginx_upstream_responses_total + weight: 1 + queries: + - query_range: 'sum(rate(nginx_upstream_responses_total{status_code="5xx", upstream=~"%{kube_namespace}-%{ci_environment_slug}-.*"}[2m]))' + label: HTTP Errors + unit: "errors / sec" +- group: Response metrics (HA Proxy) + priority: 10 + metrics: + - title: "Throughput" + y_label: "Requests / Sec" + required_metrics: + - haproxy_frontend_http_requests_total + weight: 1 + queries: + - query_range: 'sum(rate(haproxy_frontend_http_requests_total{%{environment_filter}}[2m]))' + label: Total + unit: req / sec + - title: "HTTP Error Rate" + y_label: "Error Rate (%)" + required_metrics: + - haproxy_frontend_http_responses_total + weight: 1 + queries: + - query_range: 'sum(rate(haproxy_frontend_http_responses_total{code="5xx",%{environment_filter}}[2m])) / sum(rate(haproxy_frontend_http_responses_total{%{environment_filter}}[2m]))' + label: HTTP Errors + unit: "%" +- group: Response metrics (AWS ELB) + priority: 10 + metrics: + - title: "Throughput" + y_label: "Requests / Sec" + required_metrics: + - aws_elb_request_count_sum + weight: 1 + queries: + - query_range: 'sum(aws_elb_request_count_sum{%{environment_filter}}) / 60' + label: Total + unit: req / sec + - title: "Latency" + y_label: "Latency (ms)" + required_metrics: + - aws_elb_latency_average + weight: 1 + queries: + - query_range: 'avg(aws_elb_latency_average{%{environment_filter}}) * 1000' + label: Average + unit: ms + - title: "HTTP Error Rate" + y_label: "Error Rate (%)" + required_metrics: + - aws_elb_request_count_sum + - aws_elb_httpcode_backend_5_xx_sum + weight: 1 + queries: + - query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})' + label: HTTP Errors + unit: "%" +- group: Response metrics (NGINX) + priority: 10 + metrics: + - title: "Throughput" + y_label: "Requests / Sec" + required_metrics: + - nginx_requests_total + weight: 1 + queries: + - query_range: 'sum(rate(nginx_requests_total{server_zone!="*", server_zone!="_", %{environment_filter}}[2m]))' + label: Total + unit: req / sec + - title: "Latency" + y_label: "Latency (ms)" + required_metrics: + - nginx_upstream_response_msecs_avg + weight: 1 + queries: + - query_range: 'avg(nginx_upstream_response_msecs_avg{%{environment_filter}})' + label: Upstream + unit: ms + - title: "HTTP Error Rate" + y_label: "HTTP 500 Errors / Sec" + required_metrics: + - nginx_responses_total + weight: 1 + queries: + - query_range: 'sum(rate(nginx_responses_total{status_code="5xx", %{environment_filter}}[2m]))' + label: HTTP Errors + unit: "errors / sec" +- group: System metrics (Kubernetes) + priority: 5 + metrics: + - title: "Memory Usage" + y_label: "Memory Usage (MB)" + required_metrics: + - container_memory_usage_bytes + weight: 1 + queries: + - query_range: '(sum(container_memory_usage_bytes{container_name!="POD",%{environment_filter}}) / count(container_memory_usage_bytes{container_name!="POD",%{environment_filter}})) /1024/1024' + label: Average + unit: MB + - title: "CPU Utilization" + y_label: "CPU Utilization (%)" + required_metrics: + - container_cpu_usage_seconds_total + weight: 1 + queries: + - query_range: 'sum(rate(container_cpu_usage_seconds_total{container_name!="POD",%{environment_filter}}[2m])) / count(container_cpu_usage_seconds_total{container_name!="POD",%{environment_filter}}) * 100' + label: Average + unit: "%" diff --git a/config/redis.cache.yml.example b/config/redis.cache.yml.example new file mode 100644 index 00000000000..27478f0a93e --- /dev/null +++ b/config/redis.cache.yml.example @@ -0,0 +1,38 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379/10 + # + # url: redis://localhost:6380 + # sentinels: + # - + # host: localhost + # port: 26380 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26380 # point to sentinel, not to redis port +test: + url: redis://localhost:6379/10 + # + # url: redis://localhost:6380 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.cache.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6380 + # sentinels: + # - + # host: slave1 + # port: 26380 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26380 # point to sentinel, not to redis port diff --git a/config/redis.queues.yml.example b/config/redis.queues.yml.example new file mode 100644 index 00000000000..dab1f26b096 --- /dev/null +++ b/config/redis.queues.yml.example @@ -0,0 +1,38 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379/11 + # + # url: redis://localhost:6381 + # sentinels: + # - + # host: localhost + # port: 26381 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26381 # point to sentinel, not to redis port +test: + url: redis://localhost:6379/11 + # + # url: redis://localhost:6381 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.queues.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6381 + # sentinels: + # - + # host: slave1 + # port: 26381 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26381 # point to sentinel, not to redis port diff --git a/config/redis.shared_state.yml.example b/config/redis.shared_state.yml.example new file mode 100644 index 00000000000..9371e3619b7 --- /dev/null +++ b/config/redis.shared_state.yml.example @@ -0,0 +1,38 @@ +# If you change this file in a Merge Request, please also create +# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379/12 + # + # url: redis://localhost:6382 + # sentinels: + # - + # host: localhost + # port: 26382 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26382 # point to sentinel, not to redis port +test: + url: redis://localhost:6379/12 + # + # url: redis://localhost:6382 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.shared_state.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html + ## + # url: redis://master:6382 + # sentinels: + # - + # host: slave1 + # port: 26382 # point to sentinel, not to redis port + # - + # host: slave2 + # port: 26382 # point to sentinel, not to redis port diff --git a/config/routes.rb b/config/routes.rb index 1da226a3b57..4fd6cb5d439 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,6 +1,5 @@ require 'sidekiq/web' require 'sidekiq/cron/web' -require 'constraints/group_url_constrainer' Rails.application.routes.draw do concern :access_requestable do @@ -39,10 +38,11 @@ Rails.application.routes.draw do # Health check get 'health_check(/:checks)' => 'health_check#index', as: :health_check - scope path: '-', controller: 'health' do - get :liveness - get :readiness - get :metrics + scope path: '-' do + get 'liveness' => 'health#liveness' + get 'readiness' => 'health#readiness' + resources :metrics, only: [:index] + mount Peek::Railtie => '/peek' end # Koding route @@ -85,19 +85,7 @@ Rails.application.routes.draw do root to: "root#index" - # Since group show page is wildcard routing - # we want all other routing to be checked before matching this one - constraints(GroupUrlConstrainer.new) do - scope(path: '*id', - as: :group, - constraints: { id: Gitlab::Regex.namespace_route_regex, format: /(html|json|atom)/ }, - controller: :groups) do - get '/', action: :show - patch '/', action: :update - put '/', action: :update - delete '/', action: :destroy - end - end + draw :test if Rails.env.test? get '*unmatched_route', to: 'application#route_not_found' end diff --git a/config/routes/admin.rb b/config/routes/admin.rb index 52ba10604d4..c0748231813 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -36,7 +36,7 @@ namespace :admin do scope(path: 'groups/*id', controller: :groups, - constraints: { id: Gitlab::Regex.namespace_route_regex, format: /(html|json|atom)/ }) do + constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ }) do scope(as: :group) do put :members_update @@ -48,10 +48,18 @@ namespace :admin do end end - resources :deploy_keys, only: [:index, :new, :create, :destroy] + resources :deploy_keys, only: [:index, :new, :create, :edit, :update, :destroy] - resources :hooks, only: [:index, :create, :destroy] do - get :test + resources :hooks, only: [:index, :create, :edit, :update, :destroy] do + member do + get :test + end + + resources :hook_logs, only: [:show] do + member do + get :retry + end + end end resources :broadcast_messages, only: [:index, :edit, :create, :update, :destroy] do @@ -59,17 +67,23 @@ namespace :admin do end resource :logs, only: [:show] - resource :health_check, controller: 'health_check', only: [:show] + resource :health_check, controller: 'health_check', only: [:show] do + post :reset_storage_health + end resource :background_jobs, controller: 'background_jobs', only: [:show] resource :system_info, controller: 'system_info', only: [:show] resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ } + get 'conversational_development_index' => 'conversational_development_index#show' + resources :projects, only: [:index] - scope(path: 'projects/*namespace_id', as: :namespace) do + scope(path: 'projects/*namespace_id', + as: :namespace, + constraints: { namespace_id: Gitlab::PathRegex.full_namespace_route_regex }) do resources(:projects, path: '/', - constraints: { id: Gitlab::Regex.project_route_regex }, + constraints: { id: Gitlab::PathRegex.project_route_regex }, only: [:show]) do member do @@ -108,7 +122,7 @@ namespace :admin do resources :cohorts, only: :index - resources :builds, only: :index do + resources :jobs, only: :index do collection do post :cancel_all end diff --git a/config/routes/api.rb b/config/routes/api.rb index 69c8efc151c..ce7a7c88900 100644 --- a/config/routes/api.rb +++ b/config/routes/api.rb @@ -1,2 +1,2 @@ API::API.logger Rails.logger -mount API::API => '/api' +mount API::API => '/' diff --git a/config/routes/dashboard.rb b/config/routes/dashboard.rb index 8e380a0b0ac..d2437285cdf 100644 --- a/config/routes/dashboard.rb +++ b/config/routes/dashboard.rb @@ -4,7 +4,13 @@ resource :dashboard, controller: 'dashboard', only: [] do get :activity scope module: :dashboard do - resources :milestones, only: [:index, :show] + resources :milestones, only: [:index, :show] do + member do + get :merge_requests + get :participants + get :labels + end + end resources :labels, only: [:index] resources :groups, only: [:index] diff --git a/config/routes/git_http.rb b/config/routes/git_http.rb index 42d874eeebc..a53c94326d4 100644 --- a/config/routes/git_http.rb +++ b/config/routes/git_http.rb @@ -1,5 +1,7 @@ -scope(path: '*namespace_id/:project_id', constraints: { format: nil }) do - scope(constraints: { project_id: Gitlab::Regex.project_git_route_regex }, module: :projects) do +scope(path: '*namespace_id/:project_id', + format: nil, + constraints: { namespace_id: Gitlab::PathRegex.full_namespace_route_regex }) do + scope(constraints: { project_id: Gitlab::PathRegex.project_git_route_regex }, module: :projects) do # Git HTTP clients ('git clone' etc.) scope(controller: :git_http) do get '/info/refs', action: :info_refs @@ -26,7 +28,7 @@ scope(path: '*namespace_id/:project_id', constraints: { format: nil }) do end # Redirect /group/project/info/refs to /group/project.git/info/refs - scope(constraints: { project_id: Gitlab::Regex.project_route_regex }) do + scope(constraints: { project_id: Gitlab::PathRegex.project_route_regex }) do # Allow /info/refs, /info/refs?service=git-upload-pack, and # /info/refs?service=git-receive-pack, but nothing else. # diff --git a/config/routes/group.rb b/config/routes/group.rb index 73f69d76995..23052a6c6dc 100644 --- a/config/routes/group.rb +++ b/config/routes/group.rb @@ -1,25 +1,41 @@ +require 'constraints/group_url_constrainer' + resources :groups, only: [:index, :new, :create] scope(path: 'groups/*group_id', module: :groups, as: :group, - constraints: { group_id: Gitlab::Regex.namespace_route_regex }) do + constraints: { group_id: Gitlab::PathRegex.full_namespace_route_regex }) do resources :group_members, only: [:index, :create, :update, :destroy], concerns: :access_requestable do post :resend_invite, on: :member delete :leave, on: :collection end resource :avatar, only: [:destroy] - resources :milestones, constraints: { id: /[^\/]+/ }, only: [:index, :show, :update, :new, :create] + resources :milestones, constraints: { id: /[^\/]+/ }, only: [:index, :show, :edit, :update, :new, :create] do + member do + get :merge_requests + get :participants + get :labels + end + end resources :labels, except: [:show] do post :toggle_subscription, on: :member end + + scope path: '-' do + namespace :settings do + resource :ci_cd, only: [:show], controller: 'ci_cd' + end + + resources :variables, only: [:index, :show, :update, :create, :destroy] + end end scope(path: 'groups/*id', controller: :groups, - constraints: { id: Gitlab::Regex.namespace_route_regex, format: /(html|json|atom)/ }) do + constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ }) do get :edit, as: :edit_group get :issues, as: :issues_group get :merge_requests, as: :merge_requests_group @@ -28,3 +44,15 @@ scope(path: 'groups/*id', get :subgroups, as: :subgroups_group get '/', action: :show, as: :group_canonical end + +constraints(GroupUrlConstrainer.new) do + scope(path: '*id', + as: :group, + constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ }, + controller: :groups) do + get '/', action: :show + patch '/', action: :update + put '/', action: :update + delete '/', action: :destroy + end +end diff --git a/config/routes/legacy_builds.rb b/config/routes/legacy_builds.rb new file mode 100644 index 00000000000..5ab2b953ce1 --- /dev/null +++ b/config/routes/legacy_builds.rb @@ -0,0 +1,22 @@ +resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do + collection do + resources :artifacts, only: [], controller: 'build_artifacts' do + collection do + get :latest_succeeded, + path: '*ref_name_and_path', + format: false + end + end + end + + member do + get :raw + end + + resource :artifacts, only: [], controller: 'build_artifacts' do + get :download + get :browse, path: 'browse(/*path)', format: false + get :file, path: 'file/*path', format: false + get :raw, path: 'raw/*path', format: false + end +end diff --git a/config/routes/profile.rb b/config/routes/profile.rb index 07c341999ea..3e4e6111ab8 100644 --- a/config/routes/profile.rb +++ b/config/routes/profile.rb @@ -5,6 +5,7 @@ resource :profile, only: [:show, :update] do put :reset_private_token put :reset_incoming_email_token + put :reset_rss_token put :update_username end @@ -22,6 +23,11 @@ resource :profile, only: [:show, :update] do end resource :preferences, only: [:show, :update] resources :keys, only: [:index, :show, :create, :destroy] + resources :gpg_keys, only: [:index, :create, :destroy] do + member do + put :revoke + end + end resources :emails, only: [:index, :create, :destroy] resources :chat_names, only: [:index, :new, :create, :destroy] do collection do diff --git a/config/routes/project.rb b/config/routes/project.rb index fa92202c1ea..06928c7b9ce 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -5,9 +5,24 @@ resources :projects, only: [:index, :new, :create] draw :git_http constraints(ProjectUrlConstrainer.new) do - scope(path: '*namespace_id', as: :namespace) do + # If the route has a wildcard segment, the segment has a regex constraint, + # the segment is potentially followed by _another_ wildcard segment, and + # the `format` option is not set to false, we need to specify that + # regex constraint _outside_ of `constraints: {}`. + # + # Otherwise, Rails will overwrite the constraint with `/.+?/`, + # which breaks some of our wildcard routes like `/blob/*id` + # and `/tree/*id` that depend on the negative lookahead inside + # `Gitlab::PathRegex.full_namespace_route_regex`, which helps the router + # determine whether a certain path segment is part of `*namespace_id`, + # `:project_id`, or `*id`. + # + # See https://github.com/rails/rails/blob/v4.2.8/actionpack/lib/action_dispatch/routing/mapper.rb#L155 + scope(path: '*namespace_id', + as: :namespace, + namespace_id: Gitlab::PathRegex.full_namespace_route_regex) do scope(path: ':project_id', - constraints: { project_id: Gitlab::Regex.project_route_regex }, + constraints: { project_id: Gitlab::PathRegex.project_route_regex }, module: :projects, as: :project) do @@ -44,20 +59,24 @@ constraints(ProjectUrlConstrainer.new) do resources :snippets, concerns: :awardable, constraints: { id: /\d+/ } do member do - get 'raw' + get :raw post :mark_as_spam end end resources :services, constraints: { id: /[^\/]+/ }, only: [:index, :edit, :update] do member do - get :test + put :test end end resource :mattermost, only: [:new, :create] - resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create] do + namespace :prometheus do + get :active_metrics + end + + resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create, :edit, :update] do member do put :enable put :disable @@ -67,34 +86,41 @@ constraints(ProjectUrlConstrainer.new) do resources :forks, only: [:index, :new, :create] resource :import, only: [:new, :create, :show] - resources :merge_requests, concerns: :awardable, constraints: { id: /\d+/ } do + resources :merge_requests, concerns: :awardable, except: [:new, :create], constraints: { id: /\d+/ } do member do - get :commits - get :diffs - get :conflicts - get :conflict_for_path - get :pipelines - get :merge_check + get :commit_change_content post :merge - get :merge_widget_refresh post :cancel_merge_when_pipeline_succeeds - get :ci_status get :pipeline_status get :ci_environments_status post :toggle_subscription post :remove_wip - get :diff_for_path - post :resolve_conflicts post :assign_related_issues + + scope constraints: { format: nil }, action: :show do + get :commits, defaults: { tab: 'commits' } + get :pipelines, defaults: { tab: 'pipelines' } + get :diffs, defaults: { tab: 'diffs' } + end + + scope constraints: { format: 'json' }, as: :json do + get :commits + get :pipelines + get :diffs, to: 'merge_requests/diffs#show' + end + + get :diff_for_path, controller: 'merge_requests/diffs' + + scope controller: 'merge_requests/conflicts' do + get :conflicts, action: :show + get :conflict_for_path + post :resolve_conflicts + end end collection do - get :branch_from - get :branch_to - get :update_branches get :diff_for_path post :bulk_update - get :new_diffs, path: 'new/diffs' end resources :discussions, only: [], constraints: { id: /\h{40}/ } do @@ -105,6 +131,29 @@ constraints(ProjectUrlConstrainer.new) do end end + controller 'merge_requests/creations', path: 'merge_requests' do + post '', action: :create, as: nil + + scope path: 'new', as: :new_merge_request do + get '', action: :new + + scope constraints: { format: nil }, action: :new do + get :diffs, defaults: { tab: 'diffs' } + get :pipelines, defaults: { tab: 'pipelines' } + end + + scope constraints: { format: 'json' }, as: :json do + get :diffs + get :pipelines + end + + get :diff_for_path + get :update_branches + get :branch_from + get :branch_to + end + end + resources :variables, only: [:index, :show, :update, :create, :destroy] resources :triggers, only: [:index, :create, :edit, :update, :destroy] do member do @@ -123,21 +172,36 @@ constraints(ProjectUrlConstrainer.new) do post :cancel post :retry get :builds + get :failures get :status end end + resources :pipeline_schedules, except: [:show] do + member do + post :take_ownership + end + end + resources :environments, except: [:destroy] do member do post :stop get :terminal get :metrics + get :additional_metrics get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', constraints: { format: nil } end collection do get :folder, path: 'folders/*id', constraints: { format: /(html|json)/ } end + + resources :deployments, only: [:index] do + member do + get :metrics + get :additional_metrics + end + end end resource :cycle_analytics, only: [:show] @@ -154,41 +218,52 @@ constraints(ProjectUrlConstrainer.new) do end end - resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do - collection do - post :cancel_all - - resources :artifacts, only: [] do - collection do - get :latest_succeeded, - path: '*ref_name_and_path', - format: false + scope '-' do + resources :jobs, only: [:index, :show], constraints: { id: /\d+/ } do + collection do + post :cancel_all + + resources :artifacts, only: [] do + collection do + get :latest_succeeded, + path: '*ref_name_and_path', + format: false + end end end - end - member do - get :status - post :cancel - post :retry - post :play - post :erase - get :trace - get :raw - end + member do + get :status + post :cancel + post :retry + post :play + post :erase + get :trace, defaults: { format: 'json' } + get :raw + end - resource :artifacts, only: [] do - get :download - get :browse, path: 'browse(/*path)', format: false - get :file, path: 'file/*path', format: false - post :keep + resource :artifacts, only: [] do + get :download + get :browse, path: 'browse(/*path)', format: false + get :file, path: 'file/*path', format: false + get :raw, path: 'raw/*path', format: false + post :keep + end end end - resources :hooks, only: [:index, :create, :destroy], constraints: { id: /\d+/ } do + draw :legacy_builds + + resources :hooks, only: [:index, :create, :edit, :update, :destroy], constraints: { id: /\d+/ } do member do get :test end + + resources :hook_logs, only: [:show] do + member do + get :retry + end + end end resources :container_registry, only: [:index, :destroy], @@ -197,7 +272,7 @@ constraints(ProjectUrlConstrainer.new) do namespace :registry do resources :repository, only: [] do resources :tags, only: [:destroy], - constraints: { id: Gitlab::Regex.container_registry_reference_regex } + constraints: { id: Gitlab::Regex.container_registry_tag_regex } end end @@ -205,6 +280,9 @@ constraints(ProjectUrlConstrainer.new) do member do put :sort_issues put :sort_merge_requests + get :merge_requests + get :participants + get :labels end end @@ -228,7 +306,8 @@ constraints(ProjectUrlConstrainer.new) do get :referenced_merge_requests get :related_branches get :can_create_branch - get :rendered_title + get :realtime_changes + post :create_merge_request end collection do post :bulk_update @@ -298,16 +377,18 @@ constraints(ProjectUrlConstrainer.new) do resources :runner_projects, only: [:create, :destroy] resources :badges, only: [:index] do collection do - scope '*ref', constraints: { ref: Gitlab::Regex.git_reference_regex } do + scope '*ref', constraints: { ref: Gitlab::PathRegex.git_reference_regex } do constraints format: /svg/ do - get :build + # Keep around until 10.0, see gitlab-org/gitlab-ce#35307 + get :build, to: "badges#pipeline" + get :pipeline get :coverage end end end end namespace :settings do - resource :members, only: [:show] + get :members, to: redirect('/%{namespace_id}/%{project_id}/project_members') resource :ci_cd, only: [:show], controller: 'ci_cd' resource :integrations, only: [:show] resource :repository, only: [:show], controller: :repository @@ -321,7 +402,7 @@ constraints(ProjectUrlConstrainer.new) do resources(:projects, path: '/', - constraints: { id: Gitlab::Regex.project_route_regex }, + constraints: { id: Gitlab::PathRegex.project_route_regex }, only: [:edit, :show, :update, :destroy]) do member do put :transfer diff --git a/config/routes/repository.rb b/config/routes/repository.rb index 5cf37a06e97..9ffdebbcff1 100644 --- a/config/routes/repository.rb +++ b/config/routes/repository.rb @@ -2,7 +2,10 @@ resource :repository, only: [:create] do member do - get 'archive', constraints: { format: Gitlab::Regex.archive_formats_regex } + get ':ref/archive', constraints: { format: Gitlab::PathRegex.archive_formats_regex, ref: /.+/ }, action: 'archive', as: 'archive' + + # deprecated since GitLab 9.5 + get 'archive', constraints: { format: Gitlab::PathRegex.archive_formats_regex }, as: 'archive_alternative' end end @@ -24,7 +27,7 @@ scope format: false do member do # tree viewer logs - get 'logs_tree', constraints: { id: Gitlab::Regex.git_reference_regex } + get 'logs_tree', constraints: { id: Gitlab::PathRegex.git_reference_regex } # Directories with leading dots erroneously get rejected if git # ref regex used in constraints. Regex verification now done in controller. get 'logs_tree/*path', action: :logs_tree, as: :logs_file, format: false, constraints: { @@ -34,7 +37,7 @@ scope format: false do end end - scope constraints: { id: Gitlab::Regex.git_reference_regex } do + scope constraints: { id: Gitlab::PathRegex.git_reference_regex } do resources :network, only: [:show] resources :graphs, only: [:show] do @@ -76,6 +79,8 @@ scope format: false do get '/tree/*id', to: 'tree#show', as: :tree get '/raw/*id', to: 'raw#show', as: :raw get '/blame/*id', to: 'blame#show', as: :blame + + get '/commits/*id/signatures', to: 'commits#signatures', as: :signatures get '/commits/*id', to: 'commits#show', as: :commits post '/create_dir/*id', to: 'tree#create_dir', as: :create_dir diff --git a/config/routes/snippets.rb b/config/routes/snippets.rb index 56534f677be..0a4ebac3ca3 100644 --- a/config/routes/snippets.rb +++ b/config/routes/snippets.rb @@ -1,10 +1,20 @@ resources :snippets, concerns: :awardable do member do - get 'raw' - get 'download' + get :raw post :mark_as_spam + end + + collection do post :preview_markdown end + + scope module: :snippets do + resources :notes, only: [:index, :create, :destroy, :update], concerns: :awardable, constraints: { id: /\d+/ } do + member do + delete :delete_attachment + end + end + end end get '/s/:username', to: redirect('/u/%{username}/snippets'), diff --git a/config/routes/test.rb b/config/routes/test.rb new file mode 100644 index 00000000000..ac477cdbbbc --- /dev/null +++ b/config/routes/test.rb @@ -0,0 +1,2 @@ +get '/unicorn_test/pid' => 'unicorn_test#pid' +post '/unicorn_test/kill' => 'unicorn_test#kill' diff --git a/config/routes/uploads.rb b/config/routes/uploads.rb index 2b22148a134..d7bca8310e4 100644 --- a/config/routes/uploads.rb +++ b/config/routes/uploads.rb @@ -1,11 +1,21 @@ scope path: :uploads do # Note attachments and User/Group/Project avatars - get ":model/:mounted_as/:id/:filename", + get "-/system/:model/:mounted_as/:id/:filename", to: "uploads#show", constraints: { model: /note|user|group|project/, mounted_as: /avatar|attachment/, filename: /[^\/]+/ } + # show uploads for models, snippets (notes) available for now + get '-/system/:model/:id/:secret/:filename', + to: 'uploads#show', + constraints: { model: /personal_snippet/, id: /\d+/, filename: /[^\/]+/ } + + # show temporary uploads + get '-/system/temp/:secret/:filename', + to: 'uploads#show', + constraints: { filename: /[^\/]+/ } + # Appearance - get ":model/:mounted_as/:id/:filename", + get "-/system/:model/:mounted_as/:id/:filename", to: "uploads#show", constraints: { model: /appearance/, mounted_as: /logo|header_logo/, filename: /.+/ } @@ -13,6 +23,12 @@ scope path: :uploads do get ":namespace_id/:project_id/:secret/:filename", to: "projects/uploads#show", constraints: { namespace_id: /[a-zA-Z.0-9_\-]+/, project_id: /[a-zA-Z.0-9_\-]+/, filename: /[^\/]+/ } + + # create uploads for models, snippets (notes) available for now + post ':model', + to: 'uploads#create', + constraints: { model: /personal_snippet/, id: /\d+/ }, + as: 'upload' end # Redirect old note attachments path to new uploads path. diff --git a/config/routes/user.rb b/config/routes/user.rb index b064a15e802..e682dcd6663 100644 --- a/config/routes/user.rb +++ b/config/routes/user.rb @@ -11,19 +11,7 @@ devise_scope :user do get '/users/almost_there' => 'confirmations#almost_there' end -constraints(UserUrlConstrainer.new) do - # Get all keys of user - get ':username.keys' => 'profiles/keys#get_keys', constraints: { username: Gitlab::Regex.namespace_route_regex } - - scope(path: ':username', - as: :user, - constraints: { username: Gitlab::Regex.namespace_route_regex }, - controller: :users) do - get '/', action: :show - end -end - -scope(constraints: { username: Gitlab::Regex.namespace_route_regex }) do +scope(constraints: { username: Gitlab::PathRegex.root_namespace_route_regex }) do scope(path: 'users/:username', as: :user, controller: :users) do @@ -34,7 +22,7 @@ scope(constraints: { username: Gitlab::Regex.namespace_route_regex }) do get :contributed, as: :contributed_projects get :snippets get :exists - get '/', to: redirect('/%{username}') + get '/', to: redirect('/%{username}'), as: nil end # Compatibility with old routing @@ -46,3 +34,15 @@ scope(constraints: { username: Gitlab::Regex.namespace_route_regex }) do get '/u/:username/snippets', to: redirect('/users/%{username}/snippets') get '/u/:username/contributed', to: redirect('/users/%{username}/contributed') end + +constraints(UserUrlConstrainer.new) do + # Get all keys of user + get ':username.keys' => 'profiles/keys#get_keys', constraints: { username: Gitlab::PathRegex.root_namespace_route_regex } + + scope(path: ':username', + as: :user, + constraints: { username: Gitlab::PathRegex.root_namespace_route_regex }, + controller: :users) do + get '/', action: :show + end +end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index c3bd73533d0..83abc83c9f0 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -23,12 +23,16 @@ - [update_merge_requests, 3] - [process_commit, 3] - [new_note, 2] + - [new_issue, 2] + - [new_merge_request, 2] - [build, 2] - [pipeline, 2] - [gitlab_shell, 2] - [email_receiver, 2] - [emails_on_push, 2] - [mailers, 2] + - [invalid_gpg_signature_update, 2] + - [create_gpg_signature, 2] - [upload_checksum, 1] - [use_key, 1] - [repository_fork, 1] @@ -40,12 +44,12 @@ - [expire_build_instance_artifacts, 1] - [group_destroy, 1] - [irker, 1] + - [namespaceless_project_destroy, 1] - [project_cache, 1] - [project_destroy, 1] - [project_export, 1] - - [project_web_hook, 1] + - [web_hook, 1] - [repository_check, 1] - - [system_hook, 1] - [git_garbage_collect, 1] - [reactive_caching, 1] - [cronjob, 1] @@ -53,3 +57,5 @@ - [pages, 1] - [system_hook_push, 1] - [update_user_activity, 1] + - [propagate_service_template, 1] + - [background_migration, 1] diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example index 40a16a32359..cc10da2bd88 100644 --- a/config/unicorn.rb.example +++ b/config/unicorn.rb.example @@ -121,6 +121,10 @@ after_fork do |server, worker| defined?(ActiveRecord::Base) and ActiveRecord::Base.establish_connection + # reset prometheus client, this will cause any opened metrics files to be closed + defined?(::Prometheus::Client.reinitialize_on_pid_change) && + Prometheus::Client.reinitialize_on_pid_change + # if preload_app is true, then you may also want to check and # restart any other shared sockets/descriptors such as Memcached, # and Redis. TokyoCabinet file handles are safe to reuse diff --git a/config/webpack.config.js b/config/webpack.config.js index cb0a57a3a41..6a347c2e660 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -3,22 +3,30 @@ var fs = require('fs'); var path = require('path'); var webpack = require('webpack'); -var StatsPlugin = require('stats-webpack-plugin'); +var StatsWriterPlugin = require('webpack-stats-plugin').StatsWriterPlugin; +var CopyWebpackPlugin = require('copy-webpack-plugin'); var CompressionPlugin = require('compression-webpack-plugin'); +var NameAllModulesPlugin = require('name-all-modules-plugin'); var BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin; var WatchMissingNodeModulesPlugin = require('react-dev-utils/WatchMissingNodeModulesPlugin'); var ROOT_PATH = path.resolve(__dirname, '..'); var IS_PRODUCTION = process.env.NODE_ENV === 'production'; -var IS_DEV_SERVER = process.argv[1].indexOf('webpack-dev-server') !== -1; +var IS_DEV_SERVER = process.argv.join(' ').indexOf('webpack-dev-server') !== -1; var DEV_SERVER_HOST = process.env.DEV_SERVER_HOST || 'localhost'; var DEV_SERVER_PORT = parseInt(process.env.DEV_SERVER_PORT, 10) || 3808; var DEV_SERVER_LIVERELOAD = process.env.DEV_SERVER_LIVERELOAD !== 'false'; var WEBPACK_REPORT = process.env.WEBPACK_REPORT; +var NO_COMPRESSION = process.env.NO_COMPRESSION; var config = { + // because sqljs requires fs. + node: { + fs: "empty" + }, context: path.join(ROOT_PATH, 'app/assets/javascripts'), entry: { + balsamiq_viewer: './blob/balsamiq_viewer.js', blob: './blob_edit/blob_bundle.js', boards: './boards/boards_bundle.js', common: './commons/index.js', @@ -26,42 +34,64 @@ var config = { common_d3: ['d3'], cycle_analytics: './cycle_analytics/cycle_analytics_bundle.js', commit_pipelines: './commit/pipelines/pipelines_bundle.js', + deploy_keys: './deploy_keys/index.js', diff_notes: './diff_notes/diff_notes_bundle.js', environments: './environments/environments_bundle.js', environments_folder: './environments/folder/environments_folder_bundle.js', filtered_search: './filtered_search/filtered_search_bundle.js', graphs: './graphs/graphs_bundle.js', + graphs_charts: './graphs/graphs_charts.js', + graphs_show: './graphs/graphs_show.js', group: './group.js', + groups: './groups/index.js', groups_list: './groups_list.js', - issuable: './issuable/issuable_bundle.js', + how_to_merge: './how_to_merge.js', issue_show: './issue_show/index.js', + integrations: './integrations', + job_details: './jobs/job_details_bundle.js', + locale: './locale/index.js', main: './main.js', merge_conflicts: './merge_conflicts/merge_conflicts_bundle.js', - merge_request_widget: './merge_request_widget/ci_bundle.js', monitoring: './monitoring/monitoring_bundle.js', network: './network/network_bundle.js', notebook_viewer: './blob/notebook_viewer.js', pdf_viewer: './blob/pdf_viewer.js', - pipelines: './pipelines/index.js', + pipelines: './pipelines/pipelines_bundle.js', + pipelines_charts: './pipelines/pipelines_charts.js', + pipelines_details: './pipelines/pipeline_details_bundle.js', + pipelines_times: './pipelines/pipelines_times.js', profile: './profile/profile_bundle.js', - protected_branches: './protected_branches/protected_branches_bundle.js', + project_import_gl: './projects/project_import_gitlab_project.js', + project_new: './projects/project_new.js', + prometheus_metrics: './prometheus_metrics', + protected_branches: './protected_branches', protected_tags: './protected_tags', + repo: './repo/index.js', + sidebar: './sidebar/sidebar_bundle.js', + schedule_form: './pipeline_schedules/pipeline_schedule_form_bundle.js', + schedules_index: './pipeline_schedules/pipeline_schedules_index_bundle.js', snippet: './snippet/snippet_bundle.js', sketch_viewer: './blob/sketch_viewer.js', stl_viewer: './blob/stl_viewer.js', terminal: './terminal/terminal_bundle.js', u2f: ['vendor/u2f'], - users: './users/users_bundle.js', + ui_development_kit: './ui_development_kit.js', + users: './users/index.js', + raven: './raven/index.js', + vue_merge_request_widget: './vue_merge_request_widget/index.js', + test: './test.js', + two_factor_auth: './two_factor_auth.js', + performance_bar: './performance_bar.js', + webpack_runtime: './webpack.js', }, output: { path: path.join(ROOT_PATH, 'public/assets/webpack'), publicPath: '/assets/webpack/', - filename: IS_PRODUCTION ? '[name].[chunkhash].bundle.js' : '[name].bundle.js' + filename: IS_PRODUCTION ? '[name].[chunkhash].bundle.js' : '[name].bundle.js', + chunkFilename: IS_PRODUCTION ? '[name].[chunkhash].chunk.js' : '[name].chunk.js', }, - devtool: 'cheap-module-source-map', - module: { rules: [ { @@ -78,22 +108,49 @@ var config = { loader: 'raw-loader', }, { - test: /\.(worker\.js|pdf)$/, + test: /\.(gif|png)$/, + loader: 'url-loader', + options: { limit: 2048 }, + }, + { + test: /\.(worker(\.min)?\.js|pdf|bmpr)$/, exclude: /node_modules/, loader: 'file-loader', + options: { + name: '[name].[hash].[ext]', + } + }, + { + test: /locale\/\w+\/(.*)\.js$/, + loader: 'exports-loader?locales', }, - ] + { + test: /monaco-editor\/\w+\/vs\/loader\.js$/, + use: [ + { loader: 'exports-loader', options: 'l.global' }, + { loader: 'imports-loader', options: 'l=>{},this=>l,AMDLoader=>this,module=>undefined' }, + ], + } + ], + + noParse: [/monaco-editor\/\w+\/vs\//], }, plugins: [ // manifest filename must match config.webpack.manifest_filename // webpack-rails only needs assetsByChunkName to function properly - new StatsPlugin('manifest.json', { - chunkModules: false, - source: false, - chunks: false, - modules: false, - assets: true + new StatsWriterPlugin({ + filename: 'manifest.json', + transform: function(data, opts) { + var stats = opts.compiler.getStats().toJson({ + chunkModules: false, + source: false, + chunks: false, + modules: false, + assets: true + }); + return JSON.stringify(stats, null, 2); + } }), // prevent pikaday from including moment.js @@ -105,10 +162,20 @@ var config = { jQuery: 'jquery', }), - // use deterministic module ids in all environments - IS_PRODUCTION ? - new webpack.HashedModuleIdsPlugin() : - new webpack.NamedModulesPlugin(), + // assign deterministic module ids + new webpack.NamedModulesPlugin(), + new NameAllModulesPlugin(), + + // assign deterministic chunk ids + new webpack.NamedChunksPlugin((chunk) => { + if (chunk.name) { + return chunk.name; + } + return chunk.modules.map((m) => { + var chunkPath = m.request.split('!').pop(); + return path.relative(m.context, chunkPath); + }).join('_'); + }), // create cacheable common library bundle for all vue chunks new webpack.optimize.CommonsChunkPlugin({ @@ -117,15 +184,25 @@ var config = { 'boards', 'commit_pipelines', 'cycle_analytics', + 'deploy_keys', 'diff_notes', 'environments', 'environments_folder', - 'issuable', + 'filtered_search', + 'groups', 'issue_show', + 'job_details', 'merge_conflicts', + 'monitoring', 'notebook_viewer', 'pdf_viewer', 'pipelines', + 'pipelines_details', + 'repo', + 'schedule_form', + 'schedules_index', + 'sidebar', + 'vue_merge_request_widget', ], minChunks: function(module, count) { return module.resource && (/vue_shared/).test(module.resource); @@ -137,15 +214,37 @@ var config = { name: 'common_d3', chunks: [ 'graphs', - 'users', 'monitoring', ], }), // create cacheable common library bundles new webpack.optimize.CommonsChunkPlugin({ - names: ['main', 'common', 'runtime'], + names: ['main', 'locale', 'common', 'webpack_runtime'], }), + + // enable scope hoisting + new webpack.optimize.ModuleConcatenationPlugin(), + + // copy pre-compiled vendor libraries verbatim + new CopyWebpackPlugin([ + { + from: path.join(ROOT_PATH, `node_modules/monaco-editor/${IS_PRODUCTION ? 'min' : 'dev'}/vs`), + to: 'monaco-editor/vs', + transform: function(content, path) { + if (/\.js$/.test(path) && !/worker/i.test(path)) { + return ( + '(function(){\n' + + 'var define = this.define, require = this.require;\n' + + 'window.define = define; window.require = require;\n' + + content + + '\n}.call(window.__monaco_context__ || (window.__monaco_context__ = {})));' + ); + } + return content; + } + } + ]), ], resolve: { @@ -155,6 +254,7 @@ var config = { 'emojis': path.join(ROOT_PATH, 'fixtures/emojis'), 'empty_states': path.join(ROOT_PATH, 'app/views/shared/empty_states'), 'icons': path.join(ROOT_PATH, 'app/views/shared/icons'), + 'images': path.join(ROOT_PATH, 'app/assets/images'), 'vendor': path.join(ROOT_PATH, 'vendor/assets/javascripts'), 'vue$': 'vue/dist/vue.esm.js', } @@ -174,11 +274,18 @@ if (IS_PRODUCTION) { }), new webpack.DefinePlugin({ 'process.env': { NODE_ENV: JSON.stringify('production') } - }), - new CompressionPlugin({ - asset: '[path].gz[query]', }) ); + + // zopfli requires a lot of compute time and is disabled in CI + if (!NO_COMPRESSION) { + // gracefully fall back to gzip if `node-zopfli` is unavailable (e.g. in CentOS 6) + try { + config.plugins.push(new CompressionPlugin({ algorithm: 'zopfli' })); + } catch(err) { + config.plugins.push(new CompressionPlugin({ algorithm: 'gzip' })); + } + } } if (IS_DEV_SERVER) { @@ -186,15 +293,19 @@ if (IS_DEV_SERVER) { config.devServer = { host: DEV_SERVER_HOST, port: DEV_SERVER_PORT, + disableHostCheck: true, headers: { 'Access-Control-Allow-Origin': '*' }, stats: 'errors-only', + hot: DEV_SERVER_LIVERELOAD, inline: DEV_SERVER_LIVERELOAD }; - config.output.publicPath = '//' + DEV_SERVER_HOST + ':' + DEV_SERVER_PORT + config.output.publicPath; config.plugins.push( // watch node_modules for changes if we encounter a missing module compile error new WatchMissingNodeModulesPlugin(path.join(ROOT_PATH, 'node_modules')) ); + if (DEV_SERVER_LIVERELOAD) { + config.plugins.push(new webpack.HotModuleReplacementPlugin()); + } } if (WEBPACK_REPORT) { |