Merge commit '5b08d59f07fc53c1e34819fac20352119d5343e6' into object-storage-ee-to-ce-backport

38 files changed, 662 insertions, 106 deletions

diff --git a/config/README.md b/config/README.md
index 0a5ea2424e0..2778d0d4f02 100644
--- a/config/README.md
+++ b/config/README.md
@@ -19,4 +19,132 @@ an ERB file and then loads the resulting YML as its configuration.
 
 This file is called `resque.yml` for historical reasons. We are **NOT**
 using Resque at the moment. It is used to specify Redis configuration
-values instead.
+values when a single database instance of Redis is desired.
+
+# Advanced Redis configuration files
+
+In more advanced configurations of Redis key-value storage, it is desirable
+to separate the keys by lifecycle and intended use to ease provisioning and
+management of scalable Redis clusters.
+
+These settings provide routing and other configuration data (such as sentinel,
+persistence policies, and other Redis customization) for connections
+to Redis single instances, Redis sentinel, and Redis clusters.
+
+If desired, the routing URL provided by these settings can be used with:
+1. Unix Socket
+    1. named socket for each Redis instance desired.
+    2. `database number` for each Redis instance desired.
+2. TCP Socket
+    1. `host name` or IP for each Redis instance desired
+    2. TCP port number for each Redis instance desired
+    3. `database number` for each Redis instance desired
+    
+## Example URL attribute formats for GitLab Redis `.yml` configuration files
+* Unix Socket, default Redis database (0)
+    * `url: unix:/path/to/redis.sock`
+    * `url: unix:/path/to/redis.sock?db=`
+* Unix Socket, Redis database 44
+    * `url: unix:/path/to/redis.sock?db=44`
+    * `url: unix:/path/to/redis.sock?extra=foo&db=44`
+* TCP Socket for Redis on localhost, port 6379, database 33
+    * `url: redis://:mynewpassword@localhost:6379/33`
+* TCP Socket for Redis on remote host `myserver`, port 6379, database 33
+    * `url: redis://:mynewpassword@myserver:6379/33`
+
+## redis.cache.yml
+
+If configured, `redis.cache.yml` overrides the
+`resque.yml` settings to configure the Redis database instance
+used for `Rails.cache` and other volatile non-persistent data which enhances
+the performance of GitLab.
+Settings here can be overridden by the environment variable
+`GITLAB_REDIS_CACHE_CONFIG_FILE` which provides
+an alternate location for configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `cache` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
+2. URL from `redis.cache.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. URL from `resque.yml`
+5. `redis://localhost:6380`
+
+The order of precedence for all other configuration settings for `cache`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
+2. the configuration file `redis.cache.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. the configuration file `resque.yml`
+
+## redis.queues.yml
+
+If configured, `redis.queues.yml` overrides the
+`resque.yml` settings to configure the Redis database instance
+used for clients of `::Gitlab::Redis::Queues`.
+These queues are intended to be the foundation
+of reliable inter-process communication between modules, whether on the same
+host node, or within a cluster.   The primary clients of the queues are
+SideKiq, Mailroom, CI Runner, Workhorse, and push services.  Settings here can
+be overridden by the environment variable
+`GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for
+configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `queues` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
+2. URL from `redis.queues.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. URL from `resque.yml`
+5. `redis://localhost:6381`
+
+The order of precedence for all other configuration settings for `queues`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
+2. the configuration file `redis.queues.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. the configuration file `resque.yml`
+
+## redis.shared_state.yml
+
+If configured, `redis.shared_state.yml` overrides the
+`resque.yml` settings to configure the Redis database instance
+used for clients of `::Gitlab::Redis::SharedState` such as session state,
+and rate limiting.
+Settings here can be overridden by the environment variable
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides
+an alternate location for configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `shared_state` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
+2. URL from `redis.shared_state.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. URL from `resque.yml`
+5. `redis://localhost:6382`
+
+The order of precedence for all other configuration settings for `shared_state`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
+2. the configuration file `redis.shared_state.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. the configuration file `resque.yml`
+
diff --git a/config/application.rb b/config/application.rb
index 2f4e2624195..47887bf8596 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -6,7 +6,9 @@ Bundler.require(:default, Rails.env)
 
 module Gitlab
   class Application < Rails::Application
-    require_dependency Rails.root.join('lib/gitlab/redis')
+    require_dependency Rails.root.join('lib/gitlab/redis/cache')
+    require_dependency Rails.root.join('lib/gitlab/redis/queues')
+    require_dependency Rails.root.join('lib/gitlab/redis/shared_state')
     require_dependency Rails.root.join('lib/gitlab/request_context')
 
     # Settings in config/environments/* take precedence over those specified here.
@@ -21,13 +23,13 @@ module Gitlab
     # https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/engine.rb#L687
     # This is a nice reference article on autoloading/eager loading:
     # http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload
-    config.eager_load_paths.push(*%W(#{config.root}/lib
+    config.eager_load_paths.push(*%W[#{config.root}/lib
                                      #{config.root}/app/models/hooks
                                      #{config.root}/app/models/members
                                      #{config.root}/app/models/project_services
                                      #{config.root}/app/workers/concerns
                                      #{config.root}/app/services/concerns
-                                     #{config.root}/app/finders/concerns))
+                                     #{config.root}/app/finders/concerns])
 
     config.generators.templates.push("#{config.root}/generator_templates")
 
@@ -142,15 +144,15 @@ module Gitlab
       end
     end
 
-    # Use Redis caching across all environments
-    redis_config_hash = Gitlab::Redis.params
-    redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE
-    redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
+    # Use caching across all environments
+    caching_config_hash = Gitlab::Redis::Cache.params
+    caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
+    caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
     if Sidekiq.server? # threaded context
-      redis_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
-      redis_config_hash[:pool_timeout] = 1
+      caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
+      caching_config_hash[:pool_timeout] = 1
     end
-    config.cache_store = :redis_store, redis_config_hash
+    config.cache_store = :redis_store, caching_config_hash
 
     config.active_record.raise_in_transactional_callbacks = true
 
@@ -179,7 +181,11 @@ module Gitlab
         end
       end
 
+      # We add the MilestonesRoutingHelper because we know that this does not
+      # conflict with the methods defined in `project_url_helpers`, and we want
+      # these methods available in the same places.
       Gitlab::Routing.add_helpers(project_url_helpers)
+      Gitlab::Routing.add_helpers(MilestonesRoutingHelper)
     end
   end
 end
diff --git a/config/boot.rb b/config/boot.rb
index 02baeab29ab..f2830ae3166 100644
--- a/config/boot.rb
+++ b/config/boot.rb
@@ -4,14 +4,3 @@ require 'rubygems'
 ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
 
 require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
-
-begin
-  require 'bootsnap/setup'
-rescue SystemCallError => exception
-  $stderr.puts "WARNING: Bootsnap failed to setup: #{exception.message}"
-end
-
-# set default directory for multiproces metrics gathering
-if ENV['RAILS_ENV'] == 'development' || ENV['RAILS_ENV'] == 'test'
-  ENV['prometheus_multiproc_dir'] ||= 'tmp/prometheus_multiproc_dir'
-end
diff --git a/config/database.yml.mysql b/config/database.yml.mysql
index db1b712d3bc..eb71d3f5fe1 100644
--- a/config/database.yml.mysql
+++ b/config/database.yml.mysql
@@ -42,3 +42,4 @@ test: &test
   password:
   # host: localhost
   # socket: /tmp/mysql.sock
+  prepared_statements: false
diff --git a/config/database.yml.postgresql b/config/database.yml.postgresql
index c517a4c0cb8..4b30982fe82 100644
--- a/config/database.yml.postgresql
+++ b/config/database.yml.postgresql
@@ -46,3 +46,4 @@ test: &test
   username: postgres
   password:
   # host: localhost
+  prepared_statements: false
diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml
index 59c7050a14d..ca5b941aebf 100644
--- a/config/dependency_decisions.yml
+++ b/config/dependency_decisions.yml
@@ -398,3 +398,9 @@
     :why: https://github.com/remy/undefsafe/blob/master/LICENSE
     :versions: []
     :when: 2017-04-10 06:30:00.002555000 Z
+- - :approve
+  - thunky
+  - :who: Mike Greiling
+    :why: https://github.com/mafintosh/thunky/blob/master/README.md#license
+    :versions: []
+    :when: 2017-08-07 05:56:09.907045000 Z
diff --git a/config/environments/test.rb b/config/environments/test.rb
index c3b788c038e..278144b8943 100644
--- a/config/environments/test.rb
+++ b/config/environments/test.rb
@@ -43,4 +43,9 @@ Rails.application.configure do
   config.cache_store = :null_store
 
   config.active_job.queue_adapter = :test
+
+  if ENV['CI'] && !ENV['RAILS_ENABLE_TEST_LOG']
+    config.logger = ActiveSupport::TaggedLogging.new(Logger.new(nil))
+    config.log_level = :fatal
+  end
 end
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 28e9a5f420a..a9edde4d6e2 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -236,7 +236,8 @@ production: &base
   # ==========================
 
   ## LDAP settings
-  # You can inspect a sample of the LDAP users with login access by running:
+  # You can test connections and inspect a sample of the LDAP users with login
+  # access by running:
   #   bundle exec rake gitlab:ldap:check RAILS_ENV=production
   ldap:
     enabled: false
@@ -259,13 +260,45 @@ production: &base
         # Example: 'Paris' or 'Acme, Ltd.'
         label: 'LDAP'
 
+        # Example: 'ldap.mydomain.com'
         host: '_your_ldap_server'
-        port: 389
-        uid: 'sAMAccountName'
-        method: 'plain' # "tls" or "ssl" or "plain"
+        # This port is an example, it is sometimes different but it is always an integer and not a string
+        port: 389 # usually 636 for SSL
+        uid: 'sAMAccountName' # This should be the attribute, not the value that maps to uid.
+
+        # Examples: 'america\\momo' or 'CN=Gitlab Git,CN=Users,DC=mydomain,DC=com'
         bind_dn: '_the_full_dn_of_the_user_you_will_bind_with'
         password: '_the_password_of_the_bind_user'
 
+        # Encryption method. The "method" key is deprecated in favor of
+        # "encryption".
+        #
+        #   Examples: "start_tls" or "simple_tls" or "plain"
+        #
+        #   Deprecated values: "tls" was replaced with "start_tls" and "ssl" was
+        #   replaced with "simple_tls".
+        #
+        encryption: 'plain'
+
+        # Enables SSL certificate verification if encryption method is
+        # "start_tls" or "simple_tls". (Defaults to false for backward-
+        # compatibility)
+        verify_certificates: false
+
+        # Specifies the path to a file containing a PEM-format CA certificate,
+        # e.g. if you need to use an internal CA.
+        #
+        #   Example: '/etc/ca.pem'
+        #
+        ca_file: ''
+
+        # Specifies the SSL version for OpenSSL to use, if the OpenSSL default
+        # is not appropriate.
+        #
+        #   Example: 'TLSv1_1'
+        #
+        ssl_version: ''
+
         # Set a timeout, in seconds, for LDAP queries. This helps avoid blocking
         # a request if the LDAP server becomes unresponsive.
         # A value of 0 means there is no timeout.
@@ -294,17 +327,20 @@ production: &base
 
         # Base where we can search for users
         #
-        #   Ex. ou=People,dc=gitlab,dc=example
+        #   Ex. 'ou=People,dc=gitlab,dc=example' or 'DC=mydomain,DC=com'
         #
         base: ''
 
         # Filter LDAP users
         #
-        #   Format: RFC 4515 http://tools.ietf.org/search/rfc4515
+        #   Format: RFC 4515 https://tools.ietf.org/search/rfc4515
         #   Ex. (employeeType=developer)
         #
         #   Note: GitLab does not support omniauth-ldap's custom filter syntax.
         #
+        #   Example for getting only specific users:
+        #   '(&(objectclass=user)(|(samaccountname=momo)(samaccountname=toto)))'
+        #
         user_filter: ''
 
         # LDAP attributes that GitLab will use to create an account for the LDAP user.
@@ -391,13 +427,13 @@ production: &base
       #             service_validate_url: '/cas/p3/serviceValidate',
       #             logout_url: '/cas/logout'} }
       # - { name: 'authentiq',
-      #     # for client credentials (client ID and secret), go to https://www.authentiq.com/
+      #     # for client credentials (client ID and secret), go to https://www.authentiq.com/developers
       #     app_id: 'YOUR_CLIENT_ID',
       #     app_secret: 'YOUR_CLIENT_SECRET',
       #     args: {
       #             scope: 'aq:name email~rs address aq:push'
-      #             # redirect_uri parameter is optional except when 'gitlab.host' in this file is set to 'localhost'
-      #             # redirect_uri: 'YOUR_REDIRECT_URI'
+      #             # callback_url parameter is optional except when 'gitlab.host' in this file is set to 'localhost'
+      #             # callback_url: 'YOUR_CALLBACK_URL'
       #           }
       #   }
       # - { name: 'github',
@@ -478,6 +514,11 @@ production: &base
         path: /home/git/repositories/
         gitaly_address: unix:/home/git/gitlab/tmp/sockets/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port)
         # gitaly_token: 'special token' # Optional: override global gitaly.token for this storage.
+        failure_count_threshold: 10 # number of failures before stopping attempts
+        failure_wait_time: 30 # Seconds after an access failure before allowing access again
+        failure_reset_time: 1800 # Time in seconds to expire failures
+        storage_timeout: 5 # Time in seconds to wait before aborting a storage access attempt
+
 
   ## Backup settings
   backup:
@@ -547,10 +588,21 @@ production: &base
     #   enabled: true
     #   host: localhost
     #   port: 3808
-  prometheus:
+
+  ## Monitoring
+  # Built in monitoring settings
+  monitoring:
     # Time between sampling of unicorn socket metrics, in seconds
     # unicorn_sampler_interval: 10
+    # IP whitelist to access monitoring endpoints
+    ip_whitelist:
+      - 127.0.0.0/8
 
+    # Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics
+    sidekiq_exporter:
+    #  enabled: true
+    #  address: localhost
+    #  port: 3807
 
   #
   # 5. Extra customization
@@ -605,6 +657,10 @@ test:
       default:
         path: tmp/tests/repositories/
         gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
+      broken:
+        path: tmp/tests/non-existent-repositories
+        gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
+
   gitaly:
     enabled: true
     token: secret
@@ -677,7 +733,7 @@ test:
         host: 127.0.0.1
         port: 3890
         uid: 'uid'
-        method: 'plain' # "tls" or "ssl" or "plain"
+        encryption: 'plain' # "start_tls" or "simple_tls" or "plain"
         base: 'dc=example,dc=com'
         user_filter: ''
         group_base: 'ou=groups,dc=example,dc=com'
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 319af2e0b66..d804fb16167 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -145,6 +145,24 @@ if Settings.ldap['enabled'] || Rails.env.test?
     server['attributes'] = {} if server['attributes'].nil?
     server['provider_name'] ||= "ldap#{key}".downcase
     server['provider_class'] = OmniAuth::Utils.camelize(server['provider_name'])
+
+    # For backwards compatibility
+    server['encryption'] ||= server['method']
+    server['encryption'] = 'simple_tls' if server['encryption'] == 'ssl'
+    server['encryption'] = 'start_tls' if server['encryption'] == 'tls'
+
+    # Certificates are not verified for backwards compatibility.
+    # This default should be flipped to true in 9.5.
+    if server['verify_certificates'].nil?
+      server['verify_certificates'] = false
+
+      message = <<-MSG.strip_heredoc
+        LDAP SSL certificate verification is disabled for backwards-compatibility.
+        Please add the "verify_certificates" option to gitlab.yml for each LDAP
+        server. Certificate verification will be enabled by default in GitLab 9.5.
+      MSG
+      Rails.logger.warn(message)
+    end
   end
 end
 
@@ -205,7 +223,7 @@ Settings.gitlab['default_can_create_group'] = true if Settings.gitlab['default_c
 Settings.gitlab['host']       ||= ENV['GITLAB_HOST'] || 'localhost'
 Settings.gitlab['ssh_host']   ||= Settings.gitlab.host
 Settings.gitlab['https']        = false if Settings.gitlab['https'].nil?
-Settings.gitlab['port']       ||= Settings.gitlab.https ? 443 : 80
+Settings.gitlab['port']       ||= ENV['GITLAB_PORT'] || (Settings.gitlab.https ? 443 : 80)
 Settings.gitlab['relative_url_root'] ||= ENV['RAILS_RELATIVE_URL_ROOT'] || ''
 Settings.gitlab['protocol'] ||= Settings.gitlab.https ? "https" : "http"
 Settings.gitlab['email_enabled'] ||= true if Settings.gitlab['email_enabled'].nil?
@@ -223,7 +241,7 @@ rescue ArgumentError # no user configured
 end
 Settings.gitlab['time_zone'] ||= nil
 Settings.gitlab['signup_enabled'] ||= true if Settings.gitlab['signup_enabled'].nil?
-Settings.gitlab['signin_enabled'] ||= true if Settings.gitlab['signin_enabled'].nil?
+Settings.gitlab['password_authentication_enabled'] ||= true if Settings.gitlab['password_authentication_enabled'].nil?
 Settings.gitlab['restricted_visibility_levels'] = Settings.__send__(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], [])
 Settings.gitlab['username_changing_enabled'] = true if Settings.gitlab['username_changing_enabled'].nil?
 Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing))(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil?
@@ -383,6 +401,10 @@ Settings.cron_jobs['remove_old_web_hook_logs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['remove_old_web_hook_logs_worker']['cron'] ||= '40 0 * * *'
 Settings.cron_jobs['remove_old_web_hook_logs_worker']['job_class'] = 'RemoveOldWebHookLogsWorker'
 
+Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
+Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
+
 #
 # GitLab Shell
 #
@@ -421,6 +443,17 @@ end
 Settings.repositories.storages.values.each do |storage|
   # Expand relative paths
   storage['path'] = Settings.absolute(storage['path'])
+  # Set failure defaults
+  storage['failure_count_threshold'] ||= 10
+  storage['failure_wait_time'] ||= 30
+  storage['failure_reset_time'] ||= 1800
+  storage['storage_timeout'] ||= 5
+  # Set turn strings into numbers
+  storage['failure_count_threshold'] = storage['failure_count_threshold'].to_i
+  storage['failure_wait_time'] = storage['failure_wait_time'].to_i
+  storage['failure_reset_time'] = storage['failure_reset_time'].to_i
+  # We might want to have a timeout shorter than 1 second.
+  storage['storage_timeout'] = storage['storage_timeout'].to_f
 end
 
 #
@@ -447,10 +480,6 @@ Settings.backup['pg_schema']    = nil
 Settings.backup['path']         = Settings.absolute(Settings.backup['path'] || "tmp/backups/")
 Settings.backup['archive_permissions'] ||= 0600
 Settings.backup['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'connection' => nil })
-# Convert upload connection settings to use symbol keys, to make Fog happy
-if Settings.backup['upload']['connection']
-  Settings.backup['upload']['connection'] = Hash[Settings.backup['upload']['connection'].map { |k, v| [k.to_sym, v] }]
-end
 Settings.backup['upload']['multipart_chunk_size'] ||= 104857600
 Settings.backup['upload']['encryption'] ||= nil
 Settings.backup['upload']['storage_class'] ||= nil
@@ -500,10 +529,15 @@ Settings.webpack.dev_server['host']    ||= 'localhost'
 Settings.webpack.dev_server['port']    ||= 3808
 
 #
-# Prometheus metrics settings
+# Monitoring settings
 #
-Settings['prometheus'] ||= Settingslogic.new({})
-Settings.prometheus['unicorn_sampler_interval'] ||= 10
+Settings['monitoring'] ||= Settingslogic.new({})
+Settings.monitoring['ip_whitelist'] ||= ['127.0.0.1/8']
+Settings.monitoring['unicorn_sampler_interval'] ||= 10
+Settings.monitoring['sidekiq_exporter'] ||= Settingslogic.new({})
+Settings.monitoring.sidekiq_exporter['enabled'] ||= false
+Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost'
+Settings.monitoring.sidekiq_exporter['port'] ||= 3807
 
 #
 # Testing settings
diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb
index 9e24f42d284..92ce4dd03cd 100644
--- a/config/initializers/6_validations.rb
+++ b/config/initializers/6_validations.rb
@@ -7,6 +7,13 @@ def find_parent_path(name, path)
   Gitlab.config.repositories.storages.detect do |n, rs|
     name != n && Pathname.new(rs['path']).realpath == parent
   end
+rescue Errno::EIO, Errno::ENOENT => e
+  warning = "WARNING: couldn't verify #{path} (#{name}). "\
+            "If this is an external storage, it might be offline."
+  message = "#{warning}\n#{e.message}"
+  Rails.logger.error("#{message}\n\t" + e.backtrace.join("\n\t"))
+
+  nil
 end
 
 def storage_validation_error(message)
@@ -29,6 +36,15 @@ def validate_storages_config
     if !repository_storage.is_a?(Hash) || repository_storage['path'].nil?
       storage_validation_error("#{name} is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example")
     end
+
+    %w(failure_count_threshold failure_wait_time failure_reset_time storage_timeout).each do |setting|
+      # Falling back to the defaults is fine!
+      next if repository_storage[setting].nil?
+
+      unless repository_storage[setting].to_f > 0
+        storage_validation_error("#{setting}, for storage `#{name}` needs to be greater than 0")
+      end
+    end
   end
 end
 
diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
new file mode 100644
index 00000000000..54c797e0714
--- /dev/null
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -0,0 +1,18 @@
+require 'prometheus/client'
+
+Prometheus::Client.configure do |config|
+  config.logger = Rails.logger
+
+  config.initial_mmap_file_size = 4 * 1024
+  config.multiprocess_files_dir = ENV['prometheus_multiproc_dir']
+
+  if Rails.env.development? || Rails.env.test?
+    config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir')
+  end
+end
+
+Sidekiq.configure_server do |config|
+  config.on(:startup) do
+    Gitlab::Metrics::SidekiqMetricsExporter.instance.start
+  end
+end
diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb
index ae2ca258df1..af4967521b8 100644
--- a/config/initializers/7_redis.rb
+++ b/config/initializers/7_redis.rb
@@ -1,3 +1,8 @@
-# Make sure we initialize a Redis connection pool before Sidekiq starts
-# multi-threaded execution.
-Gitlab::Redis.with { nil }
+# Make sure we initialize a Redis connection pool before multi-threaded
+# execution starts by
+# 1. Sidekiq
+# 2. Rails.cache
+# 3. HTTP clients
+Gitlab::Redis::Cache.with { nil }
+Gitlab::Redis::Queues.with { nil }
+Gitlab::Redis::SharedState.with { nil }
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index d56fd7a6cfa..2aeb94d47cd 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -114,16 +114,19 @@ def instrument_classes(instrumentation)
   # This is a Rails scope so we have to instrument it manually.
   instrumentation.instrument_method(Project, :visible_to_user)
 
+  # Needed for https://gitlab.com/gitlab-org/gitlab-ce/issues/34509
+  instrumentation.instrument_method(MarkupHelper, :link_to_gfm)
+
   # Needed for https://gitlab.com/gitlab-org/gitlab-ce/issues/30224#note_32306159
   instrumentation.instrument_instance_method(MergeRequestDiff, :load_commits)
 end
 # rubocop:enable Metrics/AbcSize
 
-Gitlab::Metrics::UnicornSampler.initialize_instance(Settings.prometheus.unicorn_sampler_interval).start
+Gitlab::Metrics::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start
 
 Gitlab::Application.configure do |config|
   # 0 should be Sentry to catch errors in this middleware
-  config.middleware.insert(1, Gitlab::Metrics::ConnectionRackMiddleware)
+  config.middleware.insert(1, Gitlab::Metrics::RequestsRackMiddleware)
 end
 
 if Gitlab::Metrics.enabled?
@@ -174,6 +177,10 @@ if Gitlab::Metrics.enabled?
           loc && loc[0].start_with?(models) && method.source =~ regex
         end
       end
+
+    # Ability is in app/models, is not an ActiveRecord model, but should still
+    # be instrumented.
+    Gitlab::Metrics::Instrumentation.instrument_methods(Ability)
   end
 
   Gitlab::Metrics::Instrumentation.configure do |config|
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index 3b1317030bc..3aed2136f1b 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -192,7 +192,7 @@ Devise.setup do |config|
   #
   # The :"*/*" and "*/*" formats below is required to match Internet
   # Explorer requests.
-  # config.navigational_formats = [:"*/*", "*/*", :html]
+  config.navigational_formats = [:"*/*", "*/*", :html, :zip]
 
   # The default HTTP method used to sign out a resource. Default is :delete.
   config.sign_out_via = :delete
@@ -206,11 +206,11 @@ Devise.setup do |config|
   # If you want to use other strategies, that are not supported by Devise, or
   # change the failure app, you can configure them inside the config.warden block.
   #
-  config.warden do |manager|
-    manager.failure_app = Gitlab::DeviseFailure
-    # manager.intercept_401 = false
-    # manager.default_strategies(scope: :user).unshift :some_external_strategy
-  end
+  # config.warden do |manager|
+  #   manager.failure_app = Gitlab::DeviseFailure
+  #   manager.intercept_401 = false
+  #   manager.default_strategies(scope: :user).unshift :some_external_strategy
+  # end
 
   if Gitlab::LDAP::Config.enabled?
     Gitlab::LDAP::Config.providers.each do |provider|
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index 8e2e639fc41..40e635bf2cf 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -92,9 +92,9 @@ Doorkeeper.configure do
   # Under some circumstances you might want to have applications auto-approved,
   # so that the user skips the authorization step.
   # For example if dealing with trusted a application.
-  # skip_authorization do |resource_owner, client|
-  #   client.superapp? or resource_owner.admin?
-  # end
+  skip_authorization do |resource_owner, client|
+    client.application.trusted?
+  end
 
   # WWW-Authenticate Realm (default "Doorkeeper").
   # realm "Doorkeeper"
diff --git a/config/initializers/flipper.rb b/config/initializers/flipper.rb
index 8ec9613a4b7..bfab8c77a4b 100644
--- a/config/initializers/flipper.rb
+++ b/config/initializers/flipper.rb
@@ -3,4 +3,6 @@ require 'flipper/middleware/memoizer'
 unless Rails.env.test?
   Rails.application.config.middleware.use Flipper::Middleware::Memoizer,
     lambda { Feature.flipper }
+
+  Feature.register_feature_groups
 end
diff --git a/config/initializers/gettext_rails_i18n_patch.rb b/config/initializers/gettext_rails_i18n_patch.rb
index 69118f464ca..377e5104f9d 100644
--- a/config/initializers/gettext_rails_i18n_patch.rb
+++ b/config/initializers/gettext_rails_i18n_patch.rb
@@ -33,7 +33,6 @@ module GettextI18nRailsJs
         [
           ".js",
           ".jsx",
-          ".coffee",
           ".vue"
         ].include? ::File.extname(file)
       end
diff --git a/config/initializers/grape_route_helpers_fix.rb b/config/initializers/grape_route_helpers_fix.rb
new file mode 100644
index 00000000000..d3cf9e453d0
--- /dev/null
+++ b/config/initializers/grape_route_helpers_fix.rb
@@ -0,0 +1,35 @@
+if defined?(GrapeRouteHelpers)
+  module GrapeRouteHelpers
+    class DecoratedRoute
+      # GrapeRouteHelpers gem tries to parse the versions
+      # from a string, not supporting Grape `version` array definition.
+      #
+      # Without the following fix, we get this on route helpers generation:
+      #
+      # => undefined method `scan' for ["v3", "v4"]
+      #
+      # 2.0.0 implementation of this method:
+      #
+      # ```
+      # def route_versions
+      #   version_pattern = /[^\[",\]\s]+/
+      #   if route_version
+      #     route_version.scan(version_pattern)
+      #   else
+      #     [nil]
+      #   end
+      # end
+      # ```
+      def route_versions
+        return [nil] if route_version.nil? || route_version.empty?
+
+        if route_version.is_a?(String)
+          version_pattern = /[^\[",\]\s]+/
+          route_version.scan(version_pattern)
+        else
+          route_version
+        end
+      end
+    end
+  end
+end
diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb
new file mode 100644
index 00000000000..21fe8d72459
--- /dev/null
+++ b/config/initializers/lograge.rb
@@ -0,0 +1,24 @@
+# Only use Lograge for Rails
+unless Sidekiq.server?
+  filename = File.join(Rails.root, 'log', "#{Rails.env}_json.log")
+
+  Rails.application.configure do
+    config.lograge.enabled = true
+    # Store the lograge JSON files in a separate file
+    config.lograge.keep_original_rails_log = true
+    # Don't use the Logstash formatter since this requires logstash-event, an
+    # unmaintained gem that monkey patches `Time`
+    config.lograge.formatter = Lograge::Formatters::Json.new
+    config.lograge.logger = ActiveSupport::Logger.new(filename)
+    # Add request parameters to log output
+    config.lograge.custom_options = lambda do |event|
+      {
+        time: event.time.utc.iso8601(3),
+        params: event.payload[:params].except(*%w(controller action format)),
+        remote_ip: event.payload[:remote_ip],
+        user_id: event.payload[:user_id],
+        username: event.payload[:username]
+      }
+    end
+  end
+end
diff --git a/config/initializers/mysql_set_length_for_binary_indexes.rb b/config/initializers/mysql_set_length_for_binary_indexes.rb
new file mode 100644
index 00000000000..de0bc5322aa
--- /dev/null
+++ b/config/initializers/mysql_set_length_for_binary_indexes.rb
@@ -0,0 +1,21 @@
+# This patches ActiveRecord so indexes for binary columns created using the
+# MySQL adapter apply a length of 20. Otherwise MySQL can't create an index on
+# binary columns.
+
+module MysqlSetLengthForBinaryIndex
+  def add_index(table_name, column_names, options = {})
+    Array(column_names).each do |column_name|
+      column = ActiveRecord::Base.connection.columns(table_name).find { |c| c.name == column_name }
+
+      if column&.type == :binary
+        options[:length] = 20
+      end
+    end
+
+    super(table_name, column_names, options)
+  end
+end
+
+if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter)
+  ActiveRecord::ConnectionAdapters::Mysql2Adapter.send(:prepend, MysqlSetLengthForBinaryIndex)
+end
diff --git a/config/initializers/omniauth.rb b/config/initializers/omniauth.rb
index f7fa6d1c2de..56c279ffcf4 100644
--- a/config/initializers/omniauth.rb
+++ b/config/initializers/omniauth.rb
@@ -1,13 +1,16 @@
 if Gitlab::LDAP::Config.enabled?
   module OmniAuth::Strategies
-    server = Gitlab.config.ldap.servers.values.first
-    klass = server['provider_class']
-    const_set(klass, Class.new(LDAP)) unless klass == 'LDAP'
+    Gitlab::LDAP::Config.available_servers.each do |server|
+      # do not redeclare LDAP
+      next if server['provider_name'] == 'ldap'
+      const_set(server['provider_class'], Class.new(LDAP))
+    end
   end
 
   OmniauthCallbacksController.class_eval do
-    server = Gitlab.config.ldap.servers.values.first
-    alias_method server['provider_name'], :ldap
+    Gitlab::LDAP::Config.available_servers.each do |server|
+      alias_method server['provider_name'], :ldap
+    end
   end
 end
 
@@ -16,7 +19,7 @@ OmniAuth.config.allowed_request_methods = [:post]
 # In case of auto sign-in, the GET method is used (users don't get to click on a button)
 OmniAuth.config.allowed_request_methods << :get if Gitlab.config.omniauth.auto_sign_in_with_provider.present?
 OmniAuth.config.before_request_phase do |env|
-  OmniAuth::RequestForgeryProtection.call(env)
+  Gitlab::RequestForgeryProtection.call(env)
 end
 
 if Gitlab.config.omniauth.enabled
diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb
index 65432caac2a..a54d53cbbe2 100644
--- a/config/initializers/peek.rb
+++ b/config/initializers/peek.rb
@@ -1,4 +1,4 @@
-Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.params) }
+Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Cache.params) }
 
 Peek.into Peek::Views::Host
 Peek.into Peek::Views::PerformanceBar
@@ -26,7 +26,3 @@ class PEEK_DB_CLIENT
 end
 
 PEEK_DB_VIEW.prepend ::Gitlab::PerformanceBar::PeekQueryTracker
-
-class Peek::Views::PerformanceBar::ProcessUtilization
-  prepend ::Gitlab::PerformanceBar::PeekPerformanceBarWithRackBody
-end
diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb
index 8919f7640fe..e8213ac8ba4 100644
--- a/config/initializers/session_store.rb
+++ b/config/initializers/session_store.rb
@@ -19,12 +19,12 @@ cookie_key = if Rails.env.development?
 if Rails.env.test?
   Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
 else
-  redis_config = Gitlab::Redis.params
-  redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE
+  sessions_config = Gitlab::Redis::SharedState.params
+  sessions_config[:namespace] = Gitlab::Redis::SharedState::SESSION_NAMESPACE
 
   Gitlab::Application.config.session_store(
     :redis_store, # Using the cookie_store would enable session replay attacks.
-    servers: redis_config,
+    servers: sessions_config,
     key: cookie_key,
     secure: Gitlab.config.gitlab.https,
     httponly: true,
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index ecd73956488..a1cc9655319 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -1,12 +1,12 @@
-# Custom Redis configuration
-redis_config_hash = Gitlab::Redis.params
-redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
+# Custom Queues configuration
+queues_config_hash = Gitlab::Redis::Queues.params
+queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE
 
 # Default is to retry 25 times with exponential backoff. That's too much.
 Sidekiq.default_worker_options = { retry: 3 }
 
 Sidekiq.configure_server do |config|
-  config.redis = redis_config_hash
+  config.redis = queues_config_hash
 
   config.server_middleware do |chain|
     chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
@@ -54,7 +54,7 @@ Sidekiq.configure_server do |config|
 end
 
 Sidekiq.configure_client do |config|
-  config.redis = redis_config_hash
+  config.redis = queues_config_hash
 
   config.client_middleware do |chain|
     chain.add Gitlab::SidekiqStatus::ClientMiddleware
@@ -74,5 +74,5 @@ begin
       end
     end
   end
-rescue Redis::BaseError, SocketError, Errno::ENOENT, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED
+rescue Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED
 end
diff --git a/config/mail_room.yml b/config/mail_room.yml
index 88d93d4bc6b..c3a5be8d38c 100644
--- a/config/mail_room.yml
+++ b/config/mail_room.yml
@@ -21,7 +21,7 @@
       :delivery_method: sidekiq
       :delivery_options:
         :redis_url: <%= config[:redis_url].to_json %>
-        :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %>
+        :namespace: <%= Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE %>
         :queue: email_receiver
         :worker: EmailReceiverWorker
         <% if config[:sentinels] %>
@@ -36,7 +36,7 @@
       :arbitration_method: redis
       :arbitration_options:
         :redis_url: <%= config[:redis_url].to_json %>
-        :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %>
+        :namespace: <%= Gitlab::Redis::Queues::MAILROOM_NAMESPACE %>
         <% if config[:sentinels] %>
         :sentinels:
           <% config[:sentinels].each do |sentinel| %>
diff --git a/config/prometheus/additional_metrics.yml b/config/prometheus/additional_metrics.yml
index d33fae4182d..5eb01d62924 100644
--- a/config/prometheus/additional_metrics.yml
+++ b/config/prometheus/additional_metrics.yml
@@ -1,4 +1,25 @@
-- group: AWS Elastic Load Balancer
+- group: Response metrics (HA Proxy)
+  priority: 10
+  metrics:
+  - title: "Throughput"
+    y_label: "Requests / Sec"
+    required_metrics:
+      - haproxy_frontend_http_requests_total
+    weight: 1
+    queries:
+    - query_range: 'sum(rate(haproxy_frontend_http_requests_total{%{environment_filter}}[2m]))'
+      label: Total
+      unit: req / sec
+  - title: "HTTP Error Rate"
+    y_label: "Error Rate (%)"
+    required_metrics:
+      - haproxy_frontend_http_responses_total
+    weight: 1
+    queries:
+    - query_range: 'sum(rate(haproxy_frontend_http_responses_total{code="5xx",%{environment_filter}}[2m])) / sum(rate(haproxy_frontend_http_responses_total{%{environment_filter}}[2m]))'
+      label: HTTP Errors
+      unit: "%"
+- group: Response metrics (AWS ELB)
   priority: 10
   metrics:
   - title: "Throughput"
@@ -7,7 +28,7 @@
       - aws_elb_request_count_sum
     weight: 1
     queries:
-    - query_range: 'sum(aws_elb_request_count_sum{%{environment_filter}}) * 60'
+    - query_range: 'sum(aws_elb_request_count_sum{%{environment_filter}}) / 60'
       label: Total
       unit: req / sec
   - title: "Latency"
@@ -29,7 +50,7 @@
     - query_range: 'sum(aws_elb_httpcode_backend_5_xx_sum{%{environment_filter}}) / sum(aws_elb_request_count_sum{%{environment_filter}})'
       label: HTTP Errors
       unit: "%"
-- group: NGINX
+- group: Response metrics (NGINX)
   priority: 10
   metrics:
   - title: "Throughput"
@@ -56,10 +77,10 @@
       - nginx_responses_total
     weight: 1
     queries:
-    - query_range: 'sum(nginx_responses_total{status_code="5xx", %{environment_filter}}) / sum(nginx_responses_total{server_zone!="*", server_zone!="_", %{environment_filter}})'
+    - query_range: 'sum(rate(nginx_responses_total{status_code="5xx", %{environment_filter}}[2m])) / sum(rate(nginx_requests_total{server_zone!="*", server_zone!="_", %{environment_filter}}[2m]))'
       label: HTTP Errors
       unit: "%"
-- group: Kubernetes
+- group: System metrics (Kubernetes)
   priority: 5
   metrics:
   - title: "Memory Usage"
diff --git a/config/redis.cache.yml.example b/config/redis.cache.yml.example
new file mode 100644
index 00000000000..27478f0a93e
--- /dev/null
+++ b/config/redis.cache.yml.example
@@ -0,0 +1,38 @@
+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+  url: redis://localhost:6379/10
+  #
+  # url: redis://localhost:6380
+  # sentinels:
+  #   -
+  #     host: localhost
+  #     port: 26380 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26380 # point to sentinel, not to redis port
+test:
+  url: redis://localhost:6379/10
+  #
+  # url: redis://localhost:6380
+production:
+  # Redis (single instance)
+  url: unix:/var/run/redis/redis.cache.sock
+  ##
+  # Redis + Sentinel (for HA)
+  #
+  # Please read instructions carefully before using it as you may lose data:
+  # http://redis.io/topics/sentinel
+  #
+  # You must specify a list of a few sentinels that will handle client connection
+  # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+  ##
+  # url: redis://master:6380
+  # sentinels:
+  #   -
+  #     host: slave1
+  #     port: 26380 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26380 # point to sentinel, not to redis port
diff --git a/config/redis.queues.yml.example b/config/redis.queues.yml.example
new file mode 100644
index 00000000000..dab1f26b096
--- /dev/null
+++ b/config/redis.queues.yml.example
@@ -0,0 +1,38 @@
+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+  url: redis://localhost:6379/11
+  #
+  # url: redis://localhost:6381
+  # sentinels:
+  #   -
+  #     host: localhost
+  #     port: 26381 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26381 # point to sentinel, not to redis port
+test:
+  url: redis://localhost:6379/11
+  #
+  # url: redis://localhost:6381
+production:
+  # Redis (single instance)
+  url: unix:/var/run/redis/redis.queues.sock
+  ##
+  # Redis + Sentinel (for HA)
+  #
+  # Please read instructions carefully before using it as you may lose data:
+  # http://redis.io/topics/sentinel
+  #
+  # You must specify a list of a few sentinels that will handle client connection
+  # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+  ##
+  # url: redis://master:6381
+  # sentinels:
+  #   -
+  #     host: slave1
+  #     port: 26381 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26381 # point to sentinel, not to redis port
diff --git a/config/redis.shared_state.yml.example b/config/redis.shared_state.yml.example
new file mode 100644
index 00000000000..9371e3619b7
--- /dev/null
+++ b/config/redis.shared_state.yml.example
@@ -0,0 +1,38 @@
+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+  url: redis://localhost:6379/12
+  #
+  # url: redis://localhost:6382
+  # sentinels:
+  #   -
+  #     host: localhost
+  #     port: 26382 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26382 # point to sentinel, not to redis port
+test:
+  url: redis://localhost:6379/12
+  #
+  # url: redis://localhost:6382
+production:
+  # Redis (single instance)
+  url: unix:/var/run/redis/redis.shared_state.sock
+  ##
+  # Redis + Sentinel (for HA)
+  #
+  # Please read instructions carefully before using it as you may lose data:
+  # http://redis.io/topics/sentinel
+  #
+  # You must specify a list of a few sentinels that will handle client connection
+  # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+  ##
+  # url: redis://master:6382
+  # sentinels:
+  #   -
+  #     host: slave1
+  #     port: 26382 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26382 # point to sentinel, not to redis port
diff --git a/config/routes/admin.rb b/config/routes/admin.rb
index 5427bab93ce..c0748231813 100644
--- a/config/routes/admin.rb
+++ b/config/routes/admin.rb
@@ -67,7 +67,9 @@ namespace :admin do
   end
 
   resource :logs, only: [:show]
-  resource :health_check, controller: 'health_check', only: [:show]
+  resource :health_check, controller: 'health_check', only: [:show] do
+    post :reset_storage_health
+  end
   resource :background_jobs, controller: 'background_jobs', only: [:show]
   resource :system_info, controller: 'system_info', only: [:show]
   resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ }
diff --git a/config/routes/api.rb b/config/routes/api.rb
index 69c8efc151c..ce7a7c88900 100644
--- a/config/routes/api.rb
+++ b/config/routes/api.rb
@@ -1,2 +1,2 @@
 API::API.logger Rails.logger
-mount API::API => '/api'
+mount API::API => '/'
diff --git a/config/routes/profile.rb b/config/routes/profile.rb
index 3dc890e5785..3e4e6111ab8 100644
--- a/config/routes/profile.rb
+++ b/config/routes/profile.rb
@@ -23,6 +23,11 @@ resource :profile, only: [:show, :update] do
     end
     resource :preferences, only: [:show, :update]
     resources :keys, only: [:index, :show, :create, :destroy]
+    resources :gpg_keys, only: [:index, :create, :destroy] do
+      member do
+        put :revoke
+      end
+    end
     resources :emails, only: [:index, :create, :destroy]
     resources :chat_names, only: [:index, :new, :create, :destroy] do
       collection do
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 62cab25c763..06928c7b9ce 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -272,7 +272,7 @@ constraints(ProjectUrlConstrainer.new) do
       namespace :registry do
         resources :repository, only: [] do
           resources :tags, only: [:destroy],
-                           constraints: { id: Gitlab::Regex.container_registry_reference_regex }
+                           constraints: { id: Gitlab::Regex.container_registry_tag_regex }
         end
       end
 
@@ -379,7 +379,9 @@ constraints(ProjectUrlConstrainer.new) do
         collection do
           scope '*ref', constraints: { ref: Gitlab::PathRegex.git_reference_regex } do
             constraints format: /svg/ do
-              get :build
+              # Keep around until 10.0, see gitlab-org/gitlab-ce#35307
+              get :build, to: "badges#pipeline"
+              get :pipeline
               get :coverage
             end
           end
diff --git a/config/routes/repository.rb b/config/routes/repository.rb
index 11911636fa7..2ba16035ece 100644
--- a/config/routes/repository.rb
+++ b/config/routes/repository.rb
@@ -2,7 +2,7 @@
 
 resource :repository, only: [:create] do
   member do
-    get 'archive', constraints: { format: Gitlab::PathRegex.archive_formats_regex }
+    get ':ref/archive', constraints: { format: Gitlab::PathRegex.archive_formats_regex, ref: /.+/ }, action: 'archive', as: 'archive'
   end
 end
 
@@ -76,6 +76,8 @@ scope format: false do
     get '/tree/*id', to: 'tree#show', as: :tree
     get '/raw/*id', to: 'raw#show', as: :raw
     get '/blame/*id', to: 'blame#show', as: :blame
+
+    get '/commits/*id/signatures', to: 'commits#signatures', as: :signatures
     get '/commits/*id', to: 'commits#show', as: :commits
 
     post '/create_dir/*id', to: 'tree#create_dir', as: :create_dir
diff --git a/config/routes/uploads.rb b/config/routes/uploads.rb
index a49e244af1a..e9c9aa8b2f9 100644
--- a/config/routes/uploads.rb
+++ b/config/routes/uploads.rb
@@ -1,21 +1,21 @@
 scope path: :uploads do
   # Note attachments and User/Group/Project avatars
-  get "system/:model/:mounted_as/:id/:filename",
+  get "-/system/:model/:mounted_as/:id/:filename",
       to:           "uploads#show",
       constraints:  { model: /note|user|group|project/, mounted_as: /avatar|attachment/, filename: /[^\/]+/ }
 
   # show uploads for models, snippets (notes) available for now
-  get ':model/:id/:secret/:filename',
+  get 'system/:model/:id/:secret/:filename',
     to: 'uploads#show',
     constraints: { model: /personal_snippet/, id: /\d+/, filename: /[^\/]+/ }
 
   # show temporary uploads
-  get 'temp/:secret/:filename',
+  get 'system/temp/:secret/:filename',
     to: 'uploads#show',
     constraints: { filename: /[^\/]+/ }
 
   # Appearance
-  get "system/:model/:mounted_as/:id/:filename",
+  get "-/system/:model/:mounted_as/:id/:filename",
       to:           "uploads#show",
       constraints:  { model: /appearance/, mounted_as: /logo|header_logo/, filename: /.+/ }
 
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 1d9e69a2408..83abc83c9f0 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -23,12 +23,16 @@
   - [update_merge_requests, 3]
   - [process_commit, 3]
   - [new_note, 2]
+  - [new_issue, 2]
+  - [new_merge_request, 2]
   - [build, 2]
   - [pipeline, 2]
   - [gitlab_shell, 2]
   - [email_receiver, 2]
   - [emails_on_push, 2]
   - [mailers, 2]
+  - [invalid_gpg_signature_update, 2]
+  - [create_gpg_signature, 2]
   - [upload_checksum, 1]
   - [use_key, 1]
   - [repository_fork, 1]
diff --git a/config/unicorn.rb.example b/config/unicorn.rb.example
index 40a16a32359..cc10da2bd88 100644
--- a/config/unicorn.rb.example
+++ b/config/unicorn.rb.example
@@ -121,6 +121,10 @@ after_fork do |server, worker|
   defined?(ActiveRecord::Base) and
     ActiveRecord::Base.establish_connection
 
+  # reset prometheus client, this will cause any opened metrics files to be closed
+  defined?(::Prometheus::Client.reinitialize_on_pid_change) &&
+    Prometheus::Client.reinitialize_on_pid_change
+
   # if preload_app is true, then you may also want to check and
   # restart any other shared sockets/descriptors such as Memcached,
   # and Redis.  TokyoCabinet file handles are safe to reuse
diff --git a/config/webpack.config.js b/config/webpack.config.js
index c3fdca59a86..8e1b80cd39f 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -3,7 +3,8 @@
 var fs = require('fs');
 var path = require('path');
 var webpack = require('webpack');
-var StatsPlugin = require('stats-webpack-plugin');
+var StatsWriterPlugin = require('webpack-stats-plugin').StatsWriterPlugin;
+var CopyWebpackPlugin = require('copy-webpack-plugin');
 var CompressionPlugin = require('compression-webpack-plugin');
 var NameAllModulesPlugin = require('name-all-modules-plugin');
 var BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
@@ -39,9 +40,12 @@ var config = {
     environments_folder:  './environments/folder/environments_folder_bundle.js',
     filtered_search:      './filtered_search/filtered_search_bundle.js',
     graphs:               './graphs/graphs_bundle.js',
+    graphs_charts:        './graphs/graphs_charts.js',
+    graphs_show:          './graphs/graphs_show.js',
     group:                './group.js',
     groups:               './groups/index.js',
     groups_list:          './groups_list.js',
+    how_to_merge:         './how_to_merge.js',
     issue_show:           './issue_show/index.js',
     integrations:         './integrations',
     job_details:          './jobs/job_details_bundle.js',
@@ -53,11 +57,16 @@ var config = {
     notebook_viewer:      './blob/notebook_viewer.js',
     pdf_viewer:           './blob/pdf_viewer.js',
     pipelines:            './pipelines/pipelines_bundle.js',
-    pipelines_details:     './pipelines/pipeline_details_bundle.js',
+    pipelines_charts:     './pipelines/pipelines_charts.js',
+    pipelines_details:    './pipelines/pipeline_details_bundle.js',
+    pipelines_times:      './pipelines/pipelines_times.js',
     profile:              './profile/profile_bundle.js',
+    project_import_gl:    './projects/project_import_gitlab_project.js',
+    project_new:          './projects/project_new.js',
     prometheus_metrics:   './prometheus_metrics',
-    protected_branches:   './protected_branches/protected_branches_bundle.js',
+    protected_branches:   './protected_branches',
     protected_tags:       './protected_tags',
+    repo:                 './repo/index.js',
     sidebar:              './sidebar/sidebar_bundle.js',
     schedule_form:        './pipeline_schedules/pipeline_schedule_form_bundle.js',
     schedules_index:      './pipeline_schedules/pipeline_schedules_index_bundle.js',
@@ -66,10 +75,12 @@ var config = {
     stl_viewer:           './blob/stl_viewer.js',
     terminal:             './terminal/terminal_bundle.js',
     u2f:                  ['vendor/u2f'],
-    users:                './users/users_bundle.js',
+    ui_development_kit:   './ui_development_kit.js',
+    users:                './users/index.js',
     raven:                './raven/index.js',
     vue_merge_request_widget: './vue_merge_request_widget/index.js',
     test:                 './test.js',
+    two_factor_auth:      './two_factor_auth.js',
     performance_bar:      './performance_bar.js',
     webpack_runtime:      './webpack.js',
   },
@@ -102,26 +113,44 @@ var config = {
         options: { limit: 2048 },
       },
       {
-        test: /\.(worker\.js|pdf|bmpr)$/,
+        test: /\.(worker(\.min)?\.js|pdf|bmpr)$/,
         exclude: /node_modules/,
         loader: 'file-loader',
+        options: {
+          name: '[name].[hash].[ext]',
+        }
       },
       {
         test: /locale\/\w+\/(.*)\.js$/,
         loader: 'exports-loader?locales',
       },
-    ]
+      {
+        test: /monaco-editor\/\w+\/vs\/loader\.js$/,
+        use: [
+          { loader: 'exports-loader', options: 'l.global' },
+          { loader: 'imports-loader', options: 'l=>{},this=>l,AMDLoader=>this,module=>undefined' },
+        ],
+      }
+    ],
+
+    noParse: [/monaco-editor\/\w+\/vs\//],
   },
 
   plugins: [
     // manifest filename must match config.webpack.manifest_filename
     // webpack-rails only needs assetsByChunkName to function properly
-    new StatsPlugin('manifest.json', {
-      chunkModules: false,
-      source: false,
-      chunks: false,
-      modules: false,
-      assets: true
+    new StatsWriterPlugin({
+      filename: 'manifest.json',
+      transform: function(data, opts) {
+        var stats = opts.compiler.getStats().toJson({
+          chunkModules: false,
+          source: false,
+          chunks: false,
+          modules: false,
+          assets: true
+        });
+        return JSON.stringify(stats, null, 2);
+      }
     }),
 
     // prevent pikaday from including moment.js
@@ -169,6 +198,7 @@ var config = {
         'pdf_viewer',
         'pipelines',
         'pipelines_details',
+        'repo',
         'schedule_form',
         'schedules_index',
         'sidebar',
@@ -184,7 +214,6 @@ var config = {
       name: 'common_d3',
       chunks: [
         'graphs',
-        'users',
         'monitoring',
       ],
     }),
@@ -193,6 +222,26 @@ var config = {
     new webpack.optimize.CommonsChunkPlugin({
       names: ['main', 'locale', 'common', 'webpack_runtime'],
     }),
+
+    // copy pre-compiled vendor libraries verbatim
+    new CopyWebpackPlugin([
+      {
+        from: path.join(ROOT_PATH, `node_modules/monaco-editor/${IS_PRODUCTION ? 'min' : 'dev'}/vs`),
+        to: 'monaco-editor/vs',
+        transform: function(content, path) {
+          if (/\.js$/.test(path) && !/worker/i.test(path)) {
+            return (
+              '(function(){\n' +
+              'var define = this.define, require = this.require;\n' +
+              'window.define = define; window.require = require;\n' +
+              content +
+              '\n}.call(window.__monaco_context__ || (window.__monaco_context__ = {})));'
+            );
+          }
+          return content;
+        }
+      }
+    ]),
   ],
 
   resolve: {
@@ -241,6 +290,7 @@ if (IS_DEV_SERVER) {
   config.devServer = {
     host: DEV_SERVER_HOST,
     port: DEV_SERVER_PORT,
+    disableHostCheck: true,
     headers: { 'Access-Control-Allow-Origin': '*' },
     stats: 'errors-only',
     hot: DEV_SERVER_LIVERELOAD,