Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (1723 commits)
  Resolve "Editor icons"
  Refactor issuable destroy action
  Ignore routes matching legacy_*_redirect in route specs
  Gitlab::Git::RevList and LfsChanges use lazy popen
  Gitlab::Git::Popen can lazily hand output to a block
  Merge branch 'master-i18n' into 'master'
  Remove unique validation from external_url in Environment
  Expose `duration` in Job API entity
  Add TimeCop freeze for DST and Regular time
  Harcode project visibility
  update a changelog
  Put a condition to old migration that adds fast_forward column to MRs
  Expose project visibility as CI variable
  fix flaky tests by removing unneeded clicks and focus actions
  fix flaky test in gfm_autocomplete_spec.rb
  Use Gitlab::Git operations for repository mirroring
  Encapsulate git operations for mirroring in Gitlab::Git
  Create a Wiki Repository's raw_repository properly
  Add `Gitlab::Git::Repository#fetch` command
  Fix Gitlab::Metrics::System#real_time and #monotonic_time doc
  ...

27 files changed, 216 insertions, 134 deletions

diff --git a/config/application.rb b/config/application.rb
index da9bb25c8b9..5100ec5d2b7 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -29,6 +29,7 @@ module Gitlab
                                      #{config.root}/app/models/project_services
                                      #{config.root}/app/workers/concerns
                                      #{config.root}/app/services/concerns
+                                     #{config.root}/app/serializers/concerns
                                      #{config.root}/app/finders/concerns])
 
     config.generators.templates.push("#{config.root}/generator_templates")
@@ -105,8 +106,7 @@ module Gitlab
     config.assets.precompile << "lib/ace.js"
     config.assets.precompile << "vendor/assets/fonts/*"
     config.assets.precompile << "test.css"
-    config.assets.precompile << "new_nav.css"
-    config.assets.precompile << "new_sidebar.css"
+    config.assets.precompile << "locale/**/app.js"
 
     # Version of your assets, change this if you want to expire all your assets
     config.assets.version = '1.0'
@@ -155,6 +155,9 @@ module Gitlab
     ENV['GITLAB_PATH_OUTSIDE_HOOK'] = ENV['PATH']
     ENV['GIT_TERMINAL_PROMPT'] = '0'
 
+    # Gitlab Read-only middleware support
+    config.middleware.insert_after ActionDispatch::Flash, 'Gitlab::Middleware::ReadOnly'
+
     config.generators do |g|
       g.factory_girl false
     end
diff --git a/config/database.yml.mysql b/config/database.yml.mysql
index eb71d3f5fe1..98c2abe9f5e 100644
--- a/config/database.yml.mysql
+++ b/config/database.yml.mysql
@@ -10,7 +10,7 @@ production:
   pool: 10
   username: git
   password: "secure password"
-  # host: localhost
+  host: localhost
   # socket: /tmp/mysql.sock
 
 #
@@ -25,7 +25,22 @@ development:
   pool: 5
   username: root
   password: "secure password"
-  # host: localhost
+  host: localhost
+  # socket: /tmp/mysql.sock
+
+#
+# Staging specific
+#
+staging:
+  adapter: mysql2
+  encoding: utf8
+  collation: utf8_general_ci
+  reconnect: false
+  database: gitlabhq_staging
+  pool: 10
+  username: git
+  password: "secure password"
+  host: localhost
   # socket: /tmp/mysql.sock
 
 # Warning: The database defined as "test" will be erased and
@@ -40,6 +55,6 @@ test: &test
   pool: 5
   username: root
   password:
-  # host: localhost
+  host: localhost
   # socket: /tmp/mysql.sock
   prepared_statements: false
diff --git a/config/database.yml.postgresql b/config/database.yml.postgresql
index 4b30982fe82..baded682e46 100644
--- a/config/database.yml.postgresql
+++ b/config/database.yml.postgresql
@@ -6,10 +6,9 @@ production:
   encoding: unicode
   database: gitlabhq_production
   pool: 10
-  # username: git
-  # password:
-  # host: localhost
-  # port: 5432
+  username: git
+  password: "secure password"
+  host: localhost
 
 #
 # Development specific
@@ -20,8 +19,8 @@ development:
   database: gitlabhq_development
   pool: 5
   username: postgres
-  password:
-  # host: localhost
+  password: "secure password"
+  host: localhost
 
 #
 # Staging specific
@@ -30,10 +29,10 @@ staging:
   adapter: postgresql
   encoding: unicode
   database: gitlabhq_staging
-  pool: 5
-  username: postgres
-  password:
-  # host: localhost
+  pool: 10
+  username: git
+  password: "secure password"
+  host: localhost
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -45,5 +44,5 @@ test: &test
   pool: 5
   username: postgres
   password:
-  # host: localhost
+  host: localhost
   prepared_statements: false
diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml
index db31b01a7d2..3af7f7bd5c0 100644
--- a/config/dependency_decisions.yml
+++ b/config/dependency_decisions.yml
@@ -464,3 +464,10 @@
     :why: Our own library - https://gitlab.com/gitlab-org/gitlab-svgs
     :versions: []
     :when: 2017-09-19 14:36:32.795496000 Z
+- - :license
+  - pikaday
+  - MIT
+  - :who: 
+    :why: 
+    :versions: []
+    :when: 2017-10-17 17:46:12.367554000 Z
diff --git a/config/environments/test.rb b/config/environments/test.rb
index 278144b8943..d09e51e766a 100644
--- a/config/environments/test.rb
+++ b/config/environments/test.rb
@@ -1,6 +1,7 @@
 Rails.application.configure do
   # Make sure the middleware is inserted first in middleware chain
   config.middleware.insert_before('ActionDispatch::Static', 'Gitlab::Testing::RequestBlockerMiddleware')
+  config.middleware.insert_before('ActionDispatch::Static', 'Gitlab::Testing::RequestInspectorMiddleware')
 
   # Settings specified here will take precedence over those in config/application.rb
 
@@ -16,7 +17,7 @@ Rails.application.configure do
   config.cache_classes = ENV['CACHE_CLASSES'] == 'true'
 
   # Configure static asset server for tests with Cache-Control for performance
-  config.assets.digest = false
+  config.assets.compile = false if ENV['CI']
   config.serve_static_files = true
   config.static_cache_control = "public, max-age=3600"
 
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 9b496822e93..7547ba4a8fa 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -89,7 +89,7 @@ production: &base
     # This happens when the commit is pushed or merged into the default branch of a project.
     # When not specified the default issue_closing_pattern as specified below will be used.
     # Tip: you can test your closing pattern at http://rubular.com.
-    # issue_closing_pattern: '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing))(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)'
+    # issue_closing_pattern: '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing)|[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)'
 
     ## Default project features settings
     default_projects_features:
@@ -164,6 +164,7 @@ production: &base
     host: example.com
     port: 80 # Set to 443 if you serve the pages with HTTPS
     https: false # Set to true if you serve the pages with HTTPS
+    artifacts_server: true
     # external_http: ["1.1.1.1:80", "[2001::1]:80"] # If defined, enables custom domain support in GitLab Pages
     # external_https: ["1.1.1.1:443", "[2001::1]:443"] # If defined, enables custom domain and certificate support in GitLab Pages
 
@@ -499,6 +500,8 @@ production: &base
 
   # Gitaly settings
   gitaly:
+    # Path to the directory containing Gitaly client executables.
+    client_path: /home/git/gitaly/bin
     # Default Gitaly authentication token. Can be overriden per storage. Can
     # be left blank when Gitaly is running locally on a Unix socket, which
     # is the normal way to deploy Gitaly.
@@ -519,11 +522,6 @@ production: &base
         path: /home/git/repositories/
         gitaly_address: unix:/home/git/gitlab/tmp/sockets/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port)
         # gitaly_token: 'special token' # Optional: override global gitaly.token for this storage.
-        failure_count_threshold: 10 # number of failures before stopping attempts
-        failure_wait_time: 30 # Seconds after an access failure before allowing access again
-        failure_reset_time: 1800 # Time in seconds to expire failures
-        storage_timeout: 30 # Time in seconds to wait before aborting a storage access attempt
-
 
   ## Backup settings
   backup:
@@ -656,15 +654,12 @@ test:
       default:
         path: tmp/tests/repositories/
         gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
-        failure_count_threshold: 999999
-        failure_wait_time: 0
-        storage_timeout: 30
       broken:
         path: tmp/tests/non-existent-repositories
         gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
 
   gitaly:
-    enabled: true
+    client_path: tmp/tests/gitaly
     token: secret
   backup:
     path: tmp/tests/backups
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 27c1ecc7b23..12694f8016f 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -113,12 +113,14 @@ class Settings < Settingslogic
       URI.parse(url_without_path).host
     end
 
-    # Random cron time every Sunday to load balance usage pings
-    def cron_random_weekly_time
+    # Runs every minute in a random ten-minute period on Sundays, to balance the
+    # load on the server receiving these pings. The usage ping is safe to run
+    # multiple times because of a 24 hour exclusive lock.
+    def cron_for_usage_ping
       hour = rand(24)
-      minute = rand(60)
+      minute = rand(6)
 
-      "#{minute} #{hour} * * 0"
+      "#{minute}0-#{minute}9 #{hour} * * 0"
     end
   end
 end
@@ -257,7 +259,7 @@ Settings.gitlab['signup_enabled'] ||= true if Settings.gitlab['signup_enabled'].
 Settings.gitlab['password_authentication_enabled'] ||= true if Settings.gitlab['password_authentication_enabled'].nil?
 Settings.gitlab['restricted_visibility_levels'] = Settings.__send__(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], [])
 Settings.gitlab['username_changing_enabled'] = true if Settings.gitlab['username_changing_enabled'].nil?
-Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing))(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil?
+Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing)|[Ii]mplement(?:s|ed|ing)?)(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil?
 Settings.gitlab['default_projects_features'] ||= {}
 Settings.gitlab['webhook_timeout'] ||= 10
 Settings.gitlab['max_attachment_size'] ||= 10
@@ -316,15 +318,16 @@ Settings.registry['path']            = Settings.absolute(Settings.registry['path
 # Pages
 #
 Settings['pages'] ||= Settingslogic.new({})
-Settings.pages['enabled']         = false if Settings.pages['enabled'].nil?
-Settings.pages['path']            = Settings.absolute(Settings.pages['path'] || File.join(Settings.shared['path'], "pages"))
-Settings.pages['https']           = false if Settings.pages['https'].nil?
-Settings.pages['host']            ||= "example.com"
-Settings.pages['port']            ||= Settings.pages.https ? 443 : 80
-Settings.pages['protocol']        ||= Settings.pages.https ? "https" : "http"
-Settings.pages['url']             ||= Settings.__send__(:build_pages_url)
-Settings.pages['external_http']   ||= false unless Settings.pages['external_http'].present?
-Settings.pages['external_https']  ||= false unless Settings.pages['external_https'].present?
+Settings.pages['enabled']           = false if Settings.pages['enabled'].nil?
+Settings.pages['path']              = Settings.absolute(Settings.pages['path'] || File.join(Settings.shared['path'], "pages"))
+Settings.pages['https']             = false if Settings.pages['https'].nil?
+Settings.pages['host']              ||= "example.com"
+Settings.pages['port']              ||= Settings.pages.https ? 443 : 80
+Settings.pages['protocol']          ||= Settings.pages.https ? "https" : "http"
+Settings.pages['url']               ||= Settings.__send__(:build_pages_url)
+Settings.pages['external_http']     ||= false unless Settings.pages['external_http'].present?
+Settings.pages['external_https']    ||= false unless Settings.pages['external_https'].present?
+Settings.pages['artifacts_server']  ||= Settings.pages['enabled'] if Settings.pages['artifacts_server'].nil?
 
 #
 # Git LFS
@@ -397,7 +400,7 @@ Settings.cron_jobs['stuck_import_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_import_jobs_worker']['cron'] ||= '15 * * * *'
 Settings.cron_jobs['stuck_import_jobs_worker']['job_class'] = 'StuckImportJobsWorker'
 Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_random_weekly_time)
+Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_for_usage_ping)
 Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
 
 Settings.cron_jobs['schedule_update_user_activity_worker'] ||= Settingslogic.new({})
@@ -452,17 +455,6 @@ Settings.repositories.storages.each do |key, storage|
 
   # Expand relative paths
   storage['path'] = Settings.absolute(storage['path'])
-  # Set failure defaults
-  storage['failure_count_threshold'] ||= 10
-  storage['failure_wait_time'] ||= 30
-  storage['failure_reset_time'] ||= 1800
-  storage['storage_timeout'] ||= 5
-  # Set turn strings into numbers
-  storage['failure_count_threshold'] = storage['failure_count_threshold'].to_i
-  storage['failure_wait_time'] = storage['failure_wait_time'].to_i
-  storage['failure_reset_time'] = storage['failure_reset_time'].to_i
-  # We might want to have a timeout shorter than 1 second.
-  storage['storage_timeout'] = storage['storage_timeout'].to_f
 
   Settings.repositories.storages[key] = storage
 end
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index e1a59d8c152..2d8704622b6 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -123,7 +123,9 @@ def instrument_classes(instrumentation)
 end
 # rubocop:enable Metrics/AbcSize
 
-Gitlab::Metrics::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start
+unless Sidekiq.server?
+  Gitlab::Metrics::UnicornSampler.initialize_instance(Settings.monitoring.unicorn_sampler_interval).start
+end
 
 Gitlab::Application.configure do |config|
   # 0 should be Sentry to catch errors in this middleware
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index 3aed2136f1b..c6ec0aeda7b 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -36,7 +36,7 @@ Devise.setup do |config|
   # Configure which authentication keys should be case-insensitive.
   # These keys will be downcased upon creating or modifying a user and when used
   # to authenticate or find a user. Default is :email.
-  config.case_insensitive_keys = [:email]
+  config.case_insensitive_keys = [:email, :email_confirmation]
 
   # Configure which authentication keys should have whitespace stripped.
   # These keys will have whitespace before and after removed upon creating or
@@ -175,7 +175,7 @@ Devise.setup do |config|
 
   # Configure the default scope given to Warden. By default it's the first
   # devise role declared in your routes (usually :user).
-  # config.default_scope = :user
+  config.default_scope = :user  # now have an :email scope as well, so set the default
 
   # Configure sign_out behavior.
   # Sign_out action can be scoped (i.e. /users/sign_out affects only :user scope).
diff --git a/config/initializers/doorkeeper_openid_connect.rb b/config/initializers/doorkeeper_openid_connect.rb
index c58f425b19b..af174def047 100644
--- a/config/initializers/doorkeeper_openid_connect.rb
+++ b/config/initializers/doorkeeper_openid_connect.rb
@@ -1,7 +1,7 @@
 Doorkeeper::OpenidConnect.configure do
   issuer Gitlab.config.gitlab.url
 
-  jws_private_key Rails.application.secrets.jws_private_key
+  signing_key Rails.application.secrets.openid_connect_signing_key
 
   resource_owner_from_access_token do |access_token|
     User.active.find_by(id: access_token.resource_owner_id)
diff --git a/config/initializers/gettext_rails_i18n_patch.rb b/config/initializers/gettext_rails_i18n_patch.rb
index 377e5104f9d..49551319435 100644
--- a/config/initializers/gettext_rails_i18n_patch.rb
+++ b/config/initializers/gettext_rails_i18n_patch.rb
@@ -39,3 +39,17 @@ module GettextI18nRailsJs
     end
   end
 end
+
+class PoToJson
+  # This is required to modify the JS locale file output to our import needs
+  # Overwrites: https://github.com/webhippie/po_to_json/blob/master/lib/po_to_json.rb#L46
+  def generate_for_jed(language, overwrite = {})
+    @options = parse_options(overwrite.merge(language: language))
+    @parsed ||= inject_meta(parse_document)
+
+    generated = build_json_for(build_jed_for(@parsed))
+    [
+      "window.translations = #{generated};"
+    ].join(" ")
+  end
+end
diff --git a/config/initializers/grpc.rb b/config/initializers/grpc.rb
new file mode 100644
index 00000000000..b96962fe7db
--- /dev/null
+++ b/config/initializers/grpc.rb
@@ -0,0 +1,11 @@
+require 'logger'
+
+GRPC_LOGGER = Logger.new(Rails.root.join('log/grpc.log'))
+GRPC_LOGGER.level = ENV['GRPC_LOG_LEVEL'].presence || 'WARN'
+GRPC_LOGGER.progname = 'GRPC'
+
+module GRPC
+  def self.logger
+    GRPC_LOGGER
+  end
+end
diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb
index a54d53cbbe2..1cff355346c 100644
--- a/config/initializers/peek.rb
+++ b/config/initializers/peek.rb
@@ -16,6 +16,7 @@ Peek.into Peek::Views::Redis
 Peek.into Peek::Views::Sidekiq
 Peek.into Peek::Views::Rblineprof
 Peek.into Peek::Views::GC
+Peek.into Peek::Views::Gitaly
 
 # rubocop:disable Style/ClassAndModuleCamelCase
 class PEEK_DB_CLIENT
diff --git a/config/initializers/secret_token.rb b/config/initializers/secret_token.rb
index f9c1d2165d3..750a5b34f3b 100644
--- a/config/initializers/secret_token.rb
+++ b/config/initializers/secret_token.rb
@@ -25,7 +25,7 @@ def create_tokens
     secret_key_base: file_secret_key || generate_new_secure_token,
     otp_key_base: env_secret_key || file_secret_key || generate_new_secure_token,
     db_key_base: generate_new_secure_token,
-    jws_private_key: generate_new_rsa_private_key
+    openid_connect_signing_key: generate_new_rsa_private_key
   }
 
   missing_secrets = set_missing_keys(defaults)
diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb
index 62d0967009a..b2da3b3dc19 100644
--- a/config/initializers/sentry.rb
+++ b/config/initializers/sentry.rb
@@ -2,7 +2,7 @@
 
 require 'gitlab/current_settings'
 
-if Rails.env.production?
+def configure_sentry
   # allow it to fail: it may do so when create_from_defaults is executed before migrations are actually done
   begin
     sentry_enabled = Gitlab::CurrentSettings.current_application_settings.sentry_enabled
@@ -23,3 +23,5 @@ if Rails.env.production?
     end
   end
 end
+
+configure_sentry if Rails.env.production?
diff --git a/config/locales/doorkeeper.en.yml b/config/locales/doorkeeper.en.yml
index 14d49885fb3..0da6b14c29e 100644
--- a/config/locales/doorkeeper.en.yml
+++ b/config/locales/doorkeeper.en.yml
@@ -58,9 +58,10 @@ en:
           expired: "The access token expired"
           unknown: "The access token is invalid"
     scopes:
-      api: Access your API
-      read_user: Read user information
+      api: Access the authenticated user's API
+      read_user: Read the authenticated user's personal information
       openid: Authenticate using OpenID Connect
+      sudo: Perform API actions as any user in the system (if the authenticated user is an admin)
 
     flash:
       applications:
diff --git a/config/routes.rb b/config/routes.rb
index 5683725c8a2..fc13dc4865f 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -44,6 +44,19 @@ Rails.application.routes.draw do
     get 'readiness' => 'health#readiness'
     resources :metrics, only: [:index]
     mount Peek::Railtie => '/peek'
+
+    # Boards resources shared between group and projects
+    resources :boards, only: [] do
+      resources :lists, module: :boards, only: [:index, :create, :update, :destroy] do
+        collection do
+          post :generate
+        end
+
+        resources :issues, only: [:index, :create, :update]
+      end
+
+      resources :issues, module: :boards, only: [:index, :update]
+    end
   end
 
   # Koding route
@@ -74,19 +87,7 @@ Rails.application.routes.draw do
   # Notification settings
   resources :notification_settings, only: [:create, :update]
 
-  # Boards resources shared between group and projects
-  resources :boards do
-    resources :lists, module: :boards, only: [:index, :create, :update, :destroy] do
-      collection do
-        post :generate
-      end
-
-      resources :issues, only: [:index, :create, :update]
-    end
-
-    resources :issues, module: :boards, only: [:index, :update]
-  end
-
+  draw :google_api
   draw :import
   draw :uploads
   draw :explore
diff --git a/config/routes/ci.rb b/config/routes/ci.rb
index cbd4c2db852..60c1724bc05 100644
--- a/config/routes/ci.rb
+++ b/config/routes/ci.rb
@@ -1,5 +1,5 @@
 namespace :ci do
   resource :lint, only: [:show, :create]
 
-  root to: redirect('/')
+  root to: redirect('')
 end
diff --git a/config/routes/google_api.rb b/config/routes/google_api.rb
new file mode 100644
index 00000000000..a119b47c176
--- /dev/null
+++ b/config/routes/google_api.rb
@@ -0,0 +1,7 @@
+scope '-' do
+  namespace :google_api do
+    resource :auth, only: [], controller: :authorizations do
+      match :callback, via: [:get, :post]
+    end
+  end
+end
diff --git a/config/routes/group.rb b/config/routes/group.rb
index 23052a6c6dc..f4d520a2518 100644
--- a/config/routes/group.rb
+++ b/config/routes/group.rb
@@ -1,51 +1,54 @@
 require 'constraints/group_url_constrainer'
 
-resources :groups, only: [:index, :new, :create]
-
-scope(path: 'groups/*group_id',
-      module: :groups,
-      as: :group,
-      constraints: { group_id: Gitlab::PathRegex.full_namespace_route_regex }) do
-  resources :group_members, only: [:index, :create, :update, :destroy], concerns: :access_requestable do
-    post :resend_invite, on: :member
-    delete :leave, on: :collection
+resources :groups, only: [:index, :new, :create] do
+  post :preview_markdown
+end
+
+constraints(GroupUrlConstrainer.new) do
+  scope(path: 'groups/*id',
+        controller: :groups,
+        constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ }) do
+    get :edit, as: :edit_group
+    get :issues, as: :issues_group
+    get :merge_requests, as: :merge_requests_group
+    get :projects, as: :projects_group
+    get :activity, as: :activity_group
+    get '/', action: :show, as: :group_canonical
   end
 
-  resource :avatar, only: [:destroy]
-  resources :milestones, constraints: { id: /[^\/]+/ }, only: [:index, :show, :edit, :update, :new, :create] do
-    member do
-      get :merge_requests
-      get :participants
-      get :labels
+  scope(path: 'groups/*group_id',
+        module: :groups,
+        as: :group,
+        constraints: { group_id: Gitlab::PathRegex.full_namespace_route_regex }) do
+    resources :group_members, only: [:index, :create, :update, :destroy], concerns: :access_requestable do
+      post :resend_invite, on: :member
+      delete :leave, on: :collection
     end
-  end
 
-  resources :labels, except: [:show] do
-    post :toggle_subscription, on: :member
-  end
+    resource :avatar, only: [:destroy]
+    resources :milestones, constraints: { id: /[^\/]+/ }, only: [:index, :show, :edit, :update, :new, :create] do
+      member do
+        get :merge_requests
+        get :participants
+        get :labels
+      end
+    end
 
-  scope path: '-' do
-    namespace :settings do
-      resource :ci_cd, only: [:show], controller: 'ci_cd'
+    resources :labels, except: [:show] do
+      post :toggle_subscription, on: :member
     end
 
-    resources :variables, only: [:index, :show, :update, :create, :destroy]
-  end
-end
+    scope path: '-' do
+      namespace :settings do
+        resource :ci_cd, only: [:show], controller: 'ci_cd'
+      end
 
-scope(path: 'groups/*id',
-      controller: :groups,
-      constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ }) do
-  get :edit, as: :edit_group
-  get :issues, as: :issues_group
-  get :merge_requests, as: :merge_requests_group
-  get :projects, as: :projects_group
-  get :activity, as: :activity_group
-  get :subgroups, as: :subgroups_group
-  get '/', action: :show, as: :group_canonical
-end
+      resources :variables, only: [:index, :show, :update, :create, :destroy]
+
+      resources :children, only: [:index]
+    end
+  end
 
-constraints(GroupUrlConstrainer.new) do
   scope(path: '*id',
         as: :group,
         constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ },
diff --git a/config/routes/help.rb b/config/routes/help.rb
index d53822da9ec..2ea8bfd7aed 100644
--- a/config/routes/help.rb
+++ b/config/routes/help.rb
@@ -1,4 +1,5 @@
-get 'help'           => 'help#index'
-get 'help/shortcuts' => 'help#shortcuts'
-get 'help/ui'        => 'help#ui'
-get 'help/*path'     => 'help#show', as: :help_page
+get 'help'                        => 'help#index'
+get 'help/shortcuts'              => 'help#shortcuts'
+get 'help/ui'                     => 'help#ui'
+get 'help/instance_configuration' => 'help#instance_configuration'
+get 'help/*path'                  => 'help#show', as: :help_page
diff --git a/config/routes/profile.rb b/config/routes/profile.rb
index 3e4e6111ab8..bcfc17a5f66 100644
--- a/config/routes/profile.rb
+++ b/config/routes/profile.rb
@@ -1,9 +1,11 @@
+# for secondary email confirmations - uses the same confirmation controller as :users
+devise_for :emails, path: 'profile/emails', controllers: { confirmations: :confirmations }
+
 resource :profile, only: [:show, :update] do
   member do
     get :audit_log
     get :applications, to: 'oauth/applications#index'
 
-    put :reset_private_token
     put :reset_incoming_email_token
     put :reset_rss_token
     put :update_username
@@ -28,7 +30,11 @@ resource :profile, only: [:show, :update] do
         put :revoke
       end
     end
-    resources :emails, only: [:index, :create, :destroy]
+    resources :emails, only: [:index, :create, :destroy] do
+      member do
+        put :resend_confirmation_instructions
+      end
+    end
     resources :chat_names, only: [:index, :new, :create, :destroy] do
       collection do
         delete :deny
diff --git a/config/routes/project.rb b/config/routes/project.rb
index b36d13888cd..746c0c46677 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -183,6 +183,16 @@ constraints(ProjectUrlConstrainer.new) do
         end
       end
 
+      resources :clusters, except: [:edit] do
+        collection do
+          get :login
+        end
+
+        member do
+          get :status, format: :json
+        end
+      end
+
       resources :environments, except: [:destroy] do
         member do
           post :stop
@@ -271,13 +281,19 @@ constraints(ProjectUrlConstrainer.new) do
 
       namespace :registry do
         resources :repository, only: [] do
-          resources :tags, only: [:destroy],
-                           constraints: { id: Gitlab::Regex.container_registry_tag_regex }
+          # We default to JSON format in the controller to avoid ambiguity.
+          # `latest.json` could either be a request for a tag named `latest`
+          # in JSON format, or a request for tag named `latest.json`.
+          scope format: false do
+            resources :tags, only: [:index, :destroy],
+                             constraints: { id: Gitlab::Regex.container_registry_tag_regex }
+          end
         end
       end
 
       resources :milestones, constraints: { id: /\d+/ } do
         member do
+          post :promote
           put :sort_issues
           put :sort_merge_requests
           get :merge_requests
@@ -378,7 +394,7 @@ constraints(ProjectUrlConstrainer.new) do
         end
       end
       namespace :settings do
-        get :members, to: redirect('/%{namespace_id}/%{project_id}/project_members')
+        get :members, to: redirect("%{namespace_id}/%{project_id}/project_members")
         resource :ci_cd, only: [:show], controller: 'ci_cd'
         resource :integrations, only: [:show]
         resource :repository, only: [:show], controller: :repository
diff --git a/config/routes/snippets.rb b/config/routes/snippets.rb
index 0a4ebac3ca3..81bc890d86b 100644
--- a/config/routes/snippets.rb
+++ b/config/routes/snippets.rb
@@ -17,5 +17,5 @@ resources :snippets, concerns: :awardable do
   end
 end
 
-get '/s/:username', to: redirect('/u/%{username}/snippets'),
+get '/s/:username', to: redirect('u/%{username}/snippets'),
                     constraints: { username: /[a-zA-Z.0-9_\-]+(?<!\.atom)/ }
diff --git a/config/routes/user.rb b/config/routes/user.rb
index e682dcd6663..733a3f6ce9a 100644
--- a/config/routes/user.rb
+++ b/config/routes/user.rb
@@ -22,17 +22,17 @@ scope(constraints: { username: Gitlab::PathRegex.root_namespace_route_regex }) d
     get :contributed, as: :contributed_projects
     get :snippets
     get :exists
-    get '/', to: redirect('/%{username}'), as: nil
+    get '/', to: redirect('%{username}'), as: nil
   end
 
   # Compatibility with old routing
   # TODO (dzaporozhets): remove in 10.0
-  get '/u/:username', to: redirect('/%{username}')
+  get '/u/:username', to: redirect('%{username}')
   # TODO (dzaporozhets): remove in 9.0
-  get '/u/:username/groups', to: redirect('/users/%{username}/groups')
-  get '/u/:username/projects', to: redirect('/users/%{username}/projects')
-  get '/u/:username/snippets', to: redirect('/users/%{username}/snippets')
-  get '/u/:username/contributed', to: redirect('/users/%{username}/contributed')
+  get '/u/:username/groups', to: redirect('users/%{username}/groups')
+  get '/u/:username/projects', to: redirect('users/%{username}/projects')
+  get '/u/:username/snippets', to: redirect('users/%{username}/snippets')
+  get '/u/:username/contributed', to: redirect('users/%{username}/contributed')
 end
 
 constraints(UserUrlConstrainer.new) do
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index d169c38a693..e2bb766ee47 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -62,3 +62,6 @@
   - [update_user_activity, 1]
   - [propagate_service_template, 1]
   - [background_migration, 1]
+  - [gcp_cluster, 1]
+  - [project_migrate_hashed_storage, 1]
+  - [storage_migrator, 1]
diff --git a/config/webpack.config.js b/config/webpack.config.js
index 3404715fe30..f7a7182a627 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -26,6 +26,7 @@ var config = {
   },
   context: path.join(ROOT_PATH, 'app/assets/javascripts'),
   entry: {
+    account:              './profile/account/index.js',
     balsamiq_viewer:      './blob/balsamiq_viewer.js',
     blob:                 './blob_edit/blob_bundle.js',
     boards:               './boards/boards_bundle.js',
@@ -68,6 +69,7 @@ var config = {
     prometheus_metrics:   './prometheus_metrics',
     protected_branches:   './protected_branches',
     protected_tags:       './protected_tags',
+    registry_list:        './registry/index.js',
     repo:                 './repo/index.js',
     sidebar:              './sidebar/sidebar_bundle.js',
     schedule_form:        './pipeline_schedules/pipeline_schedule_form_bundle.js',
@@ -82,6 +84,7 @@ var config = {
     vue_merge_request_widget: './vue_merge_request_widget/index.js',
     test:                 './test.js',
     two_factor_auth:      './two_factor_auth.js',
+    users:                './users/index.js',
     performance_bar:      './performance_bar.js',
     webpack_runtime:      './webpack.js',
   },
@@ -122,10 +125,6 @@ var config = {
         }
       },
       {
-        test: /locale\/\w+\/(.*)\.js$/,
-        loader: 'exports-loader?locales',
-      },
-      {
         test: /monaco-editor\/\w+\/vs\/loader\.js$/,
         use: [
           { loader: 'exports-loader', options: 'l.global' },
@@ -200,6 +199,7 @@ var config = {
         'pdf_viewer',
         'pipelines',
         'pipelines_details',
+        'registry_list',
         'repo',
         'schedule_form',
         'schedules_index',
@@ -216,13 +216,15 @@ var config = {
       name: 'common_d3',
       chunks: [
         'graphs',
+        'graphs_show',
         'monitoring',
+        'users',
       ],
     }),
 
     // create cacheable common library bundles
     new webpack.optimize.CommonsChunkPlugin({
-      names: ['main', 'locale', 'common', 'webpack_runtime'],
+      names: ['main', 'common', 'webpack_runtime'],
     }),
 
     // enable scope hoisting
@@ -234,7 +236,7 @@ var config = {
         from: path.join(ROOT_PATH, `node_modules/monaco-editor/${IS_PRODUCTION ? 'min' : 'dev'}/vs`),
         to: 'monaco-editor/vs',
         transform: function(content, path) {
-          if (/\.js$/.test(path) && !/worker/i.test(path)) {
+          if (/\.js$/.test(path) && !/worker/i.test(path) && !/typescript/i.test(path)) {
             return (
               '(function(){\n' +
               'var define = this.define, require = this.require;\n' +