Merge remote-tracking branch 'upstream/master' into no-ivar-in-modules

* upstream/master: (671 commits)
  Make rubocop happy
  Use guard clause
  Improve language
  Prettify
  Use temp branch
  Pass info about who started the job and which job triggered it
  Docs: add indexes for monitoring and performance monitoring
  clearer-documentation-on-inline-diffs
  Add docs for commit diff discussion in merge requests
  sorting for tags api
  Clear BatchLoader after each spec to prevent holding onto records longer than necessary
  Include project in BatchLoader key to prevent returning blobs for the wrong project
  moved lfs_blob_ids method into ExtractsPath module
  Converted JS modules into exported modules
  spec fixes
  Bump gitlab-shell version to 5.10.3
  Clear caches before updating MR diffs
  Use new Ruby version 2.4 in GitLab QA images
  moved lfs blob fetch from extractspath file
  Update GitLab QA dependencies
  ...

15 files changed, 97 insertions, 50 deletions

diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 7f6e68ceed6..c8b6018bc1b 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -649,6 +649,8 @@ test:
     # user: YOUR_USERNAME
   pages:
     path: tmp/tests/pages
+  artifacts:
+    path: tmp/tests/artifacts
   repositories:
     storages:
       default:
diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
index 43b1e943897..eb7959e4da6 100644
--- a/config/initializers/7_prometheus_metrics.rb
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -11,14 +11,7 @@ Prometheus::Client.configure do |config|
     config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir')
   end
 
-  config.pid_provider = -> do
-    worker_id = Prometheus::Client::Support::Unicorn.worker_id
-    if worker_id.nil?
-      "process_pid_#{Process.pid}"
-    else
-      "worker_id_#{worker_id}"
-    end
-  end
+  config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider)
 end
 
 Gitlab::Application.configure do |config|
diff --git a/config/initializers/active_record_schema_ignore_tables.rb b/config/initializers/active_record_schema_ignore_tables.rb
new file mode 100644
index 00000000000..661135f8ade
--- /dev/null
+++ b/config/initializers/active_record_schema_ignore_tables.rb
@@ -0,0 +1,2 @@
+# Ignore table used temporarily in background migration
+ActiveRecord::SchemaDumper.ignore_tables = ["untracked_files_for_uploads"]
diff --git a/config/initializers/asset_sync.rb b/config/initializers/asset_sync.rb
new file mode 100644
index 00000000000..db8500f6231
--- /dev/null
+++ b/config/initializers/asset_sync.rb
@@ -0,0 +1,31 @@
+AssetSync.configure do |config|
+  # Disable the asset_sync gem by default. If it is enabled, but not configured,
+  # asset_sync will cause the build to fail.
+  config.enabled =  if ENV.has_key?('ASSET_SYNC_ENABLED')
+                      ENV['ASSET_SYNC_ENABLED'] == 'true'
+                    else
+                      false
+                    end
+
+  # Pulled from https://github.com/AssetSync/asset_sync/blob/v2.2.0/lib/asset_sync/engine.rb#L15-L40
+  # This allows us to disable asset_sync by default and configure through environment variables
+  # Updates to asset_sync gem should be checked
+  config.fog_provider = ENV['FOG_PROVIDER'] if ENV.has_key?('FOG_PROVIDER')
+  config.fog_directory = ENV['FOG_DIRECTORY'] if ENV.has_key?('FOG_DIRECTORY')
+  config.fog_region = ENV['FOG_REGION'] if ENV.has_key?('FOG_REGION')
+
+  config.aws_access_key_id = ENV['AWS_ACCESS_KEY_ID'] if ENV.has_key?('AWS_ACCESS_KEY_ID')
+  config.aws_secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('AWS_SECRET_ACCESS_KEY')
+  config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true  if ENV.has_key?('AWS_REDUCED_REDUNDANCY')
+
+  config.rackspace_username = ENV['RACKSPACE_USERNAME'] if ENV.has_key?('RACKSPACE_USERNAME')
+  config.rackspace_api_key = ENV['RACKSPACE_API_KEY'] if ENV.has_key?('RACKSPACE_API_KEY')
+
+  config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] if ENV.has_key?('GOOGLE_STORAGE_ACCESS_KEY_ID')
+  config.google_storage_secret_access_key = ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY'] if ENV.has_key?('GOOGLE_STORAGE_SECRET_ACCESS_KEY')
+
+  config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep"
+
+  config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION')
+  config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
+end
diff --git a/config/initializers/flipper.rb b/config/initializers/flipper.rb
index bfab8c77a4b..cc9167d29b9 100644
--- a/config/initializers/flipper.rb
+++ b/config/initializers/flipper.rb
@@ -1,8 +1,22 @@
-require 'flipper/middleware/memoizer'
+require 'flipper/adapters/active_record'
+require 'flipper/adapters/active_support_cache_store'
 
-unless Rails.env.test?
-  Rails.application.config.middleware.use Flipper::Middleware::Memoizer,
-    lambda { Feature.flipper }
+Flipper.configure do |config|
+  config.default do
+    adapter = Flipper::Adapters::ActiveRecord.new(
+      feature_class: Feature::FlipperFeature, gate_class: Feature::FlipperGate)
+    cached_adapter = Flipper::Adapters::ActiveSupportCacheStore.new(
+      adapter,
+      Rails.cache,
+      expires_in: 10.seconds)
+
+    Flipper.new(cached_adapter)
+  end
+end
 
-  Feature.register_feature_groups
+Feature.register_feature_groups
+
+unless Rails.env.test?
+  require 'flipper/middleware/memoizer'
+  Rails.application.config.middleware.use Flipper::Middleware::Memoizer
 end
diff --git a/config/initializers/forbid_sidekiq_in_transactions.rb b/config/initializers/forbid_sidekiq_in_transactions.rb
index a78711fe599..bedd57ede04 100644
--- a/config/initializers/forbid_sidekiq_in_transactions.rb
+++ b/config/initializers/forbid_sidekiq_in_transactions.rb
@@ -13,20 +13,19 @@ module Sidekiq
 
     module ClassMethods
       module NoSchedulingFromTransactions
-        NESTING = ::Rails.env.test? ? 1 : 0
-
         %i(perform_async perform_at perform_in).each do |name|
           define_method(name) do |*args|
-            return super(*args) if Sidekiq::Worker.skip_transaction_check
-            return super(*args) unless ActiveRecord::Base.connection.open_transactions > NESTING
+            if !Sidekiq::Worker.skip_transaction_check && AfterCommitQueue.inside_transaction?
+              raise <<-MSG.strip_heredoc
+                `#{self}.#{name}` cannot be called inside a transaction as this can lead to
+                race conditions when the worker runs before the transaction is committed and
+                tries to access a model that has not been saved yet.
 
-            raise <<-MSG.strip_heredoc
-              `#{self}.#{name}` cannot be called inside a transaction as this can lead to
-              race conditions when the worker runs before the transaction is committed and
-              tries to access a model that has not been saved yet.
+                Use an `after_commit` hook, or include `AfterCommitQueue` and use a `run_after_commit` block instead.
+              MSG
+            end
 
-              Use an `after_commit` hook, or include `AfterCommitQueue` and use a `run_after_commit` block instead.
-            MSG
+            super(*args)
           end
         end
       end
diff --git a/config/initializers/gollum.rb b/config/initializers/gollum.rb
index 2fd47a3f4d3..f1066f83dd9 100644
--- a/config/initializers/gollum.rb
+++ b/config/initializers/gollum.rb
@@ -1,3 +1,7 @@
+# WARNING changes in this file must be manually propagated to gitaly-ruby.
+#
+# https://gitlab.com/gitlab-org/gitaly/blob/master/ruby/lib/gitlab/gollum.rb
+
 module Gollum
   GIT_ADAPTER = "rugged".freeze
 end
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index a1cc9655319..0f164e628f9 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -42,6 +42,8 @@ Sidekiq.configure_server do |config|
 
   Gitlab::SidekiqThrottler.execute!
 
+  Gitlab::SidekiqVersioning.install!
+
   config = Gitlab::Database.config ||
     Rails.application.config.database_configuration[Rails.env]
   config['pool'] = Sidekiq.options[:concurrency]
@@ -60,19 +62,3 @@ Sidekiq.configure_client do |config|
     chain.add Gitlab::SidekiqStatus::ClientMiddleware
   end
 end
-
-# The Sidekiq client API always adds the queue to the Sidekiq queue
-# list, but mail_room and gitlab-shell do not. This is only necessary
-# for monitoring.
-config = YAML.load_file(Rails.root.join('config', 'sidekiq_queues.yml').to_s)
-
-begin
-  Sidekiq.redis do |conn|
-    conn.pipelined do
-      config[:queues].each do |queue|
-        conn.sadd('queues', queue[0])
-      end
-    end
-  end
-rescue Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED
-end
diff --git a/config/no_todos_messages.yml b/config/no_todos_messages.yml
index 264a975b614..da721a9b6e6 100644
--- a/config/no_todos_messages.yml
+++ b/config/no_todos_messages.yml
@@ -3,9 +3,9 @@
 #
 # If you come up with a fun one, please feel free to contribute it to GitLab!
 # https://about.gitlab.com/contributing/
---- 
-- Good job! Looks like you don't have any todos left.
+---
+- Good job! Looks like you don't have any todos left
 - Isn't an empty todo list beautiful?
 - Give yourself a pat on the back!
 - Nothing left to do, high five!
-- Henceforth you shall be known as "Todo Destroyer".
+- Henceforth you shall be known as "Todo Destroyer"
diff --git a/config/routes.rb b/config/routes.rb
index 4f27fea0e92..016140e0ede 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -42,6 +42,7 @@ Rails.application.routes.draw do
   scope path: '-' do
     get 'liveness' => 'health#liveness'
     get 'readiness' => 'health#readiness'
+    post 'storage_check' => 'health#storage_check'
     resources :metrics, only: [:index]
     mount Peek::Railtie => '/peek'
 
diff --git a/config/routes/admin.rb b/config/routes/admin.rb
index c0748231813..e22fb440abc 100644
--- a/config/routes/admin.rb
+++ b/config/routes/admin.rb
@@ -97,7 +97,7 @@ namespace :admin do
 
   resource :appearances, only: [:show, :create, :update], path: 'appearance' do
     member do
-      get :preview
+      get :preview_sign_in
       delete :logo
       delete :header_logos
     end
diff --git a/config/routes/group.rb b/config/routes/group.rb
index db99e10bb9a..976837a246d 100644
--- a/config/routes/group.rb
+++ b/config/routes/group.rb
@@ -49,6 +49,12 @@ constraints(GroupUrlConstrainer.new) do
       post :resend_invite, on: :member
       delete :leave, on: :collection
     end
+
+    resources :uploads, only: [:create] do
+      collection do
+        get ":secret/:filename", action: :show, as: :show, constraints: { filename: /[^\/]+/ }
+      end
+    end
   end
 
   scope(path: '*id',
diff --git a/config/routes/project.rb b/config/routes/project.rb
index bdafaba3ab3..093da10f57f 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -183,10 +183,16 @@ constraints(ProjectUrlConstrainer.new) do
         end
       end
 
-      resources :clusters, except: [:edit] do
+      resources :clusters, except: [:edit, :create] do
         collection do
-          get :login
-          get '/providers/gcp/new', action: :new_gcp
+          scope :providers do
+            get '/user/new', to: 'clusters/user#new'
+            post '/user', to: 'clusters/user#create'
+
+            get '/gcp/new', to: 'clusters/gcp#new'
+            get '/gcp/login', to: 'clusters/gcp#login'
+            post '/gcp', to: 'clusters/gcp#create'
+          end
         end
 
         member do
@@ -429,7 +435,7 @@ constraints(ProjectUrlConstrainer.new) do
         get :download_export
         get :activity
         get :refs
-        put :new_issue_address
+        put :new_issuable_address
       end
     end
   end
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index bc7c431731a..31a38f2b508 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -25,8 +25,6 @@
   - [new_note, 2]
   - [new_issue, 2]
   - [new_merge_request, 2]
-  - [build, 2]
-  - [pipeline, 2]
   - [pipeline_processing, 5]
   - [pipeline_creation, 4]
   - [pipeline_default, 3]
@@ -38,11 +36,12 @@
   - [mailers, 2]
   - [invalid_gpg_signature_update, 2]
   - [create_gpg_signature, 2]
+  - [rebase, 2]
   - [upload_checksum, 1]
   - [repository_fork, 1]
   - [repository_import, 1]
   - [github_importer, 1]
-  - [github_importer_advance_stage, 1]
+  - [github_import_advance_stage, 1]
   - [project_service, 1]
   - [delete_user, 1]
   - [delete_merged_branches, 1]
diff --git a/config/webpack.config.js b/config/webpack.config.js
index f7a7182a627..78ced4c3e8c 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -117,6 +117,10 @@ var config = {
         options: { limit: 2048 },
       },
       {
+        test: /\_worker\.js$/,
+        loader: 'worker-loader',
+      },
+      {
         test: /\.(worker(\.min)?\.js|pdf|bmpr)$/,
         exclude: /node_modules/,
         loader: 'file-loader',