Merge remote-tracking branch 'upstream/master' into artifacts-from-ref-and-build-name

* upstream/master: (123 commits)
  Limit the size of SVGs when viewing them as blobs
  Add a spec for ProjectsFinder project_ids_relation option
  Fix ProjectsFinder spec
  Pass project IDs relation to ProjectsFinder instead of using a block
  Speed up todos queries by limiting the projects set we join with
  Used phantomjs variable
  Ability to specify branches for pivotal tracker integration
  Fix a memory leak caused by Banzai::Filter::SanitizationFilter
  Update phantomjs link
  Remove sleeping and replace escaped text.
  Filters test fix
  Fixed filtering tests
  Removed screenshot command :poop:
  Updated failing tests
  Used mirrored version on GitLab
  Updated tests
  Fix `U2fSpec` for PhantomJS versions > 2.
  Fix file downloading
  Install latest stable phantomjs
  Use new PhantomJS version
  ...

5 files changed, 98 insertions, 42 deletions

diff --git a/config/initializers/5_backend.rb b/config/initializers/5_backend.rb
index e026151a032..ed88c8ee1b8 100644
--- a/config/initializers/5_backend.rb
+++ b/config/initializers/5_backend.rb
@@ -1,6 +1,3 @@
-# GIT over HTTP
-require_dependency Rails.root.join('lib/gitlab/backend/grack_auth')
-
 # GIT over SSH
 require_dependency Rails.root.join('lib/gitlab/backend/shell')
 
diff --git a/config/initializers/metrics.rb b/config/initializers/metrics.rb
index cc8208db3c1..52522e099e7 100644
--- a/config/initializers/metrics.rb
+++ b/config/initializers/metrics.rb
@@ -148,6 +148,9 @@ if Gitlab::Metrics.enabled?
 
     config.instrument_methods(Gitlab::Highlight)
     config.instrument_instance_methods(Gitlab::Highlight)
+
+    # This is a Rails scope so we have to instrument it manually.
+    config.instrument_method(Project, :visible_to_user)
   end
 
   GC::Profiler.enable
diff --git a/config/initializers/mime_types.rb b/config/initializers/mime_types.rb
index 3e553120205..f498732feca 100644
--- a/config/initializers/mime_types.rb
+++ b/config/initializers/mime_types.rb
@@ -12,3 +12,10 @@ Mime::Type.register_alias "text/html",  :md
 Mime::Type.register "video/mp4",  :mp4, [], [:m4v, :mov]
 Mime::Type.register "video/webm", :webm
 Mime::Type.register "video/ogg",  :ogv
+
+middlewares = Gitlab::Application.config.middleware
+middlewares.swap(ActionDispatch::ParamsParser, ActionDispatch::ParamsParser, {
+  Mime::Type.lookup('application/vnd.git-lfs+json') => lambda do |body|
+    ActiveSupport::JSON.decode(body)
+  end
+})
diff --git a/config/initializers/secret_token.rb b/config/initializers/secret_token.rb
index dae3a4a9a93..291fa6c0abc 100644
--- a/config/initializers/secret_token.rb
+++ b/config/initializers/secret_token.rb
@@ -2,49 +2,86 @@
 
 require 'securerandom'
 
-# Your secret key for verifying the integrity of signed cookies.
-# If you change this key, all old signed cookies will become invalid!
-# Make sure the secret is at least 30 characters and all random,
-# no regular words or you'll be exposed to dictionary attacks.
-
-def find_secure_token
-  token_file = Rails.root.join('.secret')
-  if ENV.key?('SECRET_KEY_BASE')
-    ENV['SECRET_KEY_BASE']
-  elsif File.exist? token_file
-    # Use the existing token.
-    File.read(token_file).chomp
-  else
-    # Generate a new token of 64 random hexadecimal characters and store it in token_file.
-    token = SecureRandom.hex(64)
-    File.write(token_file, token)
-    token
+# Transition material in .secret to the secret_key_base key in config/secrets.yml.
+# Historically, ENV['SECRET_KEY_BASE'] takes precedence over .secret, so we maintain that
+# behavior.
+#
+# It also used to be the case that the key material in ENV['SECRET_KEY_BASE'] or .secret
+# was used to encrypt OTP (two-factor authentication) data so if present, we copy that key
+# material into config/secrets.yml under otp_key_base.
+#
+# Finally, if we have successfully migrated all secrets to config/secrets.yml, delete the
+# .secret file to avoid confusion.
+#
+def create_tokens
+  secret_file = Rails.root.join('.secret')
+  file_secret_key = File.read(secret_file).chomp if File.exist?(secret_file)
+  env_secret_key = ENV['SECRET_KEY_BASE']
+
+  # Ensure environment variable always overrides secrets.yml.
+  Rails.application.secrets.secret_key_base = env_secret_key if env_secret_key.present?
+
+  defaults = {
+    secret_key_base: file_secret_key || generate_new_secure_token,
+    otp_key_base: env_secret_key || file_secret_key || generate_new_secure_token,
+    db_key_base: generate_new_secure_token
+  }
+
+  missing_secrets = set_missing_keys(defaults)
+  write_secrets_yml(missing_secrets) unless missing_secrets.empty?
+
+  begin
+    File.delete(secret_file) if file_secret_key
+  rescue => e
+    warn "Error deleting useless .secret file: #{e}"
   end
 end
 
-Rails.application.config.secret_token = find_secure_token
-Rails.application.config.secret_key_base = find_secure_token
-
-# CI
 def generate_new_secure_token
   SecureRandom.hex(64)
 end
 
-if Rails.application.secrets.db_key_base.blank?
-  warn "Missing `db_key_base` for '#{Rails.env}' environment. The secrets will be generated and stored in `config/secrets.yml`"
+def warn_missing_secret(secret)
+  warn "Missing Rails.application.secrets.#{secret} for #{Rails.env} environment. The secret will be generated and stored in config/secrets.yml."
+end
 
-  all_secrets = YAML.load_file('config/secrets.yml') if File.exist?('config/secrets.yml')
-  all_secrets ||= {}
+def set_missing_keys(defaults)
+  defaults.stringify_keys.each_with_object({}) do |(key, default), missing|
+    if Rails.application.secrets[key].blank?
+      warn_missing_secret(key)
 
-  # generate secrets
-  env_secrets = all_secrets[Rails.env.to_s] || {}
-  env_secrets['db_key_base'] ||= generate_new_secure_token
-  all_secrets[Rails.env.to_s] = env_secrets
+      missing[key] = Rails.application.secrets[key] = default
+    end
+  end
+end
+
+def write_secrets_yml(missing_secrets)
+  secrets_yml = Rails.root.join('config/secrets.yml')
+  rails_env = Rails.env.to_s
+  secrets = YAML.load_file(secrets_yml) if File.exist?(secrets_yml)
+  secrets ||= {}
+  secrets[rails_env] ||= {}
+
+  secrets[rails_env].merge!(missing_secrets) do |key, old, new|
+    # Previously, it was possible this was set to the literal contents of an Erb
+    # expression that evaluated to an empty value. We don't want to support that
+    # specifically, just ensure we don't break things further.
+    #
+    if old.present?
+      warn <<EOM
+Rails.application.secrets.#{key} was blank, but the literal value in config/secrets.yml was:
+  #{old}
 
-  # save secrets
-  File.open('config/secrets.yml', 'w', 0600) do |file|
-    file.write(YAML.dump(all_secrets))
+This probably isn't the expected value for this secret. To keep using a literal Erb string in config/secrets.yml, replace `<%` with `<%%`.
+EOM
+
+      exit 1 # rubocop:disable Rails/Exit
+    end
+
+    new
   end
 
-  Rails.application.secrets.db_key_base = env_secrets['db_key_base']
+  File.write(secrets_yml, YAML.dump(secrets), mode: 'w', perm: 0600)
 end
+
+create_tokens
diff --git a/config/routes.rb b/config/routes.rb
index 256c7dacd59..70bdb1d5beb 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -84,9 +84,6 @@ Rails.application.routes.draw do
   # Health check
   get 'health_check(/:checks)' => 'health_check#index', as: :health_check
 
-  # Enable Grack support (for LFS only)
-  mount Grack::AuthSpawner, at: '/', constraints: lambda { |request| /[-\/\w\.]+\.git\/(info\/lfs|gitlab-lfs)/.match(request.path_info) }, via: [:get, :post, :put]
-
   # Help
   get 'help'           => 'help#index'
   get 'help/shortcuts' => 'help#shortcuts'
@@ -471,7 +468,7 @@ Rails.application.routes.draw do
         post :unarchive
         post :housekeeping
         post :toggle_star
-        post :markdown_preview
+        post :preview_markdown
         post :export
         post :remove_export
         post :generate_new_export
@@ -482,11 +479,26 @@ Rails.application.routes.draw do
       end
 
       scope module: :projects do
-        # Git HTTP clients ('git clone' etc.)
         scope constraints: { id: /.+\.git/, format: nil } do
+          # Git HTTP clients ('git clone' etc.)
           get '/info/refs', to: 'git_http#info_refs'
           post '/git-upload-pack', to: 'git_http#git_upload_pack'
           post '/git-receive-pack', to: 'git_http#git_receive_pack'
+
+          # Git LFS API (metadata)
+          post '/info/lfs/objects/batch', to: 'lfs_api#batch'
+          post '/info/lfs/objects', to: 'lfs_api#deprecated'
+          get '/info/lfs/objects/*oid', to: 'lfs_api#deprecated'
+
+          # GitLab LFS object storage
+          scope constraints: { oid: /[a-f0-9]{64}/ } do
+            get '/gitlab-lfs/objects/*oid', to: 'lfs_storage#download'
+
+            scope constraints: { size: /[0-9]+/ } do
+              put '/gitlab-lfs/objects/*oid/*size/authorize', to: 'lfs_storage#upload_authorize'
+              put '/gitlab-lfs/objects/*oid/*size', to: 'lfs_storage#upload_finalize'
+            end
+          end
         end
 
         # Allow /info/refs, /info/refs?service=git-upload-pack, and
@@ -660,7 +672,7 @@ Rails.application.routes.draw do
           get '/wikis/*id', to: 'wikis#show', as: 'wiki', constraints: WIKI_SLUG_ID
           delete '/wikis/*id', to: 'wikis#destroy', constraints: WIKI_SLUG_ID
           put '/wikis/*id', to: 'wikis#update', constraints: WIKI_SLUG_ID
-          post '/wikis/*id/markdown_preview', to: 'wikis#markdown_preview', constraints: WIKI_SLUG_ID, as: 'wiki_markdown_preview'
+          post '/wikis/*id/preview_markdown', to: 'wikis#preview_markdown', constraints: WIKI_SLUG_ID, as: 'wiki_preview_markdown'
         end
 
         resource :repository, only: [:create] do