Merge remote-tracking branch 'upstream/master' into artifacts-from-ref-and-build-name

* upstream/master: (516 commits)
  Rename `run` task helper method to prevent conflict with StateMachine
  Add a method in Project to return a cached value of total count of projects
  Add Changelog entry for Grape upgrade [ci skip]
  Fix Grape tests.
  Retain old behavior
  Update Grape from 0.13.0 to 0.15.0.
  adds second batch of tests changed to active tense
  fixes part1 of files to start using active tense
  Clarify the features for generating default label sets
  Update tree view to sort folders with submodules
  Memoize CI config node validator to prevent leaks
  fix MR source project assignment
  remove offending empty line
  Rails prefers require_dependency so that it won't require twice:
  use Unix line endings for API documentation
  use long options for curl examples in API documentation (!5703)
  Prefixes removed branches name with PR number when importing PR from GH
  Update CHANGELOG
  Remove SHA suffix for removed branches name when importing PR from GH
  add linting script for documentation
  ...

13 files changed, 132 insertions, 81 deletions

diff --git a/config/application.rb b/config/application.rb
index 06ebb14a5fe..4a9ed41cbf8 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -107,7 +107,8 @@ module Gitlab
       end
     end
 
-    redis_config_hash = Gitlab::Redis.redis_store_options
+    # Use Redis caching across all environments
+    redis_config_hash = Gitlab::Redis.params
     redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE
     redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
     config.cache_store = :redis_store, redis_config_hash
diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml
index 293f2b71d65..74325872b09 100644
--- a/config/dependency_decisions.yml
+++ b/config/dependency_decisions.yml
@@ -68,6 +68,25 @@
     :why: https://opensource.org/licenses/BSD-2-Clause
     :versions: []
     :when: 2016-05-02 05:55:09.796363000 Z
+- - :whitelist
+  - LGPLv2+
+  - :who: Stan Hu
+    :why: Equivalent to LGPLv2
+    :versions: []
+    :when: 2016-06-07 17:14:10.907682000 Z
+- - :whitelist
+  - Artistic 2.0
+  - :who: Josh Frye
+    :why: Disk/mount information display on Admin pages
+    :versions: []
+    :when: 2016-06-29 16:32:45.432113000 Z
+- - :whitelist
+  - Simplified BSD
+  - :who: Douwe Maan
+    :why: https://opensource.org/licenses/BSD-2-Clause
+    :versions: []
+    :when: 2016-07-26 21:24:07.248480000 Z
+
 
 # LICENSE BLACKLIST
 - - :blacklist
@@ -175,15 +194,3 @@
     :why: https://github.com/jmcnevin/rubypants/blob/master/LICENSE.rdoc
     :versions: []
     :when: 2016-05-02 05:56:50.696858000 Z
-- - :whitelist
-  - LGPLv2+
-  - :who: Stan Hu
-    :why: Equivalent to LGPLv2
-    :versions: []
-    :when: 2016-06-07 17:14:10.907682000 Z
-- - :whitelist
-  - Artistic 2.0
-  - :who: Josh Frye
-    :why: Disk/mount information display on Admin pages
-    :versions: []
-    :when: 2016-06-29 16:32:45.432113000 Z
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 86f55210487..deac3b0f0f9 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -287,9 +287,12 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
 Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
-Settings.cron_jobs['gitlab_remove_project_export_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['gitlab_remove_project_export_worker']['cron'] ||= '0 * * * *'
-Settings.cron_jobs['gitlab_remove_project_export_worker']['job_class'] = 'GitlabRemoveProjectExportWorker'
+Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
+Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
+Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
+Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'
 
 #
 # GitLab Shell
diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb
index 37746968675..d92f64e1647 100644
--- a/config/initializers/6_validations.rb
+++ b/config/initializers/6_validations.rb
@@ -26,4 +26,4 @@ def validate_storages
   end
 end
 
-validate_storages unless Rails.env.test?
+validate_storages unless Rails.env.test? || ENV['SKIP_STORAGE_VALIDATION'] == 'true'
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index 73977341b73..a0a8f88584c 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -100,6 +100,9 @@ Devise.setup do |config|
   # secure: true in order to force SSL only cookies.
   # config.cookie_options = {}
 
+  # Send a notification email when the user's password is changed
+  config.send_password_change_notification = true
+
   # ==> Configuration for :validatable
   # Range for password length. Default is 6..128.
   config.password_length = 8..128
diff --git a/config/initializers/metrics.rb b/config/initializers/metrics.rb
index c4266ab8ba5..cc8208db3c1 100644
--- a/config/initializers/metrics.rb
+++ b/config/initializers/metrics.rb
@@ -136,7 +136,18 @@ if Gitlab::Metrics.enabled?
     config.instrument_instance_methods(Rouge::Plugins::Redcarpet)
     config.instrument_instance_methods(Rouge::Formatters::HTMLGitlab)
 
+    [:XML, :HTML].each do |namespace|
+      namespace_mod = Nokogiri.const_get(namespace)
+
+      config.instrument_methods(namespace_mod)
+      config.instrument_methods(namespace_mod::Document)
+    end
+
     config.instrument_methods(Rinku)
+    config.instrument_instance_methods(Repository)
+
+    config.instrument_methods(Gitlab::Highlight)
+    config.instrument_instance_methods(Gitlab::Highlight)
   end
 
   GC::Profiler.enable
diff --git a/config/initializers/request_profiler.rb b/config/initializers/request_profiler.rb
new file mode 100644
index 00000000000..a9aa802681a
--- /dev/null
+++ b/config/initializers/request_profiler.rb
@@ -0,0 +1,5 @@
+require 'gitlab/request_profiler/middleware'
+
+Rails.application.configure do |config|
+  config.middleware.use(Gitlab::RequestProfiler::Middleware)
+end
diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb
index 0d9d87bac00..70be2617cab 100644
--- a/config/initializers/session_store.rb
+++ b/config/initializers/session_store.rb
@@ -13,9 +13,9 @@ end
 if Rails.env.test?
   Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
 else
-  redis_config = Gitlab::Redis.redis_store_options
+  redis_config = Gitlab::Redis.params
   redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE
-  
+
   Gitlab::Application.config.session_store(
     :redis_store, # Using the cookie_store would enable session replay attacks.
     servers: redis_config,
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index 5e839327e7a..f7e714cd6bc 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -1,12 +1,14 @@
+# Custom Redis configuration
+redis_config_hash = Gitlab::Redis.params
+redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
+
 Sidekiq.configure_server do |config|
-  config.redis = {
-    url: Gitlab::Redis.url,
-    namespace: Gitlab::Redis::SIDEKIQ_NAMESPACE
-  }
+  config.redis = redis_config_hash
 
   config.server_middleware do |chain|
     chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
     chain.add Gitlab::SidekiqMiddleware::MemoryKiller if ENV['SIDEKIQ_MEMORY_KILLER_MAX_RSS']
+    chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware unless ENV['SIDEKIQ_REQUEST_STORE'] == '0'
   end
 
   # Sidekiq-cron: load recurring jobs from gitlab.yml
@@ -38,8 +40,5 @@ Sidekiq.configure_server do |config|
 end
 
 Sidekiq.configure_client do |config|
-  config.redis = {
-    url: Gitlab::Redis.url,
-    namespace: Gitlab::Redis::SIDEKIQ_NAMESPACE
-  }
+  config.redis = redis_config_hash
 end
diff --git a/config/initializers/trusted_proxies.rb b/config/initializers/trusted_proxies.rb
index 30770b71e24..cd869657c53 100644
--- a/config/initializers/trusted_proxies.rb
+++ b/config/initializers/trusted_proxies.rb
@@ -7,6 +7,8 @@ module Rack
   class Request
     def trusted_proxy?(ip)
       Rails.application.config.action_dispatch.trusted_proxies.any? { |proxy| proxy === ip }
+    rescue IPAddr::InvalidAddressError
+      false
     end
   end
 end
diff --git a/config/mail_room.yml b/config/mail_room.yml
index 7cab24b295e..c639f8260aa 100644
--- a/config/mail_room.yml
+++ b/config/mail_room.yml
@@ -1,47 +1,36 @@
+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
 :mailboxes:
-<%
-require "yaml"
-require "json"
-require_relative "lib/gitlab/redis" unless defined?(Gitlab::Redis)
+  <%
+    require_relative "lib/gitlab/mail_room" unless defined?(Gitlab::MailRoom)
+    config = Gitlab::MailRoom.config
 
-rails_env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
-
-config_file = ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] || "config/gitlab.yml"
-if File.exists?(config_file)
-  all_config = YAML.load_file(config_file)[rails_env]
-
-  config = all_config["incoming_email"] || {}
-  config['enabled']    = false    if config['enabled'].nil?
-  config['port']       = 143      if config['port'].nil?
-  config['ssl']        = false    if config['ssl'].nil?
-  config['start_tls']  = false    if config['start_tls'].nil?
-  config['mailbox']    = "inbox"  if config['mailbox'].nil?
-
-  if config['enabled'] && config['address']
-    redis_url = Gitlab::Redis.new(rails_env).url
-    %>
+    if Gitlab::MailRoom.enabled?
+  %>
     -
-      :host: <%= config['host'].to_json %>
-      :port: <%= config['port'].to_json %>
-      :ssl: <%= config['ssl'].to_json %>
-      :start_tls: <%= config['start_tls'].to_json %>
-      :email: <%= config['user'].to_json %>
-      :password: <%= config['password'].to_json %>
+      :host: <%= config[:host].to_json %>
+      :port: <%= config[:port].to_json %>
+      :ssl: <%= config[:ssl].to_json %>
+      :start_tls: <%= config[:start_tls].to_json %>
+      :email: <%= config[:user].to_json %>
+      :password: <%= config[:password].to_json %>
+      :idle_timeout: 60
 
-      :name: <%= config['mailbox'].to_json %>
+      :name: <%= config[:mailbox].to_json %>
 
       :delete_after_delivery: true
 
       :delivery_method: sidekiq
       :delivery_options:
-        :redis_url: <%= redis_url.to_json %>
-        :namespace: resque:gitlab
+        :redis_url: <%= config[:redis_url].to_json %>
+        :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %>
         :queue: incoming_email
         :worker: EmailReceiverWorker
 
       :arbitration_method: redis
       :arbitration_options:
-        :redis_url: <%= redis_url.to_json %>
-        :namespace: mail_room:gitlab
+        :redis_url: <%= config[:redis_url].to_json %>
+        :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %>
+
   <% end %>
-<% end %>
diff --git a/config/resque.yml.example b/config/resque.yml.example
index d98f43f71b2..0c19d8bc1d3 100644
--- a/config/resque.yml.example
+++ b/config/resque.yml.example
@@ -1,6 +1,34 @@
 # If you change this file in a Merge Request, please also create
 # a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
 #
-development: redis://localhost:6379
-test: redis://localhost:6379
-production: unix:/var/run/redis/redis.sock
+development:
+  url: redis://localhost:6379
+  # sentinels:
+  #   -
+  #     host: localhost
+  #     port: 26380 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26381 # point to sentinel, not to redis port
+test:
+  url: redis://localhost:6379
+production:
+  # Redis (single instance)
+  url: unix:/var/run/redis/redis.sock
+  ##
+  # Redis + Sentinel (for HA)
+  #
+  # Please read instructions carefully before using it as you may lose data:
+  # http://redis.io/topics/sentinel
+  #
+  # You must specify a list of a few sentinels that will handle client connection
+  # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+  ##
+  # url: redis://master:6379
+  # sentinels:
+  #   -
+  #     host: slave1
+  #     port: 26379 # point to sentinel, not to redis port
+  #   -
+  #     host: slave2
+  #     port: 26379 # point to sentinel, not to redis port
diff --git a/config/routes.rb b/config/routes.rb
index ccc50f5cd08..256c7dacd59 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -42,10 +42,9 @@ Rails.application.routes.draw do
 
     resource :lint, only: [:show, :create]
 
-    resources :projects do
+    resources :projects, only: [:index, :show] do
       member do
         get :status, to: 'projects#badge'
-        get :integration
       end
     end
 
@@ -144,13 +143,13 @@ Rails.application.routes.draw do
       get :jobs
     end
 
-    resource :gitlab, only: [:create, :new], controller: :gitlab do
+    resource :gitlab, only: [:create], controller: :gitlab do
       get :status
       get :callback
       get :jobs
     end
 
-    resource :bitbucket, only: [:create, :new], controller: :bitbucket do
+    resource :bitbucket, only: [:create], controller: :bitbucket do
       get :status
       get :callback
       get :jobs
@@ -243,7 +242,6 @@ Rails.application.routes.draw do
         get :projects
         get :keys
         get :groups
-        put :team_update
         put :block
         put :unblock
         put :unlock
@@ -281,6 +279,7 @@ Rails.application.routes.draw do
     resource :health_check, controller: 'health_check', only: [:show]
     resource :background_jobs, controller: 'background_jobs', only: [:show]
     resource :system_info, controller: 'system_info', only: [:show]
+    resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ }
 
     resources :namespaces, path: '/projects', constraints: { id: /[a-zA-Z.0-9_\-]+/ }, only: [] do
       root to: 'projects#index', as: :projects
@@ -300,7 +299,7 @@ Rails.application.routes.draw do
       end
     end
 
-    resource :appearances, path: 'appearance' do
+    resource :appearances, only: [:show, :create, :update], path: 'appearance' do
       member do
         get :preview
         delete :logo
@@ -309,7 +308,7 @@ Rails.application.routes.draw do
     end
 
     resource :application_settings, only: [:show, :update] do
-      resources :services
+      resources :services, only: [:index, :edit, :update]
       put :reset_runners_token
       put :reset_health_check_token
       put :clear_repository_check_states
@@ -346,7 +345,7 @@ Rails.application.routes.draw do
     end
 
     scope module: :profiles do
-      resource :account, only: [:show, :update] do
+      resource :account, only: [:show] do
         member do
           delete :unlink
         end
@@ -358,7 +357,7 @@ Rails.application.routes.draw do
         end
       end
       resource :preferences, only: [:show, :update]
-      resources :keys
+      resources :keys, only: [:index, :show, :new, :create, :destroy]
       resources :emails, only: [:index, :create, :destroy]
       resource :avatar, only: [:destroy]
 
@@ -627,13 +626,17 @@ Rails.application.routes.draw do
 
         get '/compare/:from...:to', to: 'compare#show', as: 'compare', constraints: { from: /.+/, to: /.+/ }
 
-        resources :network, only: [:show], constraints: { id: /(?:[^.]|\.(?!json$))+/, format: /json/ }
+        # Don't use format parameter as file extension (old 3.0.x behavior)
+        # See http://guides.rubyonrails.org/routing.html#route-globbing-and-wildcard-segments
+        scope format: false do
+          resources :network, only: [:show], constraints: { id: Gitlab::Regex.git_reference_regex }
 
-        resources :graphs, only: [:show], constraints: { id: /(?:[^.]|\.(?!json$))+/, format: /json/ } do
-          member do
-            get :commits
-            get :ci
-            get :languages
+          resources :graphs, only: [:show], constraints: { id: Gitlab::Regex.git_reference_regex } do
+            member do
+              get :commits
+              get :ci
+              get :languages
+            end
           end
         end
 
@@ -660,7 +663,7 @@ Rails.application.routes.draw do
           post '/wikis/*id/markdown_preview', to: 'wikis#markdown_preview', constraints: WIKI_SLUG_ID, as: 'wiki_markdown_preview'
         end
 
-        resource :repository, only: [:show, :create] do
+        resource :repository, only: [:create] do
           member do
             get 'archive', constraints: { format: Gitlab::Regex.archive_formats_regex }
           end
@@ -742,7 +745,7 @@ Rails.application.routes.draw do
           end
         end
 
-        resources :environments, only: [:index, :show, :new, :create, :destroy]
+        resources :environments
 
         resources :builds, only: [:index, :show], constraints: { id: /\d+/ } do
           collection do
@@ -790,7 +793,7 @@ Rails.application.routes.draw do
           end
         end
 
-        resources :labels, constraints: { id: /\d+/ } do
+        resources :labels, except: [:show], constraints: { id: /\d+/ } do
           collection do
             post :generate
             post :set_priorities
@@ -815,7 +818,7 @@ Rails.application.routes.draw do
           end
         end
 
-        resources :project_members, except: [:new, :edit], constraints: { id: /[a-zA-Z.\/0-9_\-#%+]+/ }, concerns: :access_requestable do
+        resources :project_members, except: [:show, :new, :edit], constraints: { id: /[a-zA-Z.\/0-9_\-#%+]+/ }, concerns: :access_requestable do
           collection do
             delete :leave