Merge branch 'master' into persistent-callouts

12 files changed, 126 insertions, 63 deletions

diff --git a/config/application.rb b/config/application.rb
index 2067428ff62..751307de975 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -6,6 +6,7 @@ Bundler.require(:default, Rails.env)
 
 module Gitlab
   class Application < Rails::Application
+    require_dependency Rails.root.join('lib/gitlab/redis/wrapper')
     require_dependency Rails.root.join('lib/gitlab/redis/cache')
     require_dependency Rails.root.join('lib/gitlab/redis/queues')
     require_dependency Rails.root.join('lib/gitlab/redis/shared_state')
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 25f4085deb2..33230b9355d 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -152,6 +152,12 @@ production: &base
     # The location where LFS objects are stored (default: shared/lfs-objects).
     # storage_path: shared/lfs-objects
 
+  ## Uploads (attachments, avatars, etc...)
+  uploads:
+    # The location where uploads objects are stored (default: public/).
+    # storage_path: public/
+    # base_dir: uploads/-/system
+
   ## GitLab Pages
   pages:
     enabled: false
@@ -644,6 +650,8 @@ test:
     enabled: false
   artifacts:
     path: tmp/tests/artifacts
+  uploads:
+    storage_path: tmp/tests/public
   gitlab:
     host: localhost
     port: 80
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 899e612ffbd..5ad46d47cb6 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -110,7 +110,7 @@ class Settings < Settingslogic
       url = "http://#{url}" unless url.start_with?('http')
 
       # Get rid of the path so that we don't even have to encode it
-      url_without_path = url.sub(%r{(https?://[^\/]+)/?.*}, '\1')
+      url_without_path = url.sub(%r{(https?://[^/]+)/?.*}, '\1')
 
       URI.parse(url_without_path).host
     end
@@ -300,8 +300,10 @@ Settings.incoming_email['enabled'] = false if Settings.incoming_email['enabled']
 #
 Settings['artifacts'] ||= Settingslogic.new({})
 Settings.artifacts['enabled']      = true if Settings.artifacts['enabled'].nil?
-Settings.artifacts['path']         = Settings.absolute(Settings.artifacts['path'] || File.join(Settings.shared['path'], "artifacts"))
-Settings.artifacts['max_size']   ||= 100 # in megabytes
+Settings.artifacts['storage_path'] = Settings.absolute(Settings.artifacts.values_at('path', 'storage_path').compact.first || File.join(Settings.shared['path'], "artifacts"))
+# Settings.artifact['path'] is deprecated, use `storage_path` instead
+Settings.artifacts['path']         = Settings.artifacts['storage_path']
+Settings.artifacts['max_size'] ||= 100 # in megabytes
 
 #
 # Registry
@@ -339,6 +341,13 @@ Settings.lfs['enabled']      = true if Settings.lfs['enabled'].nil?
 Settings.lfs['storage_path'] = Settings.absolute(Settings.lfs['storage_path'] || File.join(Settings.shared['path'], "lfs-objects"))
 
 #
+# Uploads
+#
+Settings['uploads'] ||= Settingslogic.new({})
+Settings.uploads['storage_path'] = Settings.absolute(Settings.uploads['storage_path'] || 'public')
+Settings.uploads['base_dir'] = Settings.uploads['base_dir'] || 'uploads/-/system'
+
+#
 # Mattermost
 #
 Settings['mattermost'] ||= Settingslogic.new({})
@@ -469,10 +478,10 @@ end
 # repository_downloads_path value.
 #
 repositories_storages          = Settings.repositories.storages.values
-repository_downloads_path      = Settings.gitlab['repository_downloads_path'].to_s.gsub(/\/$/, '')
+repository_downloads_path      = Settings.gitlab['repository_downloads_path'].to_s.gsub(%r{/$}, '')
 repository_downloads_full_path = File.expand_path(repository_downloads_path, Settings.gitlab['user_home'])
 
-if repository_downloads_path.blank? || repositories_storages.any? { |rs| [repository_downloads_path, repository_downloads_full_path].include?(rs['path'].gsub(/\/$/, '')) }
+if repository_downloads_path.blank? || repositories_storages.any? { |rs| [repository_downloads_path, repository_downloads_full_path].include?(rs['path'].gsub(%r{/$}, '')) }
   Settings.gitlab['repository_downloads_path'] = File.join(Settings.shared['path'], 'cache/archive')
 end
 
diff --git a/config/initializers/active_record_data_types.rb b/config/initializers/active_record_data_types.rb
index 0359e14b232..fda13d0c4cb 100644
--- a/config/initializers/active_record_data_types.rb
+++ b/config/initializers/active_record_data_types.rb
@@ -54,7 +54,7 @@ elsif Gitlab::Database.mysql?
     def initialize_type_map(mapping)
       super mapping
 
-      mapping.register_type(%r(timestamp)i) do |sql_type|
+      mapping.register_type(/timestamp/i) do |sql_type|
         precision = extract_precision(sql_type)
         ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter::MysqlDateTimeWithTimeZone.new(precision: precision)
       end
diff --git a/config/initializers/ar5_pg_10_support.rb b/config/initializers/ar5_pg_10_support.rb
index 6fae770015c..a529c74a8ce 100644
--- a/config/initializers/ar5_pg_10_support.rb
+++ b/config/initializers/ar5_pg_10_support.rb
@@ -1,57 +1,59 @@
 raise "Vendored ActiveRecord 5 code! Delete #{__FILE__}!" if ActiveRecord::VERSION::MAJOR >= 5
 
-require 'active_record/connection_adapters/postgresql_adapter'
-require 'active_record/connection_adapters/postgresql/schema_statements'
-
-#
-# Monkey-patch the refused Rails 4.2 patch at https://github.com/rails/rails/pull/31330
-#
-# Updates sequence logic to support PostgreSQL 10.
-#
-# rubocop:disable all
-module ActiveRecord
-  module ConnectionAdapters
-
-    # We need #postgresql_version to be public as in ActiveRecord 5 for seed_fu
-    # to work. In ActiveRecord 4, it is protected.
-    # https://github.com/mbleigh/seed-fu/issues/123
-    class PostgreSQLAdapter
-      public :postgresql_version
-    end
+if Gitlab::Database.postgresql?
+  require 'active_record/connection_adapters/postgresql_adapter'
+  require 'active_record/connection_adapters/postgresql/schema_statements'
+
+  #
+  # Monkey-patch the refused Rails 4.2 patch at https://github.com/rails/rails/pull/31330
+  #
+  # Updates sequence logic to support PostgreSQL 10.
+  #
+  # rubocop:disable all
+  module ActiveRecord
+    module ConnectionAdapters
+
+      # We need #postgresql_version to be public as in ActiveRecord 5 for seed_fu
+      # to work. In ActiveRecord 4, it is protected.
+      # https://github.com/mbleigh/seed-fu/issues/123
+      class PostgreSQLAdapter
+        public :postgresql_version
+      end
 
-    module PostgreSQL
-      module SchemaStatements
-        # Resets the sequence of a table's primary key to the maximum value.
-        def reset_pk_sequence!(table, pk = nil, sequence = nil) #:nodoc:
-          unless pk and sequence
-            default_pk, default_sequence = pk_and_sequence_for(table)
+      module PostgreSQL
+        module SchemaStatements
+          # Resets the sequence of a table's primary key to the maximum value.
+          def reset_pk_sequence!(table, pk = nil, sequence = nil) #:nodoc:
+            unless pk and sequence
+              default_pk, default_sequence = pk_and_sequence_for(table)
 
-            pk ||= default_pk
-            sequence ||= default_sequence
-          end
+              pk ||= default_pk
+              sequence ||= default_sequence
+            end
 
-          if @logger && pk && !sequence
-            @logger.warn "#{table} has primary key #{pk} with no default sequence"
-          end
+            if @logger && pk && !sequence
+              @logger.warn "#{table} has primary key #{pk} with no default sequence"
+            end
 
-          if pk && sequence
-            quoted_sequence = quote_table_name(sequence)
-            max_pk = select_value("SELECT MAX(#{quote_column_name pk}) FROM #{quote_table_name(table)}")
-            if max_pk.nil?
-              if postgresql_version >= 100000
-                minvalue = select_value("SELECT seqmin FROM pg_sequence WHERE seqrelid = #{quote(quoted_sequence)}::regclass")
-              else
-                minvalue = select_value("SELECT min_value FROM #{quoted_sequence}")
+            if pk && sequence
+              quoted_sequence = quote_table_name(sequence)
+              max_pk = select_value("SELECT MAX(#{quote_column_name pk}) FROM #{quote_table_name(table)}")
+              if max_pk.nil?
+                if postgresql_version >= 100000
+                  minvalue = select_value("SELECT seqmin FROM pg_sequence WHERE seqrelid = #{quote(quoted_sequence)}::regclass")
+                else
+                  minvalue = select_value("SELECT min_value FROM #{quoted_sequence}")
+                end
               end
-            end
 
-            select_value <<-end_sql, 'SCHEMA'
-              SELECT setval(#{quote(quoted_sequence)}, #{max_pk ? max_pk : minvalue}, #{max_pk ? true : false})
-            end_sql
+              select_value <<-end_sql, 'SCHEMA'
+                SELECT setval(#{quote(quoted_sequence)}, #{max_pk ? max_pk : minvalue}, #{max_pk ? true : false})
+              end_sql
+            end
           end
         end
       end
     end
   end
+  # rubocop:enable all
 end
-# rubocop:enable all
diff --git a/config/initializers/grape_route_helpers_fix.rb b/config/initializers/grape_route_helpers_fix.rb
index d3cf9e453d0..612cca3dfbd 100644
--- a/config/initializers/grape_route_helpers_fix.rb
+++ b/config/initializers/grape_route_helpers_fix.rb
@@ -1,5 +1,21 @@
 if defined?(GrapeRouteHelpers)
   module GrapeRouteHelpers
+    module AllRoutes
+      # Bringing in PR https://github.com/reprah/grape-route-helpers/pull/21 due to abandonment.
+      #
+      # Without the following fix, when two helper methods are the same, but have different arguments
+      # (for example: api_v1_cats_owners_path(id: 1) vs api_v1_cats_owners_path(id: 1, owner_id: 2))
+      # if the helper method with the least number of arguments is defined first (because the route was defined first)
+      # then it will shadow the longer route.
+      #
+      # The fix is to sort descending by amount of arguments
+      def decorated_routes
+        @decorated_routes ||= all_routes
+                                .map { |r| DecoratedRoute.new(r) }
+                                .sort_by { |r| -r.dynamic_path_segments.count }
+      end
+    end
+
     class DecoratedRoute
       # GrapeRouteHelpers gem tries to parse the versions
       # from a string, not supporting Grape `version` array definition.
diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb
index e74b95f1646..11759801112 100644
--- a/config/initializers/peek.rb
+++ b/config/initializers/peek.rb
@@ -7,10 +7,12 @@ if Gitlab::Database.mysql?
   require 'peek-mysql2'
   PEEK_DB_CLIENT = ::Mysql2::Client
   PEEK_DB_VIEW = Peek::Views::Mysql2
-else
+elsif Gitlab::Database.postgresql?
   require 'peek-pg'
   PEEK_DB_CLIENT = ::PG::Connection
   PEEK_DB_VIEW = Peek::Views::PG
+else
+  raise "Unsupported database adapter for peek!"
 end
 
 Peek.into PEEK_DB_VIEW
diff --git a/config/routes/admin.rb b/config/routes/admin.rb
index e22fb440abc..3cca1210e39 100644
--- a/config/routes/admin.rb
+++ b/config/routes/admin.rb
@@ -1,5 +1,5 @@
 namespace :admin do
-  resources :users, constraints: { id: /[a-zA-Z.\/0-9_\-]+/ } do
+  resources :users, constraints: { id: %r{[a-zA-Z./0-9_\-]+} } do
     resources :keys, only: [:show, :destroy]
     resources :identities, except: [:show]
     resources :impersonation_tokens, only: [:index, :create] do
@@ -24,6 +24,8 @@ namespace :admin do
   resource :impersonation, only: :destroy
 
   resources :abuse_reports, only: [:index, :destroy]
+  resources :gitaly_servers, only: [:index]
+
   resources :spam_logs, only: [:index, :destroy] do
     member do
       post :mark_as_ham
diff --git a/config/routes/group.rb b/config/routes/group.rb
index 976837a246d..24c76bc55ab 100644
--- a/config/routes/group.rb
+++ b/config/routes/group.rb
@@ -35,7 +35,7 @@ constraints(GroupUrlConstrainer.new) do
       post :toggle_subscription, on: :member
     end
 
-    resources :milestones, constraints: { id: /[^\/]+/ }, only: [:index, :show, :edit, :update, :new, :create] do
+    resources :milestones, constraints: { id: %r{[^/]+} }, only: [:index, :show, :edit, :update, :new, :create] do
       member do
         get :merge_requests
         get :participants
@@ -52,7 +52,7 @@ constraints(GroupUrlConstrainer.new) do
 
     resources :uploads, only: [:create] do
       collection do
-        get ":secret/:filename", action: :show, as: :show, constraints: { filename: /[^\/]+/ }
+        get ":secret/:filename", action: :show, as: :show, constraints: { filename: %r{[^/]+} }
       end
     end
   end
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 0496bd85b4e..bcaa68c8ce5 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -40,7 +40,7 @@ constraints(ProjectUrlConstrainer.new) do
       #
       # Templates
       #
-      get '/templates/:template_type/:key' => 'templates#show', as: :template, constraints: { key: /[^\/]+/ }
+      get '/templates/:template_type/:key' => 'templates#show', as: :template, constraints: { key: %r{[^/]+} }
 
       resource  :avatar, only: [:show, :destroy]
       resources :commit, only: [:show], constraints: { id: /\h{7,40}/ } do
@@ -55,7 +55,7 @@ constraints(ProjectUrlConstrainer.new) do
       end
 
       resource :pages, only: [:show, :destroy] do
-        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: /[^\/]+/ }
+        resources :domains, only: [:show, :new, :create, :destroy], controller: 'pages_domains', constraints: { id: %r{[^/]+} }
       end
 
       resources :snippets, concerns: :awardable, constraints: { id: /\d+/ } do
@@ -65,7 +65,7 @@ constraints(ProjectUrlConstrainer.new) do
         end
       end
 
-      resources :services, constraints: { id: /[^\/]+/ }, only: [:index, :edit, :update] do
+      resources :services, constraints: { id: %r{[^/]+} }, only: [:index, :edit, :update] do
         member do
           put :test
         end
@@ -346,7 +346,7 @@ constraints(ProjectUrlConstrainer.new) do
         end
       end
 
-      resources :project_members, except: [:show, :new, :edit], constraints: { id: /[a-zA-Z.\/0-9_\-#%+]+/ }, concerns: :access_requestable do
+      resources :project_members, except: [:show, :new, :edit], constraints: { id: %r{[a-zA-Z./0-9_\-#%+]+} }, concerns: :access_requestable do
         collection do
           delete :leave
 
@@ -379,7 +379,7 @@ constraints(ProjectUrlConstrainer.new) do
 
       resources :uploads, only: [:create] do
         collection do
-          get ":secret/:filename", action: :show, as: :show, constraints: { filename: /[^\/]+/ }
+          get ":secret/:filename", action: :show, as: :show, constraints: { filename: %r{[^/]+} }
         end
       end
 
diff --git a/config/routes/uploads.rb b/config/routes/uploads.rb
index d7bca8310e4..6370645bcb9 100644
--- a/config/routes/uploads.rb
+++ b/config/routes/uploads.rb
@@ -2,17 +2,17 @@ scope path: :uploads do
   # Note attachments and User/Group/Project avatars
   get "-/system/:model/:mounted_as/:id/:filename",
       to:           "uploads#show",
-      constraints:  { model: /note|user|group|project/, mounted_as: /avatar|attachment/, filename: /[^\/]+/ }
+      constraints:  { model: /note|user|group|project/, mounted_as: /avatar|attachment/, filename: %r{[^/]+} }
 
   # show uploads for models, snippets (notes) available for now
   get '-/system/:model/:id/:secret/:filename',
     to: 'uploads#show',
-    constraints: { model: /personal_snippet/, id: /\d+/, filename: /[^\/]+/ }
+    constraints: { model: /personal_snippet/, id: /\d+/, filename: %r{[^/]+} }
 
   # show temporary uploads
   get '-/system/temp/:secret/:filename',
     to: 'uploads#show',
-    constraints: { filename: /[^\/]+/ }
+    constraints: { filename: %r{[^/]+} }
 
   # Appearance
   get "-/system/:model/:mounted_as/:id/:filename",
@@ -22,7 +22,7 @@ scope path: :uploads do
   # Project markdown uploads
   get ":namespace_id/:project_id/:secret/:filename",
     to:           "projects/uploads#show",
-    constraints:  { namespace_id: /[a-zA-Z.0-9_\-]+/, project_id: /[a-zA-Z.0-9_\-]+/, filename: /[^\/]+/ }
+    constraints:  { namespace_id: /[a-zA-Z.0-9_\-]+/, project_id: /[a-zA-Z.0-9_\-]+/, filename: %r{[^/]+} }
 
   # create uploads for models, snippets (notes) available for now
   post ':model',
@@ -34,4 +34,4 @@ end
 # Redirect old note attachments path to new uploads path.
 get "files/note/:id/:filename",
   to:           redirect("uploads/note/attachment/%{id}/%{filename}"),
-  constraints:  { filename: /[^\/]+/ }
+  constraints:  { filename: %r{[^/]+} }
diff --git a/config/webpack.config.js b/config/webpack.config.js
index 783677b5b8d..7f3fe551a03 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -3,6 +3,7 @@
 var crypto = require('crypto');
 var fs = require('fs');
 var path = require('path');
+var glob = require('glob');
 var webpack = require('webpack');
 var StatsWriterPlugin = require('webpack-stats-plugin').StatsWriterPlugin;
 var CopyWebpackPlugin = require('copy-webpack-plugin');
@@ -20,6 +21,26 @@ var DEV_SERVER_LIVERELOAD = process.env.DEV_SERVER_LIVERELOAD !== 'false';
 var WEBPACK_REPORT = process.env.WEBPACK_REPORT;
 var NO_COMPRESSION = process.env.NO_COMPRESSION;
 
+// generate automatic entry points
+var autoEntries = {};
+var pageEntries = glob.sync('pages/**/index.js', { cwd: path.join(ROOT_PATH, 'app/assets/javascripts') });
+
+// filter out entries currently imported dynamically in dispatcher.js
+var dispatcher = fs.readFileSync(path.join(ROOT_PATH, 'app/assets/javascripts/dispatcher.js')).toString();
+var dispatcherChunks = dispatcher.match(/(?!import\('.\/)pages\/[^']+/g);
+
+pageEntries.forEach(( path ) => {
+  let chunkPath = path.replace(/\/index\.js$/, '');
+  if (!dispatcherChunks.includes(chunkPath)) {
+    let chunkName = chunkPath.replace(/\//g, '.');
+    autoEntries[chunkName] = './' + path;
+  }
+});
+
+// report our auto-generated bundle count
+var autoEntriesCount = Object.keys(autoEntries).length;
+console.log(`${autoEntriesCount} entries from '/pages' automatically added to webpack output.`);
+
 var config = {
   // because sqljs requires fs.
   node: {
@@ -301,6 +322,8 @@ var config = {
   }
 }
 
+config.entry = Object.assign({}, autoEntries, config.entry);
+
 if (IS_PRODUCTION) {
   config.devtool = 'source-map';
   config.plugins.push(