Merge remote-tracking branch 'upstream/master' into 28717-additional-metrics-review-branch

14 files changed, 219 insertions, 54 deletions

diff --git a/config/initializers/acts_as_taggable.rb b/config/initializers/0_acts_as_taggable.rb
index c564c0cab11..54e9fcc31db 100644
--- a/config/initializers/acts_as_taggable.rb
+++ b/config/initializers/0_acts_as_taggable.rb
@@ -3,3 +3,7 @@ ActsAsTaggableOn.strict_case_match = true
 # tags_counter enables caching count of tags which results in an update whenever a tag is added or removed
 # since the count is not used anywhere its better performance wise to disable this cache
 ActsAsTaggableOn.tags_counter = false
+
+# validate that counter cache is disabled
+raise "Counter cache is not disabled" if 
+    ActsAsTaggableOn::Tagging.reflections["tag"].options[:counter_cache]
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 5a90830b5b3..4fb4baf631f 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -368,11 +368,14 @@ Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_random_weekly_time)
 Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
 
-# Every day at 00:30
 Settings.cron_jobs['schedule_update_user_activity_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['schedule_update_user_activity_worker']['cron'] ||= '30 0 * * *'
 Settings.cron_jobs['schedule_update_user_activity_worker']['job_class'] = 'ScheduleUpdateUserActivityWorker'
 
+Settings.cron_jobs['remove_old_web_hook_logs_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['remove_old_web_hook_logs_worker']['cron'] ||= '40 0 * * *'
+Settings.cron_jobs['remove_old_web_hook_logs_worker']['job_class'] = 'RemoveOldWebHookLogsWorker'
+
 #
 # GitLab Shell
 #
diff --git a/config/initializers/fast_gettext.rb b/config/initializers/fast_gettext.rb
index a69fe0c902e..eb589ecdb52 100644
--- a/config/initializers/fast_gettext.rb
+++ b/config/initializers/fast_gettext.rb
@@ -1,5 +1,6 @@
 FastGettext.add_text_domain 'gitlab', path: File.join(Rails.root, 'locale'), type: :po
 FastGettext.default_text_domain = 'gitlab'
 FastGettext.default_available_locales = Gitlab::I18n.available_locales
+FastGettext.default_locale = :en
 
 I18n.available_locales = Gitlab::I18n.available_locales
diff --git a/config/initializers/postgresql_cte.rb b/config/initializers/postgresql_cte.rb
new file mode 100644
index 00000000000..7f0df8949db
--- /dev/null
+++ b/config/initializers/postgresql_cte.rb
@@ -0,0 +1,132 @@
+# Adds support for WITH statements when using PostgreSQL. The code here is taken
+# from https://github.com/shmay/ctes_in_my_pg which at the time of writing has
+# not been pushed to RubyGems. The license of this repository is as follows:
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2012 Dan McClain
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+module ActiveRecord
+  class Relation
+    class Merger # :nodoc:
+      def normal_values
+        NORMAL_VALUES + [:with]
+      end
+    end
+  end
+end
+
+module ActiveRecord::Querying
+  delegate :with, to: :all
+end
+
+module ActiveRecord
+  class Relation
+    # WithChain objects act as placeholder for queries in which #with does not have any parameter.
+    # In this case, #with must be chained with #recursive to return a new relation.
+    class WithChain
+      def initialize(scope)
+        @scope = scope
+      end
+
+      # Returns a new relation expressing WITH RECURSIVE
+      def recursive(*args)
+        @scope.with_values += args
+        @scope.recursive_value = true
+        @scope
+      end
+    end
+
+    def with_values
+      @values[:with] || []
+    end
+
+    def with_values=(values)
+      raise ImmutableRelation if @loaded
+      @values[:with] = values
+    end
+
+    def recursive_value=(value)
+      raise ImmutableRelation if @loaded
+      @values[:recursive] = value
+    end
+
+    def recursive_value
+      @values[:recursive]
+    end
+
+    def with(opts = :chain, *rest)
+      if opts == :chain
+        WithChain.new(spawn)
+      elsif opts.blank?
+        self
+      else
+        spawn.with!(opts, *rest)
+      end
+    end
+
+    def with!(opts = :chain, *rest) # :nodoc:
+      if opts == :chain
+        WithChain.new(self)
+      else
+        self.with_values += [opts] + rest
+        self
+      end
+    end
+
+    def build_arel
+      arel = super()
+
+      build_with(arel) if @values[:with]
+
+      arel
+    end
+
+    def build_with(arel)
+      with_statements = with_values.flat_map do |with_value|
+        case with_value
+        when String
+          with_value
+        when Hash
+          with_value.map  do |name, expression|
+            case expression
+            when String
+              select = Arel::Nodes::SqlLiteral.new "(#{expression})"
+            when ActiveRecord::Relation, Arel::SelectManager
+              select = Arel::Nodes::SqlLiteral.new "(#{expression.to_sql})"
+            end
+            Arel::Nodes::As.new Arel::Nodes::SqlLiteral.new("\"#{name}\""), select
+          end
+        when Arel::Nodes::As
+          with_value
+        end
+      end
+
+      unless with_statements.empty?
+        if recursive_value
+          arel.with :recursive, with_statements
+        else
+          arel.with with_statements
+        end
+      end
+    end
+  end
+end
diff --git a/config/initializers/server_uptime.rb b/config/initializers/server_uptime.rb
new file mode 100644
index 00000000000..46bf242e143
--- /dev/null
+++ b/config/initializers/server_uptime.rb
@@ -0,0 +1 @@
+Rails.application.config.booted_at = Time.now
diff --git a/config/routes.rb b/config/routes.rb
index 2584981bb04..846054e6917 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,6 +1,5 @@
 require 'sidekiq/web'
 require 'sidekiq/cron/web'
-require 'constraints/group_url_constrainer'
 
 Rails.application.routes.draw do
   concern :access_requestable do
@@ -85,20 +84,6 @@ Rails.application.routes.draw do
 
   root to: "root#index"
 
-  # Since group show page is wildcard routing
-  # we want all other routing to be checked before matching this one
-  constraints(GroupUrlConstrainer.new) do
-    scope(path: '*id',
-          as: :group,
-          constraints: { id: Gitlab::Regex.namespace_route_regex, format: /(html|json|atom)/ },
-          controller: :groups) do
-      get '/', action: :show
-      patch '/', action: :update
-      put '/', action: :update
-      delete '/', action: :destroy
-    end
-  end
-
   draw :test if Rails.env.test?
 
   get '*unmatched_route', to: 'application#route_not_found'
diff --git a/config/routes/admin.rb b/config/routes/admin.rb
index b1b6ef33a47..c20581b1333 100644
--- a/config/routes/admin.rb
+++ b/config/routes/admin.rb
@@ -36,7 +36,7 @@ namespace :admin do
 
   scope(path: 'groups/*id',
         controller: :groups,
-        constraints: { id: Gitlab::Regex.namespace_route_regex, format: /(html|json|atom)/ }) do
+        constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ }) do
 
     scope(as: :group) do
       put :members_update
@@ -54,6 +54,12 @@ namespace :admin do
     member do
       get :test
     end
+
+    resources :hook_logs, only: [:show] do
+      member do
+        get :retry
+      end
+    end
   end
 
   resources :broadcast_messages, only: [:index, :edit, :create, :update, :destroy] do
@@ -70,10 +76,10 @@ namespace :admin do
 
   scope(path: 'projects/*namespace_id',
         as: :namespace,
-        constraints: { namespace_id: Gitlab::Regex.namespace_route_regex }) do
+        constraints: { namespace_id: Gitlab::PathRegex.full_namespace_route_regex }) do
     resources(:projects,
               path: '/',
-              constraints: { id: Gitlab::Regex.project_route_regex },
+              constraints: { id: Gitlab::PathRegex.project_route_regex },
               only: [:show]) do
 
       member do
diff --git a/config/routes/git_http.rb b/config/routes/git_http.rb
index cdf658c3e4a..a53c94326d4 100644
--- a/config/routes/git_http.rb
+++ b/config/routes/git_http.rb
@@ -1,7 +1,7 @@
 scope(path: '*namespace_id/:project_id',
       format: nil,
-      constraints: { namespace_id: Gitlab::Regex.namespace_route_regex }) do
-  scope(constraints: { project_id: Gitlab::Regex.project_git_route_regex }, module: :projects) do
+      constraints: { namespace_id: Gitlab::PathRegex.full_namespace_route_regex }) do
+  scope(constraints: { project_id: Gitlab::PathRegex.project_git_route_regex }, module: :projects) do
     # Git HTTP clients ('git clone' etc.)
     scope(controller: :git_http) do
       get '/info/refs', action: :info_refs
@@ -28,7 +28,7 @@ scope(path: '*namespace_id/:project_id',
   end
 
   # Redirect /group/project/info/refs to /group/project.git/info/refs
-  scope(constraints: { project_id: Gitlab::Regex.project_route_regex }) do
+  scope(constraints: { project_id: Gitlab::PathRegex.project_route_regex }) do
     # Allow /info/refs, /info/refs?service=git-upload-pack, and
     # /info/refs?service=git-receive-pack, but nothing else.
     #
diff --git a/config/routes/group.rb b/config/routes/group.rb
index 7b29e0e807c..11cdff55ed8 100644
--- a/config/routes/group.rb
+++ b/config/routes/group.rb
@@ -1,9 +1,11 @@
+require 'constraints/group_url_constrainer'
+
 resources :groups, only: [:index, :new, :create]
 
 scope(path: 'groups/*group_id',
       module: :groups,
       as: :group,
-      constraints: { group_id: Gitlab::Regex.namespace_route_regex }) do
+      constraints: { group_id: Gitlab::PathRegex.full_namespace_route_regex }) do
   resources :group_members, only: [:index, :create, :update, :destroy], concerns: :access_requestable do
     post :resend_invite, on: :member
     delete :leave, on: :collection
@@ -25,7 +27,7 @@ end
 
 scope(path: 'groups/*id',
       controller: :groups,
-      constraints: { id: Gitlab::Regex.namespace_route_regex, format: /(html|json|atom)/ }) do
+      constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ }) do
   get :edit, as: :edit_group
   get :issues, as: :issues_group
   get :merge_requests, as: :merge_requests_group
@@ -34,3 +36,15 @@ scope(path: 'groups/*id',
   get :subgroups, as: :subgroups_group
   get '/', action: :show, as: :group_canonical
 end
+
+constraints(GroupUrlConstrainer.new) do
+  scope(path: '*id',
+        as: :group,
+        constraints: { id: Gitlab::PathRegex.full_namespace_route_regex, format: /(html|json|atom)/ },
+        controller: :groups) do
+    get '/', action: :show
+    patch '/', action: :update
+    put '/', action: :update
+    delete '/', action: :destroy
+  end
+end
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 89b14ef6527..6922f8db795 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -13,16 +13,16 @@ constraints(ProjectUrlConstrainer.new) do
   # Otherwise, Rails will overwrite the constraint with `/.+?/`,
   # which breaks some of our wildcard routes like `/blob/*id`
   # and `/tree/*id` that depend on the negative lookahead inside
-  # `Gitlab::Regex.namespace_route_regex`, which helps the router
+  # `Gitlab::PathRegex.full_namespace_route_regex`, which helps the router
   # determine whether a certain path segment is part of `*namespace_id`,
   # `:project_id`, or `*id`.
   #
   # See https://github.com/rails/rails/blob/v4.2.8/actionpack/lib/action_dispatch/routing/mapper.rb#L155
   scope(path: '*namespace_id',
         as: :namespace,
-        namespace_id: Gitlab::Regex.namespace_route_regex) do
+        namespace_id: Gitlab::PathRegex.full_namespace_route_regex) do
     scope(path: ':project_id',
-          constraints: { project_id: Gitlab::Regex.project_route_regex },
+          constraints: { project_id: Gitlab::PathRegex.project_route_regex },
           module: :projects,
           as: :project) do
 
@@ -222,6 +222,12 @@ constraints(ProjectUrlConstrainer.new) do
         member do
           get :test
         end
+
+        resources :hook_logs, only: [:show] do
+          member do
+            get :retry
+          end
+        end
       end
 
       resources :container_registry, only: [:index, :destroy],
@@ -335,7 +341,7 @@ constraints(ProjectUrlConstrainer.new) do
       resources :runner_projects, only: [:create, :destroy]
       resources :badges, only: [:index] do
         collection do
-          scope '*ref', constraints: { ref: Gitlab::Regex.git_reference_regex } do
+          scope '*ref', constraints: { ref: Gitlab::PathRegex.git_reference_regex } do
             constraints format: /svg/ do
               get :build
               get :coverage
@@ -358,7 +364,7 @@ constraints(ProjectUrlConstrainer.new) do
 
     resources(:projects,
               path: '/',
-              constraints: { id: Gitlab::Regex.project_route_regex },
+              constraints: { id: Gitlab::PathRegex.project_route_regex },
               only: [:edit, :show, :update, :destroy]) do
       member do
         put :transfer
diff --git a/config/routes/repository.rb b/config/routes/repository.rb
index 5cf37a06e97..11911636fa7 100644
--- a/config/routes/repository.rb
+++ b/config/routes/repository.rb
@@ -2,7 +2,7 @@
 
 resource :repository, only: [:create] do
   member do
-    get 'archive', constraints: { format: Gitlab::Regex.archive_formats_regex }
+    get 'archive', constraints: { format: Gitlab::PathRegex.archive_formats_regex }
   end
 end
 
@@ -24,7 +24,7 @@ scope format: false do
 
     member do
       # tree viewer logs
-      get 'logs_tree', constraints: { id: Gitlab::Regex.git_reference_regex }
+      get 'logs_tree', constraints: { id: Gitlab::PathRegex.git_reference_regex }
       # Directories with leading dots erroneously get rejected if git
       # ref regex used in constraints. Regex verification now done in controller.
       get 'logs_tree/*path', action: :logs_tree, as: :logs_file, format: false, constraints: {
@@ -34,7 +34,7 @@ scope format: false do
     end
   end
 
-  scope constraints: { id: Gitlab::Regex.git_reference_regex } do
+  scope constraints: { id: Gitlab::PathRegex.git_reference_regex } do
     resources :network, only: [:show]
 
     resources :graphs, only: [:show] do
diff --git a/config/routes/user.rb b/config/routes/user.rb
index 0f3bec9cf58..e682dcd6663 100644
--- a/config/routes/user.rb
+++ b/config/routes/user.rb
@@ -11,19 +11,7 @@ devise_scope :user do
   get '/users/almost_there' => 'confirmations#almost_there'
 end
 
-constraints(UserUrlConstrainer.new) do
-  # Get all keys of user
-  get ':username.keys' => 'profiles/keys#get_keys', constraints: { username: Gitlab::Regex.root_namespace_route_regex }
-
-  scope(path: ':username',
-        as: :user,
-        constraints: { username: Gitlab::Regex.root_namespace_route_regex },
-        controller: :users) do
-    get '/', action: :show
-  end
-end
-
-scope(constraints: { username: Gitlab::Regex.root_namespace_route_regex }) do
+scope(constraints: { username: Gitlab::PathRegex.root_namespace_route_regex }) do
   scope(path: 'users/:username',
         as: :user,
         controller: :users) do
@@ -34,7 +22,7 @@ scope(constraints: { username: Gitlab::Regex.root_namespace_route_regex }) do
     get :contributed, as: :contributed_projects
     get :snippets
     get :exists
-    get '/', to: redirect('/%{username}')
+    get '/', to: redirect('/%{username}'), as: nil
   end
 
   # Compatibility with old routing
@@ -46,3 +34,15 @@ scope(constraints: { username: Gitlab::Regex.root_namespace_route_regex }) do
   get '/u/:username/snippets', to: redirect('/users/%{username}/snippets')
   get '/u/:username/contributed', to: redirect('/users/%{username}/contributed')
 end
+
+constraints(UserUrlConstrainer.new) do
+  # Get all keys of user
+  get ':username.keys' => 'profiles/keys#get_keys', constraints: { username: Gitlab::PathRegex.root_namespace_route_regex }
+
+  scope(path: ':username',
+        as: :user,
+        constraints: { username: Gitlab::PathRegex.root_namespace_route_regex },
+        controller: :users) do
+    get '/', action: :show
+  end
+end
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 0ca1f565185..93df2d6f5ff 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -44,9 +44,8 @@
   - [project_cache, 1]
   - [project_destroy, 1]
   - [project_export, 1]
-  - [project_web_hook, 1]
+  - [web_hook, 1]
   - [repository_check, 1]
-  - [system_hook, 1]
   - [git_garbage_collect, 1]
   - [reactive_caching, 1]
   - [cronjob, 1]
diff --git a/config/webpack.config.js b/config/webpack.config.js
index 42024739fe9..c77b1d6334c 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -5,6 +5,7 @@ var path = require('path');
 var webpack = require('webpack');
 var StatsPlugin = require('stats-webpack-plugin');
 var CompressionPlugin = require('compression-webpack-plugin');
+var NameAllModulesPlugin = require('name-all-modules-plugin');
 var BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
 var WatchMissingNodeModulesPlugin = require('react-dev-utils/WatchMissingNodeModulesPlugin');
 
@@ -23,6 +24,7 @@ var config = {
   },
   context: path.join(ROOT_PATH, 'app/assets/javascripts'),
   entry: {
+    balsamiq_viewer:      './blob/balsamiq_viewer.js',
     blob:                 './blob_edit/blob_bundle.js',
     boards:               './boards/boards_bundle.js',
     common:               './commons/index.js',
@@ -47,8 +49,7 @@ var config = {
     notebook_viewer:      './blob/notebook_viewer.js',
     pdf_viewer:           './blob/pdf_viewer.js',
     pipelines:            './pipelines/index.js',
-    balsamiq_viewer:      './blob/balsamiq_viewer.js',
-    pipelines_graph:      './pipelines/graph_bundle.js',
+    pipelines_details:     './pipelines/pipeline_details_bundle.js',
     profile:              './profile/profile_bundle.js',
     protected_branches:   './protected_branches/protected_branches_bundle.js',
     protected_tags:       './protected_tags',
@@ -69,7 +70,8 @@ var config = {
   output: {
     path: path.join(ROOT_PATH, 'public/assets/webpack'),
     publicPath: '/assets/webpack/',
-    filename: IS_PRODUCTION ? '[name].[chunkhash].bundle.js' : '[name].bundle.js'
+    filename: IS_PRODUCTION ? '[name].[chunkhash].bundle.js' : '[name].bundle.js',
+    chunkFilename: IS_PRODUCTION ? '[name].[chunkhash].chunk.js' : '[name].chunk.js',
   },
 
   devtool: 'cheap-module-source-map',
@@ -100,7 +102,7 @@ var config = {
         loader: 'file-loader',
       },
       {
-        test: /locale\/[a-z]+\/(.*)\.js$/,
+        test: /locale\/\w+\/(.*)\.js$/,
         loader: 'exports-loader?locales',
       },
     ]
@@ -126,8 +128,20 @@ var config = {
       jQuery: 'jquery',
     }),
 
-    // use deterministic module ids
+    // assign deterministic module ids
     new webpack.NamedModulesPlugin(),
+    new NameAllModulesPlugin(),
+
+    // assign deterministic chunk ids
+    new webpack.NamedChunksPlugin((chunk) => {
+      if (chunk.name) {
+        return chunk.name;
+      }
+      return chunk.modules.map((m) => {
+        var chunkPath = m.request.split('!').pop();
+        return path.relative(m.context, chunkPath);
+      }).join('_');
+    }),
 
     // create cacheable common library bundle for all vue chunks
     new webpack.optimize.CommonsChunkPlugin({
@@ -146,7 +160,7 @@ var config = {
         'notebook_viewer',
         'pdf_viewer',
         'pipelines',
-        'pipelines_graph',
+        'pipelines_details',
         'schedule_form',
         'schedules_index',
         'sidebar',