Merge remote-tracking branch 'upstream/master' into 28717-additional-metrics-review-branch

21 files changed, 233 insertions, 23 deletions

diff --git a/config/application.rb b/config/application.rb
index b0533759252..8bbecf3ed0f 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -105,6 +105,7 @@ module Gitlab
     config.assets.precompile << "katex.css"
     config.assets.precompile << "katex.js"
     config.assets.precompile << "xterm/xterm.css"
+    config.assets.precompile << "peek.css"
     config.assets.precompile << "lib/ace.js"
     config.assets.precompile << "vendor/assets/fonts/*"
     config.assets.precompile << "test.css"
diff --git a/config/boot.rb b/config/boot.rb
index f2830ae3166..db5ab918021 100644
--- a/config/boot.rb
+++ b/config/boot.rb
@@ -4,3 +4,18 @@ require 'rubygems'
 ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
 
 require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+
+# set default directory for multiproces metrics gathering
+ENV['prometheus_multiproc_dir'] ||= 'tmp/prometheus_multiproc_dir'
+
+# Default Bootsnap configuration from https://github.com/Shopify/bootsnap#usage
+require 'bootsnap'
+Bootsnap.setup(
+  cache_dir:            'tmp/cache',
+  development_mode:     ENV['RAILS_ENV'] == 'development',
+  load_path_cache:      true,
+  autoload_paths_cache: true,
+  disable_trace:        false,
+  compile_cache_iseq:   true,
+  compile_cache_yaml:   true
+)
diff --git a/config/environments/production.rb b/config/environments/production.rb
index 82a19085b1d..c5cbfcf64cf 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -50,7 +50,7 @@ Rails.application.configure do
   # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new)
 
   # Enable serving of images, stylesheets, and JavaScripts from an asset server
-  # config.action_controller.asset_host = "http://assets.example.com"
+  config.action_controller.asset_host = ENV['GITLAB_CDN_HOST'] if ENV['GITLAB_CDN_HOST'].present?
 
   # Precompile additional assets (application.js, application.css, and all non-JS/CSS are already added)
   # config.assets.precompile += %w( search.js )
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index d2aeb66ebf6..0b33783869b 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -337,6 +337,10 @@ production: &base
     # showing GitLab's sign-in page (default: show the GitLab sign-in page)
     # auto_sign_in_with_provider: saml
 
+    # Sync user's email address from the specified Omniauth provider every time the user logs
+    # in (default: nil). And consequently make this field read-only.
+    # sync_email_from_provider: cas3
+
     # CAUTION!
     # This allows users to login without having a user account first. Define the allowed providers
     # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none.
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 45ea2040d23..8ddf8e4d2e4 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -156,6 +156,7 @@ Settings.omniauth['external_providers'] = [] if Settings.omniauth['external_prov
 Settings.omniauth['block_auto_created_users'] = true if Settings.omniauth['block_auto_created_users'].nil?
 Settings.omniauth['auto_link_ldap_user'] = false if Settings.omniauth['auto_link_ldap_user'].nil?
 Settings.omniauth['auto_link_saml_user'] = false if Settings.omniauth['auto_link_saml_user'].nil?
+Settings.omniauth['sync_email_from_provider'] ||= nil
 
 Settings.omniauth['providers'] ||= []
 Settings.omniauth['cas3'] ||= Settingslogic.new({})
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index 5e0eefdb154..508b886d6a0 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -113,6 +113,9 @@ def instrument_classes(instrumentation)
 
   # This is a Rails scope so we have to instrument it manually.
   instrumentation.instrument_method(Project, :visible_to_user)
+
+  # Needed for https://gitlab.com/gitlab-org/gitlab-ce/issues/30224#note_32306159
+  instrumentation.instrument_instance_method(MergeRequestDiff, :load_commits)
 end
 # rubocop:enable Metrics/AbcSize
 
diff --git a/config/initializers/active_record_data_types.rb b/config/initializers/active_record_data_types.rb
new file mode 100644
index 00000000000..beb97c6fce0
--- /dev/null
+++ b/config/initializers/active_record_data_types.rb
@@ -0,0 +1,24 @@
+# ActiveRecord custom data type for storing datetimes with timezone information.
+# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11229
+
+if Gitlab::Database.postgresql?
+  require 'active_record/connection_adapters/postgresql_adapter'
+
+  module ActiveRecord
+    module ConnectionAdapters
+      class PostgreSQLAdapter
+        NATIVE_DATABASE_TYPES.merge!(datetime_with_timezone: { name: 'timestamptz' })
+      end
+    end
+  end
+elsif Gitlab::Database.mysql?
+  require 'active_record/connection_adapters/mysql2_adapter'
+
+  module ActiveRecord
+    module ConnectionAdapters
+      class AbstractMysqlAdapter
+        NATIVE_DATABASE_TYPES.merge!(datetime_with_timezone: { name: 'timestamp' })
+      end
+    end
+  end
+end
diff --git a/config/initializers/active_record_table_definition.rb b/config/initializers/active_record_table_definition.rb
new file mode 100644
index 00000000000..4f59e35f4da
--- /dev/null
+++ b/config/initializers/active_record_table_definition.rb
@@ -0,0 +1,24 @@
+# ActiveRecord custom method definitions with timezone information.
+# See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/11229
+
+require 'active_record/connection_adapters/abstract/schema_definitions'
+
+# Appends columns `created_at` and `updated_at` to a table.
+#
+# It is used in table creation like:
+# create_table 'users' do |t|
+#   t.timestamps_with_timezone
+# end
+module ActiveRecord
+  module ConnectionAdapters
+    class TableDefinition
+      def timestamps_with_timezone(**options)
+        options[:null] = false if options[:null].nil?
+
+        [:created_at, :updated_at].each do |column_name|
+          column(column_name, :datetime_with_timezone, options)
+        end
+      end
+    end
+  end
+end
diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb
new file mode 100644
index 00000000000..65432caac2a
--- /dev/null
+++ b/config/initializers/peek.rb
@@ -0,0 +1,32 @@
+Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.params) }
+
+Peek.into Peek::Views::Host
+Peek.into Peek::Views::PerformanceBar
+if Gitlab::Database.mysql?
+  require 'peek-mysql2'
+  PEEK_DB_CLIENT = ::Mysql2::Client
+  PEEK_DB_VIEW = Peek::Views::Mysql2
+else
+  require 'peek-pg'
+  PEEK_DB_CLIENT = ::PG::Connection
+  PEEK_DB_VIEW = Peek::Views::PG
+end
+Peek.into PEEK_DB_VIEW
+Peek.into Peek::Views::Redis
+Peek.into Peek::Views::Sidekiq
+Peek.into Peek::Views::Rblineprof
+Peek.into Peek::Views::GC
+
+# rubocop:disable Style/ClassAndModuleCamelCase
+class PEEK_DB_CLIENT
+  class << self
+    attr_accessor :query_details
+  end
+  self.query_details = Concurrent::Array.new
+end
+
+PEEK_DB_VIEW.prepend ::Gitlab::PerformanceBar::PeekQueryTracker
+
+class Peek::Views::PerformanceBar::ProcessUtilization
+  prepend ::Gitlab::PerformanceBar::PeekPerformanceBarWithRackBody
+end
diff --git a/config/initializers/rugged_use_gitlab_git_attributes.rb b/config/initializers/rugged_use_gitlab_git_attributes.rb
new file mode 100644
index 00000000000..7d652799786
--- /dev/null
+++ b/config/initializers/rugged_use_gitlab_git_attributes.rb
@@ -0,0 +1,25 @@
+# We don't want to ever call Rugged::Repository#fetch_attributes, because it has
+# a lot of I/O overhead:
+# <https://gitlab.com/gitlab-org/gitlab_git/commit/340e111e040ae847b614d35b4d3173ec48329015>
+#
+# While we don't do this from within the GitLab source itself, the Linguist gem
+# has a dependency on Rugged and uses the gitattributes file when calculating
+# repository-wide language statistics:
+# <https://github.com/github/linguist/blob/v4.7.0/lib/linguist/lazy_blob.rb#L33-L36>
+#
+# The options passed by Linguist are those assumed by Gitlab::Git::Attributes
+# anyway, and there is no great efficiency gain from just fetching the listed
+# attributes with our implementation, so we ignore the additional arguments.
+#
+module Rugged
+  class Repository
+    module UseGitlabGitAttributes
+      def fetch_attributes(name, *)
+        @attributes ||= Gitlab::Git::Attributes.new(path)
+        @attributes.attributes(name)
+      end
+    end
+
+    prepend UseGitlabGitAttributes
+  end
+end
diff --git a/config/karma.config.js b/config/karma.config.js
index 40c58e7771d..978850e5d70 100644
--- a/config/karma.config.js
+++ b/config/karma.config.js
@@ -21,7 +21,18 @@ module.exports = function(config) {
 
   var karmaConfig = {
     basePath: ROOT_PATH,
-    browsers: ['PhantomJS'],
+    browsers: ['ChromeHeadlessCustom'],
+    customLaunchers: {
+      ChromeHeadlessCustom: {
+        base: 'ChromeHeadless',
+        displayName: 'Chrome',
+        flags: [
+          // chrome cannot run in sandboxed mode inside a docker container unless it is run with
+          // escalated kernel privileges (e.g. docker run --cap-add=CAP_SYS_ADMIN)
+          '--no-sandbox',
+        ],
+      }
+    },
     frameworks: ['jasmine'],
     files: [
       { pattern: 'spec/javascripts/test_bundle.js', watched: false },
@@ -45,5 +56,14 @@ module.exports = function(config) {
     };
   }
 
+  if (process.env.DEBUG) {
+    karmaConfig.logLevel = config.LOG_DEBUG;
+    process.env.CHROME_LOG_FILE = process.env.CHROME_LOG_FILE || 'chrome_debug.log';
+  }
+
+  if (process.env.CHROME_LOG_FILE) {
+    karmaConfig.customLaunchers.ChromeHeadlessCustom.flags.push('--enable-logging', '--v=1');
+  }
+
   config.set(karmaConfig);
 };
diff --git a/config/locales/de.yml b/config/locales/de.yml
index 533663a2704..38c3711c6c7 100644
--- a/config/locales/de.yml
+++ b/config/locales/de.yml
@@ -62,6 +62,43 @@ de:
     - :month
     - :year
   datetime:
+    # used in a custom scope that has been created to fix https://gitlab.com/gitlab-org/gitlab-ce/issues/32747
+    time_ago_in_words:
+      half_a_minute: vor einer halben Minute
+      less_than_x_seconds:
+        one: vor weniger als einer Sekunde
+        other: "vor weniger als %{count} Sekunden"
+      x_seconds:
+        one: vor einer Sekunde
+        other: "vor %{count} Sekunden"
+      less_than_x_minutes:
+        one: vor weniger als einer Minute
+        other: vor weniger als %{count} Minuten
+      x_minutes:
+        one: vor einer Minute
+        other: "vor %{count} Minuten"
+      about_x_hours:
+        one: vor etwa einer Stunde
+        other: "vor etwa %{count} Stunden"
+      x_days:
+        one: vor einem Tag
+        other: "vor %{count} Tagen"
+      about_x_months:
+        one: vor etwa einem Monat
+        other: "vor etwa %{count} Monaten"
+      x_months:
+        one: vor einem Monat
+        other: "vor %{count} Monaten"
+      about_x_years:
+        one: vor etwa einem Jahr
+        other: "vor etwa %{count} Jahren"
+      over_x_years:
+        one: vor mehr als einem Jahr
+        other: "vor mehr als %{count} Jahren"
+      almost_x_years:
+        one: vor fast einem Jahr
+        other: "vor fast %{count} Jahren"
+    # Used in distance_of_time_in_words(), distance_of_time_in_words_to_now(), time_ago_in_words()
     distance_in_words:
       about_x_hours:
         one: etwa eine Stunde
diff --git a/config/locales/es.yml b/config/locales/es.yml
index 0f9dc39535d..d71c6eb5047 100644
--- a/config/locales/es.yml
+++ b/config/locales/es.yml
@@ -61,6 +61,7 @@ es:
     - :month
     - :year
   datetime:
+    # used in a custom scope that has been created to fix https://gitlab.com/gitlab-org/gitlab-ce/issues/32747
     time_ago_in_words:
       half_a_minute: "hace medio minuto"
       less_than_x_seconds:
@@ -96,6 +97,7 @@ es:
       almost_x_years:
         one:   "hace casi 1 año"
         other: "hace casi %{count} años"
+    # Used in distance_of_time_in_words(), distance_of_time_in_words_to_now(), time_ago_in_words()
     distance_in_words:
       about_x_hours:
         one: alrededor de 1 hora
diff --git a/config/routes.rb b/config/routes.rb
index 846054e6917..4fd6cb5d439 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -38,10 +38,11 @@ Rails.application.routes.draw do
   # Health check
   get 'health_check(/:checks)' => 'health_check#index', as: :health_check
 
-  scope path: '-', controller: 'health' do
-    get :liveness
-    get :readiness
-    get :metrics
+  scope path: '-' do
+    get 'liveness' => 'health#liveness'
+    get 'readiness' => 'health#readiness'
+    resources :metrics, only: [:index]
+    mount Peek::Railtie => '/peek'
   end
 
   # Koding route
diff --git a/config/routes/admin.rb b/config/routes/admin.rb
index c7b639b7b3c..5427bab93ce 100644
--- a/config/routes/admin.rb
+++ b/config/routes/admin.rb
@@ -48,7 +48,7 @@ namespace :admin do
     end
   end
 
-  resources :deploy_keys, only: [:index, :new, :create, :destroy]
+  resources :deploy_keys, only: [:index, :new, :create, :edit, :update, :destroy]
 
   resources :hooks, only: [:index, :create, :edit, :update, :destroy] do
     member do
diff --git a/config/routes/dashboard.rb b/config/routes/dashboard.rb
index 8e380a0b0ac..d2437285cdf 100644
--- a/config/routes/dashboard.rb
+++ b/config/routes/dashboard.rb
@@ -4,7 +4,13 @@ resource :dashboard, controller: 'dashboard', only: [] do
   get :activity
 
   scope module: :dashboard do
-    resources :milestones, only: [:index, :show]
+    resources :milestones, only: [:index, :show] do
+      member do
+        get :merge_requests
+        get :participants
+        get :labels
+      end
+    end
     resources :labels, only: [:index]
 
     resources :groups, only: [:index]
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 10ed32a3a33..19e18c733b1 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -77,7 +77,7 @@ constraints(ProjectUrlConstrainer.new) do
         get :active_metrics
       end
 
-      resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create] do
+      resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create, :edit, :update] do
         member do
           put :enable
           put :disable
diff --git a/config/routes/snippets.rb b/config/routes/snippets.rb
index dae83734fe6..0a4ebac3ca3 100644
--- a/config/routes/snippets.rb
+++ b/config/routes/snippets.rb
@@ -2,6 +2,9 @@ resources :snippets, concerns: :awardable do
   member do
     get :raw
     post :mark_as_spam
+  end
+
+  collection do
     post :preview_markdown
   end
 
diff --git a/config/routes/uploads.rb b/config/routes/uploads.rb
index b315186b178..a49e244af1a 100644
--- a/config/routes/uploads.rb
+++ b/config/routes/uploads.rb
@@ -1,6 +1,6 @@
 scope path: :uploads do
   # Note attachments and User/Group/Project avatars
-  get ":model/:mounted_as/:id/:filename",
+  get "system/:model/:mounted_as/:id/:filename",
       to:           "uploads#show",
       constraints:  { model: /note|user|group|project/, mounted_as: /avatar|attachment/, filename: /[^\/]+/ }
 
@@ -9,8 +9,13 @@ scope path: :uploads do
     to: 'uploads#show',
     constraints: { model: /personal_snippet/, id: /\d+/, filename: /[^\/]+/ }
 
+  # show temporary uploads
+  get 'temp/:secret/:filename',
+    to: 'uploads#show',
+    constraints: { filename: /[^\/]+/ }
+
   # Appearance
-  get ":model/:mounted_as/:id/:filename",
+  get "system/:model/:mounted_as/:id/:filename",
       to:           "uploads#show",
       constraints:  { model: /appearance/, mounted_as: /logo|header_logo/, filename: /.+/ }
 
@@ -20,7 +25,7 @@ scope path: :uploads do
     constraints:  { namespace_id: /[a-zA-Z.0-9_\-]+/, project_id: /[a-zA-Z.0-9_\-]+/, filename: /[^\/]+/ }
 
   # create uploads for models, snippets (notes) available for now
-  post ':model/:id/',
+  post ':model',
     to: 'uploads#create',
     constraints: { model: /personal_snippet/, id: /\d+/ },
     as: 'upload'
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 93df2d6f5ff..1d9e69a2408 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -54,3 +54,4 @@
   - [system_hook_push, 1]
   - [update_user_activity, 1]
   - [propagate_service_template, 1]
+  - [background_migration, 1]
diff --git a/config/webpack.config.js b/config/webpack.config.js
index a49f02d7ae8..04a25edcdb2 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -18,6 +18,15 @@ var DEV_SERVER_LIVERELOAD = process.env.DEV_SERVER_LIVERELOAD !== 'false';
 var WEBPACK_REPORT = process.env.WEBPACK_REPORT;
 var NO_COMPRESSION = process.env.NO_COMPRESSION;
 
+// optional dependency `node-zopfli` is unavailable on CentOS 6
+var ZOPFLI_AVAILABLE;
+try {
+  require.resolve('node-zopfli');
+  ZOPFLI_AVAILABLE = true;
+} catch(err) {
+  ZOPFLI_AVAILABLE = false;
+}
+
 var config = {
   // because sqljs requires fs.
   node: {
@@ -40,9 +49,11 @@ var config = {
     filtered_search:      './filtered_search/filtered_search_bundle.js',
     graphs:               './graphs/graphs_bundle.js',
     group:                './group.js',
+    groups:               './groups/index.js',
     groups_list:          './groups_list.js',
     issue_show:           './issue_show/index.js',
     integrations:         './integrations',
+    job_details:          './jobs/job_details_bundle.js',
     locale:               './locale/index.js',
     main:                 './main.js',
     merge_conflicts:      './merge_conflicts/merge_conflicts_bundle.js',
@@ -50,7 +61,7 @@ var config = {
     network:              './network/network_bundle.js',
     notebook_viewer:      './blob/notebook_viewer.js',
     pdf_viewer:           './blob/pdf_viewer.js',
-    pipelines:            './pipelines/index.js',
+    pipelines:            './pipelines/pipelines_bundle.js',
     pipelines_details:     './pipelines/pipeline_details_bundle.js',
     profile:              './profile/profile_bundle.js',
     prometheus_metrics:   './prometheus_metrics',
@@ -68,6 +79,7 @@ var config = {
     raven:                './raven/index.js',
     vue_merge_request_widget: './vue_merge_request_widget/index.js',
     test:                 './test.js',
+    peek:                 './peek.js',
   },
 
   output: {
@@ -156,7 +168,9 @@ var config = {
         'environments',
         'environments_folder',
         'filtered_search',
+        'groups',
         'issue_show',
+        'job_details',
         'merge_conflicts',
         'notebook_viewer',
         'pdf_viewer',
@@ -184,15 +198,7 @@ var config = {
 
     // create cacheable common library bundles
     new webpack.optimize.CommonsChunkPlugin({
-      names: ['main', 'common', 'runtime'],
-    }),
-
-    // locale common library
-    new webpack.optimize.CommonsChunkPlugin({
-      name: 'locale',
-      chunks: [
-        'cycle_analytics',
-      ],
+      names: ['main', 'locale', 'common', 'runtime'],
     }),
   ],
 
@@ -231,7 +237,7 @@ if (IS_PRODUCTION) {
     config.plugins.push(
       new CompressionPlugin({
         asset: '[path].gz[query]',
-        algorithm: 'zopfli',
+        algorithm: ZOPFLI_AVAILABLE ? 'zopfli' : 'gzip',
       })
     );
   }