Merge remote-tracking branch 'origin/master' into support-gitaly-tls

21 files changed, 132 insertions, 52 deletions

diff --git a/config/application.rb b/config/application.rb
index 1b084e91cfb..f10b8ed5bd2 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -26,6 +26,9 @@ module Gitlab
     # setting disabled
     require_dependency Rails.root.join('lib/mysql_zero_date')
 
+    # This can be removed when we drop support for rails 4
+    require_dependency Rails.root.join('lib/rails4_migration_version')
+
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.
@@ -66,6 +69,12 @@ module Gitlab
     # config.i18n.default_locale = :de
     config.i18n.enforce_available_locales = false
 
+    # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
+    # the I18n.default_locale when a translation can not be found).
+    # We have to explicitly set default locale since 1.1.0 - see:
+    # https://github.com/svenfuchs/i18n/pull/415
+    config.i18n.fallbacks = [:en]
+
     # Translation for AR attrs is not working well for POROs like WikiPage
     config.gettext_i18n_rails.use_for_active_record_attributes = false
 
@@ -94,6 +103,9 @@ module Gitlab
     # - Webhook URLs (:hook)
     # - Sentry DSN (:sentry_dsn)
     # - File content from Web Editor (:content)
+    #
+    # NOTE: It is **IMPORTANT** to also update gitlab-workhorse's filter when adding parameters here to not
+    #       introduce another security vulnerability: https://gitlab.com/gitlab-org/gitlab-workhorse/issues/182
     config.filter_parameters += [/token$/, /password/, /secret/, /key$/]
     config.filter_parameters += %i(
       certificate
@@ -142,6 +154,7 @@ module Gitlab
     config.assets.precompile << "locale/**/app.js"
     config.assets.precompile << "emoji_sprites.css"
     config.assets.precompile << "errors.css"
+    config.assets.precompile << "csslab.css"
 
     # Import gitlab-svgs directly from vendored directory
     config.assets.paths << "#{config.root}/node_modules/@gitlab/svgs/dist"
@@ -158,6 +171,9 @@ module Gitlab
 
     config.action_view.sanitized_allowed_protocols = %w(smb)
 
+    # Nokogiri is significantly faster and uses less memory than REXML
+    ActiveSupport::XmlMini.backend = 'Nokogiri'
+
     # This middleware needs to precede ActiveRecord::QueryCache and other middlewares that
     # connect to the database.
     config.middleware.insert_after Rails::Rack::Logger, ::Gitlab::Middleware::BasicHealthCheck
diff --git a/config/boot.rb b/config/boot.rb
index 1aeacdabbad..725473ac7f6 100644
--- a/config/boot.rb
+++ b/config/boot.rb
@@ -12,5 +12,5 @@ require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
 begin
   require 'bootsnap/setup'
 rescue LoadError
-  # bootsnap is optional dependency, so if we don't have it it's fine
+  # bootsnap is an optional dependency, so if we don't have it, it's fine
 end
diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml
index 488728e26ab..af76bace577 100644
--- a/config/dependency_decisions.yml
+++ b/config/dependency_decisions.yml
@@ -470,8 +470,8 @@
 - - :license
   - pikaday
   - MIT
-  - :who: 
-    :why: 
+  - :who: Filipa Lacerda
+    :why: MIT License
     :versions: []
     :when: 2017-10-17 17:46:12.367554000 Z
 - - :license
@@ -544,13 +544,6 @@
     :why: https://github.com/xtuc/webassemblyjs/blob/master/LICENSE
     :versions: []
     :when: 2018-06-08 05:30:56.764116000 Z
-- - :license
-  - "@gitlab-org/gitlab-ui"
-  - MIT
-  - :who: Clement Ho
-    :why: Our own library
-    :versions: []
-    :when: 2018-07-17 21:02:54.529227000 Z
 - - :approve
   - lz-string
   - :who: Phil Hughes
@@ -599,3 +592,10 @@
       in compiled/distributed product so attribution not needed.
     :versions: []
     :when: 2018-10-02 19:23:54.840151000 Z
+- - :license
+  - echarts
+  - Apache 2.0
+  - :who: Adriel Santiago
+    :why: https://github.com/apache/incubator-echarts/blob/master/LICENSE
+    :versions: []
+    :when: 2018-12-07 20:46:12.421256000 Z
diff --git a/config/environments/production.rb b/config/environments/production.rb
index 71195164e7a..49a4e873093 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -65,10 +65,6 @@ Rails.application.configure do
   # Enable threaded mode
   # config.threadsafe! unless $rails_rake_task
 
-  # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
-  # the I18n.default_locale when a translation can not be found)
-  config.i18n.fallbacks = true
-
   # Send deprecation notices to registered listeners
   config.active_support.deprecation = :notify
 
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index a59600763fb..7fe85f0e0d7 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -114,6 +114,9 @@ production: &base
     # The default is 'shared/cache/archive/' relative to the root of the Rails app.
     # repository_downloads_path: shared/cache/archive/
 
+    ## Impersonation settings
+    impersonation_enabled: true
+
   ## Reply by email
   # Allow users to comment on issues and merge requests by replying to notification emails.
   # For documentation on how to set this up, see http://doc.gitlab.com/ce/administration/reply_by_email.html
@@ -545,15 +548,15 @@ production: &base
       #     app_id: 'YOUR_APP_ID',
       #     app_secret: 'YOUR_APP_SECRET' }
       # - { name: 'jwt',
-      #     app_secret: 'YOUR_APP_SECRET',
       #     args: {
-      #             algorithm: 'HS256',
-      #             uid_claim: 'email',
-      #             required_claims: ["name", "email"],
-      #             info_map: { name: "name", email: "email" },
-      #             auth_url: 'https://example.com/',
-      #             valid_within: null,
-      #           }
+      #       secret: 'YOUR_APP_SECRET',
+      #       algorithm: 'HS256', # Supported algorithms: 'RS256', 'RS384', 'RS512', 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512'
+      #       uid_claim: 'email',
+      #       required_claims: ['name', 'email'],
+      #       info_map: { name: 'name', email: 'email' },
+      #       auth_url: 'https://example.com/',
+      #       valid_within: 3600 # 1 hour
+      #     }
       #   }
       # - { name: 'saml',
       #     label: 'Our SAML Provider',
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index bd02b85c7ce..db35fa96ea2 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -153,6 +153,7 @@ Settings.gitlab['domain_whitelist'] ||= []
 Settings.gitlab['import_sources'] ||= Gitlab::ImportSources.values
 Settings.gitlab['trusted_proxies'] ||= []
 Settings.gitlab['no_todos_messages'] ||= YAML.load_file(Rails.root.join('config', 'no_todos_messages.yml'))
+Settings.gitlab['impersonation_enabled'] ||= true if Settings.gitlab['impersonation_enabled'].nil?
 Settings.gitlab['usage_ping_enabled'] = true if Settings.gitlab['usage_ping_enabled'].nil?
 
 #
@@ -301,10 +302,6 @@ Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_for_usage_ping)
 Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker'
 
-Settings.cron_jobs['remove_old_web_hook_logs_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['remove_old_web_hook_logs_worker']['cron'] ||= '40 0 * * *'
-Settings.cron_jobs['remove_old_web_hook_logs_worker']['job_class'] = 'RemoveOldWebHookLogsWorker'
-
 Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
 Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
diff --git a/config/initializers/action_dispatch_http_mime_negotiation.rb b/config/initializers/action_dispatch_http_mime_negotiation.rb
new file mode 100644
index 00000000000..bdf5b0babfb
--- /dev/null
+++ b/config/initializers/action_dispatch_http_mime_negotiation.rb
@@ -0,0 +1,19 @@
+# Starting with Rails 5, Rails tries to determine the request format based on
+# the extension of the full URL path if no explicit `format` param or `Accept`
+# header is provided, like when simply browsing to a page in your browser.
+#
+# This is undesireable in GitLab, because many of our paths will end in a ref or
+# blob name that can end with any extension, while these pages should still be
+# presented as HTML unless otherwise specified.
+
+# We override `format_from_path_extension` to disable this behavior.
+
+module ActionDispatch
+  module Http
+    module MimeNegotiation
+      def format_from_path_extension
+        nil
+      end
+    end
+  end
+end
diff --git a/config/initializers/attr_encrypted_no_db_connection.rb b/config/initializers/attr_encrypted_no_db_connection.rb
index e007666b852..7ad458929db 100644
--- a/config/initializers/attr_encrypted_no_db_connection.rb
+++ b/config/initializers/attr_encrypted_no_db_connection.rb
@@ -1,7 +1,18 @@
 module AttrEncrypted
   module Adapters
     module ActiveRecord
-      module DBConnectionQuerier
+      module GitlabMonkeyPatches
+        # Prevent attr_encrypted from defining virtual accessors for encryption
+        # data when the code and schema are out of sync. See this issue for more
+        # details: https://github.com/attr-encrypted/attr_encrypted/issues/332
+        def attribute_instance_methods_as_symbols_available?
+          false
+        end
+
+        # Prevent attr_encrypted from checking out a database connection
+        # indefinitely. The result of this method is only used when the former
+        # is true, but it is called unconditionally, so there is still value to
+        # ensuring the connection is released
         def attribute_instance_methods_as_symbols
           # Use with_connection so the connection doesn't stay pinned to the thread.
           connected = ::ActiveRecord::Base.connection_pool.with_connection(&:active?) rescue false
@@ -15,7 +26,16 @@ module AttrEncrypted
           end
         end
       end
-      prepend DBConnectionQuerier
     end
   end
 end
+
+# As of v3.1.0, the attr_encrypted gem defines the AttrEncrypted and
+# AttrEncrypted::Adapters::ActiveRecord modules, and uses "extend" to mix them
+# into the ActiveRecord::Base class. This intervention overrides utility methods
+# defined by attr_encrypted to fix two bugs, as detailed above.
+#
+# The methods are used here: https://github.com/attr-encrypted/attr_encrypted/blob/3.1.0/lib/attr_encrypted.rb#L145-158
+ActiveSupport.on_load(:active_record) do
+  extend AttrEncrypted::Adapters::ActiveRecord::GitlabMonkeyPatches
+end
diff --git a/config/initializers/correlation_id.rb b/config/initializers/correlation_id.rb
new file mode 100644
index 00000000000..2a7c138dc40
--- /dev/null
+++ b/config/initializers/correlation_id.rb
@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Rails.application.config.middleware.use(Gitlab::Middleware::CorrelationId)
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index 179e00cdbd0..67eabb0b4fc 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -103,6 +103,9 @@ Devise.setup do |config|
   # Send a notification email when the user's password is changed
   config.send_password_change_notification = true
 
+  # Send a notification email when the user's email is changed
+  config.send_email_changed_notification = true
+
   # ==> Configuration for :validatable
   # Range for password length. Default is 6..128.
   config.password_length = 8..128
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index f321b4ea763..6be5c00daaa 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -48,6 +48,13 @@ Doorkeeper.configure do
   #
   force_ssl_in_redirect_uri false
 
+  # Specify what redirect URI's you want to block during Application creation.
+  # Any redirect URI is whitelisted by default.
+  #
+  # You can use this option in order to forbid URI's with 'javascript' scheme
+  # for example.
+  forbid_redirect_uri { |uri| %w[data vbscript javascript].include?(uri.scheme.to_s.downcase) }
+
   # Provide support for an owner to be assigned to each registered application (disabled by default)
   # Optional parameter confirmation: true (default false) if you want to enforce ownership of
   # a registered application
diff --git a/config/initializers/kubeclient.rb b/config/initializers/kubeclient.rb
index 2d9f439fdc0..f8fe1156aaa 100644
--- a/config/initializers/kubeclient.rb
+++ b/config/initializers/kubeclient.rb
@@ -1,19 +1,4 @@
 class Kubeclient::Client
-  # We need to monkey patch this method until
-  # https://github.com/abonas/kubeclient/pull/323 is merged
-  def proxy_url(kind, name, port, namespace = '')
-    discover unless @discovered
-    entity_name_plural =
-      if %w[services pods nodes].include?(kind.to_s)
-        kind.to_s
-      else
-        @entities[kind.to_s].resource_name
-      end
-
-    ns_prefix = build_namespace_prefix(namespace)
-    rest_client["#{ns_prefix}#{entity_name_plural}/#{name}:#{port}/proxy"].url
-  end
-
   # Monkey patch to set `max_redirects: 0`, so that kubeclient
   # does not follow redirects and expose internal services.
   # See https://gitlab.com/gitlab-org/gitlab-ce/issues/53158
diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb
index 840404e0ec0..c897bc30e76 100644
--- a/config/initializers/lograge.rb
+++ b/config/initializers/lograge.rb
@@ -29,6 +29,7 @@ unless Sidekiq.server?
       gitaly_calls = Gitlab::GitalyClient.get_request_count
       payload[:gitaly_calls] = gitaly_calls if gitaly_calls > 0
       payload[:response] = event.payload[:response] if event.payload[:response]
+      payload[Gitlab::CorrelationId::LOG_KEY] = Gitlab::CorrelationId.current_id
 
       payload
     end
diff --git a/config/initializers/rack_attack_global.rb b/config/initializers/rack_attack_global.rb
index 45963831c41..86cb930eca9 100644
--- a/config/initializers/rack_attack_global.rb
+++ b/config/initializers/rack_attack_global.rb
@@ -33,22 +33,22 @@ class Rack::Attack
   throttle('throttle_authenticated_api', Gitlab::Throttle.authenticated_api_options) do |req|
     Gitlab::Throttle.settings.throttle_authenticated_api_enabled &&
       req.api_request? &&
-      req.authenticated_user_id
+      req.authenticated_user_id([:api])
   end
 
   throttle('throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req|
     Gitlab::Throttle.settings.throttle_authenticated_web_enabled &&
       req.web_request? &&
-      req.authenticated_user_id
+      req.authenticated_user_id([:api, :rss, :ics])
   end
 
   class Request
     def unauthenticated?
-      !authenticated_user_id
+      !authenticated_user_id([:api, :rss, :ics])
     end
 
-    def authenticated_user_id
-      Gitlab::Auth::RequestAuthenticator.new(self).user&.id
+    def authenticated_user_id(request_formats)
+      Gitlab::Auth::RequestAuthenticator.new(self).user(request_formats)&.id
     end
 
     def api_request?
diff --git a/config/initializers/sentry.rb b/config/initializers/sentry.rb
index 17d09293205..2a6c5148f71 100644
--- a/config/initializers/sentry.rb
+++ b/config/initializers/sentry.rb
@@ -24,4 +24,4 @@ def configure_sentry
   end
 end
 
-configure_sentry if Rails.env.production?
+configure_sentry if Rails.env.production? || Rails.env.development?
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index 565efc858d1..6aba6c7c21d 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -20,6 +20,8 @@ Sidekiq.configure_server do |config|
     chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] && !enable_json_logs
     chain.add Gitlab::SidekiqMiddleware::Shutdown
     chain.add Gitlab::SidekiqMiddleware::RequestStoreMiddleware unless ENV['SIDEKIQ_REQUEST_STORE'] == '0'
+    chain.add Gitlab::SidekiqMiddleware::BatchLoader
+    chain.add Gitlab::SidekiqMiddleware::CorrelationLogger
     chain.add Gitlab::SidekiqStatus::ServerMiddleware
   end
 
@@ -30,6 +32,7 @@ Sidekiq.configure_server do |config|
 
   config.client_middleware do |chain|
     chain.add Gitlab::SidekiqStatus::ClientMiddleware
+    chain.add Gitlab::SidekiqMiddleware::CorrelationInjector
   end
 
   config.on :startup do
@@ -38,7 +41,7 @@ Sidekiq.configure_server do |config|
     ActiveRecord::Base.clear_all_connections!
   end
 
-  if Feature.enabled?(:gitlab_sidekiq_reliable_fetcher)
+  if Feature::FlipperFeature.table_exists? && Feature.enabled?(:gitlab_sidekiq_reliable_fetcher)
     Sidekiq::ReliableFetcher.setup_reliable_fetch!(config)
   end
 
@@ -74,6 +77,7 @@ Sidekiq.configure_client do |config|
   config.redis = queues_config_hash
 
   config.client_middleware do |chain|
+    chain.add Gitlab::SidekiqMiddleware::CorrelationInjector
     chain.add Gitlab::SidekiqStatus::ClientMiddleware
   end
 end
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 3f1ad90dfca..7d0623cb904 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -245,6 +245,10 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         end
       end
 
+      namespace :serverless do
+        resources :functions, only: [:index]
+      end
+
       scope '-' do
         get 'archive/*id', constraints: { format: Gitlab::PathRegex.archive_formats_regex, id: /.+?/ }, to: 'repositories#archive', as: 'archive'
 
@@ -432,6 +436,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         resource :integrations, only: [:show]
         resource :repository, only: [:show], controller: :repository do
           post :create_deploy_token, path: 'deploy_token/create'
+          post :cleanup
         end
       end
 
diff --git a/config/routes/wiki.rb b/config/routes/wiki.rb
index 1a07b1c206b..2ca52e55fca 100644
--- a/config/routes/wiki.rb
+++ b/config/routes/wiki.rb
@@ -6,7 +6,7 @@ scope(controller: :wikis) do
     post '/', to: 'wikis#create'
   end
 
-  scope(path: 'wikis/*id', as: :wiki, format: false, defaults: { format: :html }) do
+  scope(path: 'wikis/*id', as: :wiki, format: false) do
     get :edit
     get :history
     post :preview_markdown
diff --git a/config/settings.rb b/config/settings.rb
index 3f3481bb65d..1b94df785a7 100644
--- a/config/settings.rb
+++ b/config/settings.rb
@@ -95,6 +95,14 @@ class Settings < Settingslogic
       Gitlab::Application.secrets.db_key_base[0..31]
     end
 
+    def attr_encrypted_db_key_base_32
+      Gitlab::Utils.ensure_utf8_size(attr_encrypted_db_key_base, bytes: 32.bytes)
+    end
+
+    def attr_encrypted_db_key_base_12
+      Gitlab::Utils.ensure_utf8_size(attr_encrypted_db_key_base, bytes: 12.bytes)
+    end
+
     # This should be used for :per_attribute_salt_and_iv mode. There is no
     # need to truncate the key because the encryptor will use the salt to
     # generate a hash of the password:
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 53e1c8778b6..5985569bef4 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -81,3 +81,6 @@
   - [delete_diff_files, 1]
   - [detect_repository_languages, 1]
   - [auto_devops, 2]
+  - [object_pool, 1]
+  - [repository_cleanup, 1]
+  - [delete_stored_files, 1]
diff --git a/config/webpack.config.js b/config/webpack.config.js
index 9ecae9790fd..b9044e13f50 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -84,7 +84,7 @@ module.exports = {
   },
 
   resolve: {
-    extensions: ['.js'],
+    extensions: ['.js', '.gql', '.graphql'],
     alias: {
       '~': path.join(ROOT_PATH, 'app/assets/javascripts'),
       emojis: path.join(ROOT_PATH, 'fixtures/emojis'),
@@ -101,6 +101,11 @@ module.exports = {
     strictExportPresence: true,
     rules: [
       {
+        type: 'javascript/auto',
+        test: /\.mjs$/,
+        use: [],
+      },
+      {
         test: /\.js$/,
         exclude: path => /node_modules|vendor[\\/]assets/.test(path) && !/\.vue\.js/.test(path),
         loader: 'babel-loader',
@@ -122,6 +127,11 @@ module.exports = {
         },
       },
       {
+        test: /\.(graphql|gql)$/,
+        exclude: /node_modules/,
+        loader: 'graphql-tag/loader',
+      },
+      {
         test: /\.svg$/,
         loader: 'raw-loader',
       },