Merge remote-tracking branch 'origin/master' into support-gitaly-tls

17 files changed, 112 insertions, 74 deletions

diff --git a/config/application.rb b/config/application.rb
index 9074cf02c46..1b084e91cfb 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -8,7 +8,7 @@ module Gitlab
   # This method is used for smooth upgrading from the current Rails 4.x to Rails 5.0.
   # https://gitlab.com/gitlab-org/gitlab-ce/issues/14286
   def self.rails5?
-    ENV["RAILS5"].in?(%w[1 true])
+    !%w[0 false].include?(ENV["RAILS5"])
   end
 
   class Application < Rails::Application
@@ -144,7 +144,7 @@ module Gitlab
     config.assets.precompile << "errors.css"
 
     # Import gitlab-svgs directly from vendored directory
-    config.assets.paths << "#{config.root}/node_modules/@gitlab-org/gitlab-svgs/dist"
+    config.assets.paths << "#{config.root}/node_modules/@gitlab/svgs/dist"
     config.assets.precompile << "icons.svg"
     config.assets.precompile << "icons.json"
     config.assets.precompile << "illustrations/*.svg"
diff --git a/config/boot.rb b/config/boot.rb
index 655c54ddb84..1aeacdabbad 100644
--- a/config/boot.rb
+++ b/config/boot.rb
@@ -1,10 +1,10 @@
 def rails5?
-  %w[1 true].include?(ENV["RAILS5"])
+  !%w[0 false].include?(ENV["RAILS5"])
 end
 
 require 'rubygems' unless rails5?
 
-gemfile = rails5? ? "Gemfile.rails5" : "Gemfile"
+gemfile = rails5? ? "Gemfile" : "Gemfile.rails4"
 ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../#{gemfile}", __dir__)
 
 # Set up gems listed in the Gemfile.
diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml
index 62760ffee3a..488728e26ab 100644
--- a/config/dependency_decisions.yml
+++ b/config/dependency_decisions.yml
@@ -461,7 +461,7 @@
     :versions: []
     :when: 2017-09-13 17:31:16.425819400 Z
 - - :license
-  - "@gitlab-org/gitlab-svgs"
+  - "@gitlab/svgs"
   - MIT
   - :who: Tim Zallmann
     :why: Our own library - GitLab License https://gitlab.com/gitlab-org/gitlab-svgs
diff --git a/config/environment.rb b/config/environment.rb
index 5d35937f7c6..3a52656a2c1 100644
--- a/config/environment.rb
+++ b/config/environment.rb
@@ -1,10 +1,10 @@
 # Load the rails application
 
 # Remove this condition when upgraded to rails 5.0.
-if %w[1 true].include?(ENV["RAILS5"])
-  require_relative 'application'
-else
+if %w[0 false].include?(ENV["RAILS5"])
   require File.expand_path('application', __dir__)
+else
+  require_relative 'application'
 end
 
 # Initialize the rails application
diff --git a/config/environments/development.rb b/config/environments/development.rb
index 23790b84e3c..494ddd72556 100644
--- a/config/environments/development.rb
+++ b/config/environments/development.rb
@@ -45,4 +45,6 @@ Rails.application.configure do
 
   # Do not log asset requests
   config.assets.quiet = true
+
+  config.allow_concurrency = defined?(::Puma)
 end
diff --git a/config/environments/production.rb b/config/environments/production.rb
index 9941987929c..71195164e7a 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -83,5 +83,5 @@ Rails.application.configure do
 
   config.eager_load = true
 
-  config.allow_concurrency = false
+  config.allow_concurrency = defined?(::Puma)
 end
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 5390ea78e62..a59600763fb 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -207,6 +207,10 @@ production: &base
         # endpoint: 'http://127.0.0.1:9000' # default: nil
         # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object'
 
+  ## Packages (maven repository so far)
+  packages:
+    enabled: false
+
   ## GitLab Pages
   pages:
     enabled: false
@@ -587,7 +591,7 @@ production: &base
   gitaly:
     # Path to the directory containing Gitaly client executables.
     client_path: /home/git/gitaly/bin
-    # Default Gitaly authentication token. Can be overriden per storage. Can
+    # Default Gitaly authentication token. Can be overridden per storage. Can
     # be left blank when Gitaly is running locally on a Unix socket, which
     # is the normal way to deploy Gitaly.
     token:
diff --git a/config/initializers/8_metrics.rb b/config/initializers/8_metrics.rb
index c8d261d415e..468f80939d7 100644
--- a/config/initializers/8_metrics.rb
+++ b/config/initializers/8_metrics.rb
@@ -98,7 +98,11 @@ end
 # check: https://github.com/rspec/rspec-mocks#settings-mocks-or-stubs-on-any-instance-of-a-class
 #
 # Related issue: https://gitlab.com/gitlab-org/gitlab-ce/issues/33587
-if Gitlab::Metrics.enabled? && !Rails.env.test?
+#
+# In development mode, we turn off eager loading when we're running
+# `rails generate migration` because eager loading short-circuits the
+# loading of our custom migration templates.
+if Gitlab::Metrics.enabled? && !Rails.env.test? && !(Rails.env.development? && defined?(Rails::Generators))
   require 'pathname'
   require 'influxdb'
   require 'connection_pool'
diff --git a/config/initializers/asset_sync.rb b/config/initializers/asset_sync.rb
deleted file mode 100644
index 7f3934853fa..00000000000
--- a/config/initializers/asset_sync.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-AssetSync.configure do |config|
-  # Disable the asset_sync gem by default. If it is enabled, but not configured,
-  # asset_sync will cause the build to fail.
-  config.enabled =  if ENV.has_key?('ASSET_SYNC_ENABLED')
-                      ENV['ASSET_SYNC_ENABLED'] == 'true'
-                    else
-                      false
-                    end
-
-  # Pulled from https://github.com/AssetSync/asset_sync/blob/v2.2.0/lib/asset_sync/engine.rb#L15-L40
-  # This allows us to disable asset_sync by default and configure through environment variables
-  # Updates to asset_sync gem should be checked
-  config.fog_provider = ENV['FOG_PROVIDER'] if ENV.has_key?('FOG_PROVIDER')
-  config.fog_directory = ENV['FOG_DIRECTORY'] if ENV.has_key?('FOG_DIRECTORY')
-  config.fog_region = ENV['FOG_REGION'] if ENV.has_key?('FOG_REGION')
-
-  config.aws_access_key_id = ENV['ASSETS_AWS_ACCESS_KEY_ID'] if ENV.has_key?('ASSETS_AWS_ACCESS_KEY_ID')
-  config.aws_secret_access_key = ENV['ASSETS_AWS_SECRET_ACCESS_KEY'] if ENV.has_key?('ASSETS_AWS_SECRET_ACCESS_KEY')
-  config.aws_reduced_redundancy = ENV['AWS_REDUCED_REDUNDANCY'] == true  if ENV.has_key?('AWS_REDUCED_REDUNDANCY')
-
-  config.rackspace_username = ENV['RACKSPACE_USERNAME'] if ENV.has_key?('RACKSPACE_USERNAME')
-  config.rackspace_api_key = ENV['RACKSPACE_API_KEY'] if ENV.has_key?('RACKSPACE_API_KEY')
-
-  config.google_storage_access_key_id = ENV['GOOGLE_STORAGE_ACCESS_KEY_ID'] if ENV.has_key?('GOOGLE_STORAGE_ACCESS_KEY_ID')
-  config.google_storage_secret_access_key = ENV['GOOGLE_STORAGE_SECRET_ACCESS_KEY'] if ENV.has_key?('GOOGLE_STORAGE_SECRET_ACCESS_KEY')
-
-  config.existing_remote_files = ENV['ASSET_SYNC_EXISTING_REMOTE_FILES'] || "keep"
-
-  config.gzip_compression = (ENV['ASSET_SYNC_GZIP_COMPRESSION'] == 'true') if ENV.has_key?('ASSET_SYNC_GZIP_COMPRESSION')
-  config.manifest = (ENV['ASSET_SYNC_MANIFEST'] == 'true') if ENV.has_key?('ASSET_SYNC_MANIFEST')
-end
diff --git a/config/initializers/fill_shards.rb b/config/initializers/fill_shards.rb
new file mode 100644
index 00000000000..18e067c8854
--- /dev/null
+++ b/config/initializers/fill_shards.rb
@@ -0,0 +1,3 @@
+if Shard.connected? && !Gitlab::Database.read_only?
+  Shard.populate!
+end
diff --git a/config/initializers/kubeclient.rb b/config/initializers/kubeclient.rb
index 7f115268b37..2d9f439fdc0 100644
--- a/config/initializers/kubeclient.rb
+++ b/config/initializers/kubeclient.rb
@@ -13,4 +13,25 @@ class Kubeclient::Client
     ns_prefix = build_namespace_prefix(namespace)
     rest_client["#{ns_prefix}#{entity_name_plural}/#{name}:#{port}/proxy"].url
   end
+
+  # Monkey patch to set `max_redirects: 0`, so that kubeclient
+  # does not follow redirects and expose internal services.
+  # See https://gitlab.com/gitlab-org/gitlab-ce/issues/53158
+  def create_rest_client(path = nil)
+    path ||= @api_endpoint.path
+    options = {
+      ssl_ca_file: @ssl_options[:ca_file],
+      ssl_cert_store: @ssl_options[:cert_store],
+      verify_ssl: @ssl_options[:verify_ssl],
+      ssl_client_cert: @ssl_options[:client_cert],
+      ssl_client_key: @ssl_options[:client_key],
+      proxy: @http_proxy_uri,
+      user: @auth_options[:username],
+      password: @auth_options[:password],
+      open_timeout: @timeouts[:open],
+      read_timeout: @timeouts[:read],
+      max_redirects: 0
+    }
+    RestClient::Resource.new(@api_endpoint.merge(path).to_s, options)
+  end
 end
diff --git a/config/initializers/mysql_set_length_for_binary_indexes.rb b/config/initializers/mysql_set_length_for_binary_indexes.rb
index 81ed2fb83de..0445d8fcae2 100644
--- a/config/initializers/mysql_set_length_for_binary_indexes.rb
+++ b/config/initializers/mysql_set_length_for_binary_indexes.rb
@@ -24,28 +24,46 @@ if defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter)
   ActiveRecord::ConnectionAdapters::Mysql2Adapter.send(:prepend, MysqlSetLengthForBinaryIndex)
 end
 
-if Gitlab.rails5?
-  module MysqlSetLengthForBinaryIndexAndIgnorePostgresOptionsForSchema
-    # This method is used in Rails 5 schema loading as t.index
-    def index(column_names, options = {})
-      options[:length] ||= {}
-      Array(column_names).each do |column_name|
-        column = columns.find { |c| c.name == column_name }
-
-        if column&.type == :binary
-          options[:length][column_name] = 20
-        end
-      end
+module MysqlSetLengthForBinaryIndexAndIgnorePostgresOptionsForSchema
+  # This method is used in Rails 5 schema loading as t.index
+  def index(column_names, options = {})
+    # Ignore indexes that use opclasses,
+    # also see config/initializers/mysql_ignore_postgresql_options.rb
+    if options[:opclasses]
+      warn "WARNING: index on columns #{column_names} uses unsupported option, skipping."
+      return
+    end
+
+    # when running rails 4 with rails 5 schema, rails 4 doesn't support multiple
+    # indexes on the same set of columns. Mysql doesn't support partial indexes, so if
+    # an index already exists and we add another index, skip it if it's partial:
+    # see https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/21492#note_102821326
+    if !Gitlab.rails5? && indexes[column_names] && options[:where]
+      warn "WARNING: index on columns #{column_names} already exists and partial index is not supported, skipping."
+      return
+    end
+
+    options[:length] ||= {}
+    Array(column_names).each do |column_name|
+      column = columns.find { |c| c.name == column_name }
 
-      # Ignore indexes that use opclasses,
-      # also see config/initializers/mysql_ignore_postgresql_options.rb
-      unless options[:opclasses]
-        super(column_names, options)
+      if column&.type == :binary
+        options[:length][column_name] = 20
       end
     end
+
+    super(column_names, options)
   end
+end
 
+def mysql_adapter?
+  defined?(ActiveRecord::ConnectionAdapters::Mysql2Adapter) && ActiveRecord::Base.connection.is_a?(ActiveRecord::ConnectionAdapters::Mysql2Adapter)
+end
+
+if Gitlab.rails5?
   if defined?(ActiveRecord::ConnectionAdapters::MySQL::TableDefinition)
     ActiveRecord::ConnectionAdapters::MySQL::TableDefinition.send(:prepend, MysqlSetLengthForBinaryIndexAndIgnorePostgresOptionsForSchema)
   end
+elsif mysql_adapter? && defined?(ActiveRecord::ConnectionAdapters::TableDefinition)
+  ActiveRecord::ConnectionAdapters::TableDefinition.send(:prepend, MysqlSetLengthForBinaryIndexAndIgnorePostgresOptionsForSchema)
 end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 795e5d4e6bc..0a43a1d9a6b 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -8,6 +8,8 @@ en:
       issue_link:
         source: Source issue
         target: Target issue
+      group:
+        path: Group URL
   errors:
     messages:
       label_already_exists_at_group_level: "already exists at group level for %{group}. Please choose another one."
diff --git a/config/routes.rb b/config/routes.rb
index 37c7f98ec98..484e05114be 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -82,6 +82,30 @@ Rails.application.routes.draw do
 
     draw :operations
     draw :instance_statistics
+
+    if ENV['GITLAB_ENABLE_CHAOS_ENDPOINTS']
+      get '/chaos/leakmem' => 'chaos#leakmem'
+      get '/chaos/cpuspin' => 'chaos#cpuspin'
+      get '/chaos/sleep' => 'chaos#sleep'
+      get '/chaos/kill' => 'chaos#kill'
+    end
+  end
+
+  concern :clusterable do
+    resources :clusters, only: [:index, :new, :show, :update, :destroy] do
+      collection do
+        post :create_user
+        post :create_gcp
+      end
+
+      member do
+        scope :applications do
+          post '/:application', to: 'clusters/applications#create', as: :install_applications
+        end
+
+        get :cluster_status, format: :json
+      end
+    end
   end
 
   draw :api
diff --git a/config/routes/group.rb b/config/routes/group.rb
index 2328b50b760..a0aeebe4b91 100644
--- a/config/routes/group.rb
+++ b/config/routes/group.rb
@@ -53,6 +53,8 @@ constraints(::Constraints::GroupUrlConstrainer.new) do
 
     resource :avatar, only: [:destroy]
 
+    concerns :clusterable
+
     resources :group_members, only: [:index, :create, :update, :destroy], concerns: :access_requestable do
       post :resend_invite, on: :member
       delete :leave, on: :collection
diff --git a/config/routes/project.rb b/config/routes/project.rb
index 73c46f72168..3f1ad90dfca 100644
--- a/config/routes/project.rb
+++ b/config/routes/project.rb
@@ -149,7 +149,9 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
       scope path: 'merge_requests', controller: 'merge_requests/creations' do
         post '', action: :create, as: nil
 
-        scope path: 'new/(:merge_request_source_branch)', as: :new_merge_request do
+        scope path: 'new', as: :new_merge_request do
+          get '', action: :new
+
           scope constraints: { format: nil }, action: :new do
             get :diffs, defaults: { tab: 'diffs' }
             get :pipelines, defaults: { tab: 'pipelines' }
@@ -163,7 +165,6 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
           get :diff_for_path
           get :branch_from
           get :branch_to
-          get '', action: :new
         end
       end
 
@@ -206,20 +207,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
         end
       end
 
-      resources :clusters, except: [:edit, :create] do
-        collection do
-          post :create_gcp
-          post :create_user
-        end
-
-        member do
-          get :status, format: :json
-
-          scope :applications do
-            post '/:application', to: 'clusters/applications#create', as: :install_applications
-          end
-        end
-      end
+      concerns :clusterable
 
       resources :environments, except: [:destroy] do
         member do
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 0e723cdeb9c..53e1c8778b6 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -29,6 +29,7 @@
   - [pipeline_creation, 4]
   - [pipeline_default, 3]
   - [pipeline_cache, 3]
+  - [deployment, 3]
   - [pipeline_hooks, 2]
   - [gitlab_shell, 2]
   - [email_receiver, 2]