Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into dispatcher-projects-c

16 files changed, 317 insertions, 84 deletions

diff --git a/lib/api/circuit_breakers.rb b/lib/api/circuit_breakers.rb
index 598c76f6168..c13154dc0ec 100644
--- a/lib/api/circuit_breakers.rb
+++ b/lib/api/circuit_breakers.rb
@@ -17,11 +17,11 @@ module API
             end
 
             def storage_health
-              @failing_storage_health ||= Gitlab::Git::Storage::Health.for_all_storages
+              @storage_health ||= Gitlab::Git::Storage::Health.for_all_storages
             end
           end
 
-          desc 'Get all failing git storages' do
+          desc 'Get all git storages' do
             detail 'This feature was introduced in GitLab 9.5'
             success Entities::RepositoryStorageHealth
           end
diff --git a/lib/api/helpers/common_helpers.rb b/lib/api/helpers/common_helpers.rb
index 322624c6092..9993caa5249 100644
--- a/lib/api/helpers/common_helpers.rb
+++ b/lib/api/helpers/common_helpers.rb
@@ -3,8 +3,10 @@ module API
     module CommonHelpers
       def convert_parameters_from_legacy_format(params)
         params.tap do |params|
-          if params[:assignee_id].present?
-            params[:assignee_ids] = [params.delete(:assignee_id)]
+          assignee_id = params.delete(:assignee_id)
+
+          if assignee_id.present?
+            params[:assignee_ids] = [assignee_id]
           end
         end
       end
diff --git a/lib/gitlab/auth/blocked_user_tracker.rb b/lib/gitlab/auth/blocked_user_tracker.rb
new file mode 100644
index 00000000000..dae03a179e4
--- /dev/null
+++ b/lib/gitlab/auth/blocked_user_tracker.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+module Gitlab
+  module Auth
+    class BlockedUserTracker
+      ACTIVE_RECORD_REQUEST_PARAMS = 'action_dispatch.request.request_parameters'
+
+      def self.log_if_user_blocked(env)
+        message = env.dig('warden.options', :message)
+
+        # Devise calls User#active_for_authentication? on the User model and then
+        # throws an exception to Warden with User#inactive_message:
+        # https://github.com/plataformatec/devise/blob/v4.2.1/lib/devise/hooks/activatable.rb#L8
+        #
+        # Since Warden doesn't pass the user record to the failure handler, we
+        # need to do a database lookup with the username. We can limit the
+        # lookups to happen when the user was blocked by checking the inactive
+        # message passed along by Warden.
+        return unless message == User::BLOCKED_MESSAGE
+
+        login = env.dig(ACTIVE_RECORD_REQUEST_PARAMS, 'user', 'login')
+
+        return unless login.present?
+
+        user = User.by_login(login)
+
+        return unless user&.blocked?
+
+        Gitlab::AppLogger.info("Failed login for blocked user: user=#{user.username} ip=#{env['REMOTE_ADDR']}")
+        SystemHooksService.new.execute_hooks_for(user, :failed_login)
+
+        true
+      rescue TypeError
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb
index b4114a3ac96..cf02030c577 100644
--- a/lib/gitlab/auth/user_auth_finders.rb
+++ b/lib/gitlab/auth/user_auth_finders.rb
@@ -96,9 +96,7 @@ module Gitlab
       end
 
       def ensure_action_dispatch_request(request)
-        return request if request.is_a?(ActionDispatch::Request)
-
-        ActionDispatch::Request.new(request.env)
+        ActionDispatch::Request.new(request.env.dup)
       end
 
       def current_request
diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb
index 74c9874d590..07b7e811a34 100644
--- a/lib/gitlab/git/conflict/resolver.rb
+++ b/lib/gitlab/git/conflict/resolver.rb
@@ -15,7 +15,7 @@ module Gitlab
           @conflicts ||= begin
             @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled|
               if is_enabled
-                gitaly_conflicts_client(@target_repository).list_conflict_files
+                gitaly_conflicts_client(@target_repository).list_conflict_files.to_a
               else
                 rugged_list_conflict_files
               end
diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb
index 976fa1ddfe6..e5a747cb987 100644
--- a/lib/gitlab/git/gitlab_projects.rb
+++ b/lib/gitlab/git/gitlab_projects.rb
@@ -44,29 +44,13 @@ module Gitlab
       # Import project via git clone --bare
       # URL must be publicly cloneable
       def import_project(source, timeout)
-        # Skip import if repo already exists
-        return false if File.exist?(repository_absolute_path)
-
-        masked_source = mask_password_in_url(source)
-
-        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
-        cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
-
-        success = run_with_timeout(cmd, timeout, nil)
-
-        unless success
-          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
-          FileUtils.rm_rf(repository_absolute_path)
-          return false
+        Gitlab::GitalyClient.migrate(:import_repository) do |is_enabled|
+          if is_enabled
+            gitaly_import_repository(source)
+          else
+            git_import_repository(source, timeout)
+          end
         end
-
-        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
-
-        # The project was imported successfully.
-        # Remove the origin URL since it may contain password.
-        remove_origin_in_repo
-
-        true
       end
 
       def fork_repository(new_shard_path, new_repository_relative_path)
@@ -231,6 +215,42 @@ module Gitlab
           raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'")
       end
 
+      def git_import_repository(source, timeout)
+        # Skip import if repo already exists
+        return false if File.exist?(repository_absolute_path)
+
+        masked_source = mask_password_in_url(source)
+
+        logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>."
+        cmd = %W(git clone --bare -- #{source} #{repository_absolute_path})
+
+        success = run_with_timeout(cmd, timeout, nil)
+
+        unless success
+          logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.")
+          FileUtils.rm_rf(repository_absolute_path)
+          return false
+        end
+
+        Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path)
+
+        # The project was imported successfully.
+        # Remove the origin URL since it may contain password.
+        remove_origin_in_repo
+
+        true
+      end
+
+      def gitaly_import_repository(source)
+        raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil)
+
+        Gitlab::GitalyClient::RepositoryService.new(raw_repository).import_repository(source)
+        true
+      rescue GRPC::BadStatus => e
+        @output << e.message
+        false
+      end
+
       def git_fork_repository(new_shard_path, new_repository_relative_path)
         from_path = repository_absolute_path
         to_path = File.join(new_shard_path, new_repository_relative_path)
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index fed05bb6c64..71b212023d6 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -177,7 +177,7 @@ module Gitlab
 
         response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout)
         consume_commits_response(response)
-      rescue GRPC::Unknown # If no repository is found, happens mainly during testing
+      rescue GRPC::NotFound # If no repository is found, happens mainly during testing
         []
       end
 
diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
new file mode 100644
index 00000000000..97c13d1fdb0
--- /dev/null
+++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module GitalyClient
+    class ConflictFilesStitcher
+      include Enumerable
+
+      def initialize(rpc_response)
+        @rpc_response = rpc_response
+      end
+
+      def each
+        current_file = nil
+
+        @rpc_response.each do |msg|
+          msg.files.each do |gitaly_file|
+            if gitaly_file.header
+              yield current_file if current_file
+
+              current_file = file_from_gitaly_header(gitaly_file.header)
+            else
+              current_file.content << gitaly_file.content
+            end
+          end
+        end
+
+        yield current_file if current_file
+      end
+
+      private
+
+      def file_from_gitaly_header(header)
+        Gitlab::Git::Conflict::File.new(
+          Gitlab::GitalyClient::Util.git_repository(header.repository),
+          header.commit_oid,
+          conflict_from_gitaly_file_header(header),
+          ''
+        )
+      end
+
+      def conflict_from_gitaly_file_header(header)
+        {
+          ours: { path: header.our_path, mode: header.our_mode },
+          theirs: { path: header.their_path }
+        }
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb
index 40f032cf873..2565d537aff 100644
--- a/lib/gitlab/gitaly_client/conflicts_service.rb
+++ b/lib/gitlab/gitaly_client/conflicts_service.rb
@@ -20,7 +20,11 @@ module Gitlab
         )
         response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request)
 
-        files_from_response(response).to_a
+        GitalyClient::ConflictFilesStitcher.new(response)
+      end
+
+      def conflicts?
+        list_conflict_files.any?
       end
 
       def resolve_conflicts(target_repository, resolution, source_branch, target_branch)
@@ -58,38 +62,6 @@ module Gitlab
           user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly
         )
       end
-
-      def files_from_response(response)
-        files = []
-
-        response.each do |msg|
-          msg.files.each do |gitaly_file|
-            if gitaly_file.header
-              files << file_from_gitaly_header(gitaly_file.header)
-            else
-              files.last.content << gitaly_file.content
-            end
-          end
-        end
-
-        files
-      end
-
-      def file_from_gitaly_header(header)
-        Gitlab::Git::Conflict::File.new(
-          Gitlab::GitalyClient::Util.git_repository(header.repository),
-          header.commit_oid,
-          conflict_from_gitaly_file_header(header),
-          ''
-        )
-      end
-
-      def conflict_from_gitaly_file_header(header)
-        {
-          ours: { path: header.our_path, mode: header.our_mode },
-          theirs: { path: header.their_path }
-        }
-      end
     end
   end
 end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index 72ee92e78dc..12016aee2a6 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -100,6 +100,21 @@ module Gitlab
         )
       end
 
+      def import_repository(source)
+        request = Gitaly::CreateRepositoryFromURLRequest.new(
+          repository: @gitaly_repo,
+          url: source
+        )
+
+        GitalyClient.call(
+          @storage,
+          :repository_service,
+          :create_repository_from_url,
+          request,
+          timeout: GitalyClient.default_timeout
+        )
+      end
+
       def rebase_in_progress?(rebase_id)
         request = Gitaly::IsRebaseInProgressRequest.new(
           repository: @gitaly_repo,
diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb
index 8d8c441a4b1..bf6981035f4 100644
--- a/lib/gitlab/kubernetes/helm/install_command.rb
+++ b/lib/gitlab/kubernetes/helm/install_command.rb
@@ -36,7 +36,11 @@ module Gitlab
         def complete_command(namespace_name)
           return unless chart
 
-          "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null"
+          if chart_values_file
+            "helm install #{chart} --name #{name} --namespace #{namespace_name} -f /data/helm/#{name}/config/values.yaml >/dev/null"
+          else
+            "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null"
+          end
         end
 
         def install_dps_command
diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb
index 97ad3c97e95..a3216759cae 100644
--- a/lib/gitlab/kubernetes/helm/pod.rb
+++ b/lib/gitlab/kubernetes/helm/pod.rb
@@ -10,9 +10,10 @@ module Gitlab
 
         def generate
           spec = { containers: [container_specification], restartPolicy: 'Never' }
+
           if command.chart_values_file
-            generate_config_map
-            spec['volumes'] = volumes_specification
+            create_config_map
+            spec[:volumes] = volumes_specification
           end
 
           ::Kubeclient::Resource.new(metadata: metadata, spec: spec)
@@ -35,19 +36,39 @@ module Gitlab
         end
 
         def labels
-          { 'gitlab.org/action': 'install', 'gitlab.org/application': command.name }
+          {
+            'gitlab.org/action': 'install',
+            'gitlab.org/application': command.name
+          }
         end
 
         def metadata
-          { name: command.pod_name, namespace: namespace_name, labels: labels }
+          {
+            name: command.pod_name,
+            namespace: namespace_name,
+            labels: labels
+          }
         end
 
         def volume_mounts_specification
-          [{ name: 'config-volume', mountPath: '/etc/config' }]
+          [
+            {
+              name: 'configuration-volume',
+              mountPath: "/data/helm/#{command.name}/config"
+            }
+          ]
         end
 
         def volumes_specification
-          [{ name: 'config-volume', configMap: { name: 'values-config' } }]
+          [
+            {
+              name: 'configuration-volume',
+              configMap: {
+                name: 'values-content-configuration',
+                items: [{ key: 'values', path: 'values.yaml' }]
+              }
+            }
+          ]
         end
 
         def generate_pod_env(command)
@@ -58,10 +79,10 @@ module Gitlab
           }.map { |key, value| { name: key, value: value } }
         end
 
-        def generate_config_map
+        def create_config_map
           resource = ::Kubeclient::Resource.new
-          resource.metadata = { name: 'values-config', namespace: namespace_name }
-          resource.data = YAML.load_file(command.chart_values_file)
+          resource.metadata = { name: 'values-content-configuration', namespace: namespace_name, labels: { name: 'values-content-configuration' } }
+          resource.data = { values: File.read(command.chart_values_file) }
           kubeclient.create_config_map(resource)
         end
       end
diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb
new file mode 100644
index 00000000000..8bf6bcb1fe2
--- /dev/null
+++ b/lib/gitlab/utils/override.rb
@@ -0,0 +1,111 @@
+module Gitlab
+  module Utils
+    module Override
+      class Extension
+        def self.verify_class!(klass, method_name)
+          instance_method_defined?(klass, method_name) ||
+            raise(
+              NotImplementedError.new(
+                "#{klass}\##{method_name} doesn't exist!"))
+        end
+
+        def self.instance_method_defined?(klass, name, include_super: true)
+          klass.instance_methods(include_super).include?(name) ||
+            klass.private_instance_methods(include_super).include?(name)
+        end
+
+        attr_reader :subject
+
+        def initialize(subject)
+          @subject = subject
+        end
+
+        def add_method_name(method_name)
+          method_names << method_name
+        end
+
+        def add_class(klass)
+          classes << klass
+        end
+
+        def verify!
+          classes.each do |klass|
+            index = klass.ancestors.index(subject)
+            parents = klass.ancestors.drop(index + 1)
+
+            method_names.each do |method_name|
+              parents.any? do |parent|
+                self.class.instance_method_defined?(
+                  parent, method_name, include_super: false)
+              end ||
+                raise(
+                  NotImplementedError.new(
+                    "#{klass}\##{method_name} doesn't exist!"))
+            end
+          end
+        end
+
+        private
+
+        def method_names
+          @method_names ||= []
+        end
+
+        def classes
+          @classes ||= []
+        end
+      end
+
+      # Instead of writing patterns like this:
+      #
+      #     def f
+      #       raise NotImplementedError unless defined?(super)
+      #
+      #       true
+      #     end
+      #
+      # We could write it like:
+      #
+      #     extend ::Gitlab::Utils::Override
+      #
+      #     override :f
+      #     def f
+      #       true
+      #     end
+      #
+      # This would make sure we're overriding something. See:
+      # https://gitlab.com/gitlab-org/gitlab-ee/issues/1819
+      def override(method_name)
+        return unless ENV['STATIC_VERIFICATION']
+
+        if is_a?(Class)
+          Extension.verify_class!(self, method_name)
+        else # We delay the check for modules
+          Override.extensions[self] ||= Extension.new(self)
+          Override.extensions[self].add_method_name(method_name)
+        end
+      end
+
+      def included(base = nil)
+        return super if base.nil? # Rails concern, ignoring it
+
+        super
+
+        if base.is_a?(Class) # We could check for Class in `override`
+          # This could be `nil` if `override` was never called
+          Override.extensions[self]&.add_class(base)
+        end
+      end
+
+      alias_method :prepended, :included
+
+      def self.extensions
+        @extensions ||= {}
+      end
+
+      def self.verify!
+        extensions.values.each(&:verify!)
+      end
+    end
+  end
+end
diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake
index e65609d7001..4beb94eeb8e 100644
--- a/lib/tasks/dev.rake
+++ b/lib/tasks/dev.rake
@@ -7,4 +7,9 @@ namespace :dev do
     Rake::Task["gitlab:setup"].invoke
     Rake::Task["gitlab:shell:setup"].invoke
   end
+
+  desc "GitLab | Eager load application"
+  task load: :environment do
+    Rails.application.eager_load!
+  end
 end
diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake
index 0e6aed32c52..12ae4199b69 100644
--- a/lib/tasks/gitlab/shell.rake
+++ b/lib/tasks/gitlab/shell.rake
@@ -54,16 +54,6 @@ namespace :gitlab do
       # (Re)create hooks
       Rake::Task['gitlab:shell:create_hooks'].invoke
 
-      # Required for debian packaging with PKGR: Setup .ssh/environment with
-      # the current PATH, so that the correct ruby version gets loaded
-      # Requires to set "PermitUserEnvironment yes" in sshd config (should not
-      # be an issue since it is more than likely that there are no "normal"
-      # user accounts on a gitlab server). The alternative is for the admin to
-      # install a ruby (1.9.3+) in the global path.
-      File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f|
-        f.puts "PATH=#{ENV['PATH']}"
-      end
-
       Gitlab::Shell.ensure_secret_token!
     end
 
diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake
index 7b63e93db0e..3ab406eff2c 100644
--- a/lib/tasks/lint.rake
+++ b/lib/tasks/lint.rake
@@ -1,5 +1,17 @@
 unless Rails.env.production?
   namespace :lint do
+    task :static_verification_env do
+      ENV['STATIC_VERIFICATION'] = 'true'
+    end
+
+    desc "GitLab | lint | Static verification"
+    task static_verification: %w[
+      lint:static_verification_env
+      dev:load
+    ] do
+      Gitlab::Utils::Override.verify!
+    end
+
     desc "GitLab | lint | Lint JavaScript files using ESLint"
     task :javascript do
       Rake::Task['eslint'].invoke