Add latest changes from gitlab-org/gitlab@master

16 files changed, 212 insertions, 46 deletions

diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb
index 994e12445b7..48d4f1a0a63 100644
--- a/lib/api/broadcast_messages.rb
+++ b/lib/api/broadcast_messages.rb
@@ -4,9 +4,6 @@ module API
   class BroadcastMessages < Grape::API
     include PaginationParams
 
-    before { authenticate! }
-    before { authenticated_as_admin! }
-
     resource :broadcast_messages do
       helpers do
         def find_message
@@ -40,6 +37,8 @@ module API
         optional :target_path, type: String, desc: 'Target path'
       end
       post do
+        authenticated_as_admin!
+
         message = BroadcastMessage.create(declared_params(include_missing: false))
 
         if message.persisted?
@@ -76,6 +75,8 @@ module API
         optional :target_path, type: String, desc: 'Target path'
       end
       put ':id' do
+        authenticated_as_admin!
+
         message = find_message
 
         if message.update(declared_params(include_missing: false))
@@ -93,6 +94,8 @@ module API
         requires :id, type: Integer, desc: 'Broadcast message ID'
       end
       delete ':id' do
+        authenticated_as_admin!
+
         message = find_message
 
         destroy_conditionally!(message)
diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb
index c8e96c7c5db..cc9d45dcae4 100644
--- a/lib/api/helpers/internal_helpers.rb
+++ b/lib/api/helpers/internal_helpers.rb
@@ -104,14 +104,12 @@ module API
 
       # rubocop:disable Gitlab/ModuleWithInstanceVariables
       def set_project
-        if params[:gl_repository]
-          @project, @repo_type = Gitlab::GlRepository.parse(params[:gl_repository])
-          @redirected_path = nil
-        elsif params[:project]
-          @project, @repo_type, @redirected_path = Gitlab::RepoPath.parse(params[:project])
-        else
-          @project, @repo_type, @redirected_path = nil, nil, nil
-        end
+        @project, @repo_type, @redirected_path =
+          if params[:gl_repository]
+            Gitlab::GlRepository.parse(params[:gl_repository])
+          elsif params[:project]
+            Gitlab::RepoPath.parse(params[:project])
+          end
       end
       # rubocop:enable Gitlab/ModuleWithInstanceVariables
 
diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb
index d41490d2ebd..3e9cf2ab320 100644
--- a/lib/constraints/project_url_constrainer.rb
+++ b/lib/constraints/project_url_constrainer.rb
@@ -4,7 +4,7 @@ module Constraints
   class ProjectUrlConstrainer
     def matches?(request, existence_check: true)
       namespace_path = request.params[:namespace_id]
-      project_path = request.params[:project_id] || request.params[:id]
+      project_path = request.params[:project_id] || request.params[:id] || request.params[:repository_id]
       full_path = [namespace_path, project_path].join('/')
 
       return false unless ProjectPathValidator.valid_path?(full_path)
diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb
index c225ca56c55..96062ae87bc 100644
--- a/lib/feature/gitaly.rb
+++ b/lib/feature/gitaly.rb
@@ -9,7 +9,6 @@ class Feature
       %w[
         cache_invalidator
         inforef_uploadpack_cache
-        get_tag_messages_go
         filter_shas_with_signatures_go
         commit_without_batch_check
       ].freeze
diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb
index 64ab5db4fcd..89a836e629f 100644
--- a/lib/gitaly/server.rb
+++ b/lib/gitaly/server.rb
@@ -53,6 +53,20 @@ module Gitaly
       storage_status&.fs_type
     end
 
+    def disk_used
+      disk_statistics_storage_status&.used
+    end
+
+    def disk_available
+      disk_statistics_storage_status&.available
+    end
+
+    # Simple convenience method for when obtaining both used and available
+    # statistics at once is preferred.
+    def disk_stats
+      disk_statistics_storage_status
+    end
+
     def address
       Gitlab::GitalyClient.address(@storage)
     rescue RuntimeError => e
@@ -65,6 +79,10 @@ module Gitaly
       @storage_status ||= info.storage_statuses.find { |s| s.storage_name == storage }
     end
 
+    def disk_statistics_storage_status
+      @disk_statistics_storage_status ||= disk_statistics.storage_statuses.find { |s| s.storage_name == storage }
+    end
+
     def matches_sha?
       match = server_version.match(SHA_VERSION_REGEX)
       return false unless match
@@ -76,7 +94,19 @@ module Gitaly
       @info ||=
         begin
           Gitlab::GitalyClient::ServerService.new(@storage).info
-        rescue GRPC::Unavailable, GRPC::DeadlineExceeded
+        rescue GRPC::Unavailable, GRPC::DeadlineExceeded => ex
+          Gitlab::ErrorTracking.track_exception(ex)
+          # This will show the server as being out of date
+          Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: [])
+        end
+    end
+
+    def disk_statistics
+      @disk_statistics ||=
+        begin
+          Gitlab::GitalyClient::ServerService.new(@storage).disk_statistics
+        rescue GRPC::Unavailable, GRPC::DeadlineExceeded => ex
+          Gitlab::ErrorTracking.track_exception(ex)
           # This will show the server as being out of date
           Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: [])
         end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 262a1ef653f..4eb1ccf32ba 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -432,10 +432,7 @@ module Gitlab
     end
 
     def self.filesystem_id(storage)
-      response = Gitlab::GitalyClient::ServerService.new(storage).info
-      storage_status = response.storage_statuses.find { |status| status.storage_name == storage }
-
-      storage_status&.filesystem_id
+      Gitlab::GitalyClient::ServerService.new(storage).storage_info&.filesystem_id
     end
 
     def self.filesystem_id_from_disk(storage)
@@ -446,6 +443,14 @@ module Gitlab
       nil
     end
 
+    def self.filesystem_disk_available(storage)
+      Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.available
+    end
+
+    def self.filesystem_disk_used(storage)
+      Gitlab::GitalyClient::ServerService.new(storage).storage_disk_statistics&.used
+    end
+
     def self.timeout(timeout_name)
       Gitlab::CurrentSettings.current_application_settings[timeout_name]
     end
diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb
index 0ade6942db9..36bda67c26e 100644
--- a/lib/gitlab/gitaly_client/server_service.rb
+++ b/lib/gitlab/gitaly_client/server_service.rb
@@ -13,6 +13,24 @@ module Gitlab
       def info
         GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new, timeout: GitalyClient.fast_timeout)
       end
+
+      def disk_statistics
+        GitalyClient.call(@storage, :server_service, :disk_statistics, Gitaly::DiskStatisticsRequest.new, timeout: GitalyClient.fast_timeout)
+      end
+
+      def storage_info
+        storage_specific(info)
+      end
+
+      def storage_disk_statistics
+        storage_specific(disk_statistics)
+      end
+
+      private
+
+      def storage_specific(response)
+        response.storage_statuses.find { |status| status.storage_name == @storage }
+      end
     end
   end
 end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index 2e27e954e79..3db6c3b51c0 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -43,6 +43,9 @@ module Gitlab
       # Initialize gon.features with any flags that should be
       # made globally available to the frontend
       push_frontend_feature_flag(:snippets_vue, default_enabled: false)
+      push_frontend_feature_flag(:monaco_snippets, default_enabled: false)
+      push_frontend_feature_flag(:monaco_blobs, default_enabled: false)
+      push_frontend_feature_flag(:monaco_ci, default_enabled: false)
     end
 
     # Exposes the state of a feature flag to the frontend code.
diff --git a/lib/gitlab/import_export/import_failure_service.rb b/lib/gitlab/import_export/import_failure_service.rb
index eeaf10870c8..d4eca551b49 100644
--- a/lib/gitlab/import_export/import_failure_service.rb
+++ b/lib/gitlab/import_export/import_failure_service.rb
@@ -12,9 +12,14 @@ module Gitlab
         @association = importable.association(:import_failures)
       end
 
-      def with_retry(relation_key, relation_index)
+      def with_retry(action:, relation_key: nil, relation_index: nil)
         on_retry = -> (exception, retry_count, *_args) do
-          log_import_failure(relation_key, relation_index, exception, retry_count)
+          log_import_failure(
+            source: action,
+            relation_key: relation_key,
+            relation_index: relation_index,
+            exception: exception,
+            retry_count: retry_count)
         end
 
         Retriable.with_context(:relation_import, on_retry: on_retry) do
@@ -22,8 +27,9 @@ module Gitlab
         end
       end
 
-      def log_import_failure(relation_key, relation_index, exception, retry_count = 0)
+      def log_import_failure(source:, relation_key: nil, relation_index: nil, exception:, retry_count: 0)
         extra = {
+          source: source,
           relation_key: relation_key,
           relation_index: relation_index,
           retry_count: retry_count
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index e598cfc143e..c4ac6a3a3f2 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -21,7 +21,9 @@ module Gitlab
         RelationRenameService.rename(@tree_hash)
 
         if relation_tree_restorer.restore
-          @project.merge_requests.set_latest_merge_request_diff_ids!
+          import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
+            @project.merge_requests.set_latest_merge_request_diff_ids!
+          end
 
           true
         else
@@ -72,6 +74,10 @@ module Gitlab
       def reader
         @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
       end
+
+      def import_failure_service
+        @import_failure_service ||= ImportFailureService.new(@project)
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb
index 44cf90fb86a..3606e1c5bd9 100644
--- a/lib/gitlab/import_export/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/relation_tree_restorer.rb
@@ -73,13 +73,17 @@ module Gitlab
 
         relation_object.assign_attributes(importable_class_sym => @importable)
 
-        import_failure_service.with_retry(relation_key, relation_index) do
+        import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do
           relation_object.save!
         end
 
         save_id_mapping(relation_key, data_hash, relation_object)
       rescue => e
-        import_failure_service.log_import_failure(relation_key, relation_index, e)
+        import_failure_service.log_import_failure(
+          source: 'process_relation_item!',
+          relation_key: relation_key,
+          relation_index: relation_index,
+          exception: e)
       end
 
       def import_failure_service
diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb
index c749816cf6a..2cf2a81f812 100644
--- a/lib/gitlab/middleware/read_only/controller.rb
+++ b/lib/gitlab/middleware/read_only/controller.rb
@@ -12,12 +12,12 @@ module Gitlab
         ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'
 
         WHITELISTED_GIT_ROUTES = {
-          'projects/git_http' => %w{git_upload_pack git_receive_pack}
+          'repositories/git_http' => %w{git_upload_pack git_receive_pack}
         }.freeze
 
         WHITELISTED_GIT_LFS_ROUTES = {
-          'projects/lfs_api' => %w{batch},
-          'projects/lfs_locks_api' => %w{verify create unlock}
+          'repositories/lfs_api' => %w{batch},
+          'repositories/lfs_locks_api' => %w{verify create unlock}
         }.freeze
 
         WHITELISTED_GIT_REVISION_ROUTES = {
diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb
index c96212f27a7..c8719023d40 100644
--- a/lib/gitlab/sidekiq_config.rb
+++ b/lib/gitlab/sidekiq_config.rb
@@ -6,6 +6,7 @@ module Gitlab
   module SidekiqConfig
     FOSS_QUEUE_CONFIG_PATH = 'app/workers/all_queues.yml'
     EE_QUEUE_CONFIG_PATH = 'ee/app/workers/all_queues.yml'
+    SIDEKIQ_QUEUES_PATH = 'config/sidekiq_queues.yml'
 
     QUEUE_CONFIG_PATHS = [
       FOSS_QUEUE_CONFIG_PATH,
@@ -13,11 +14,19 @@ module Gitlab
     ].compact.freeze
 
     # For queues that don't have explicit workers - default and mailers
-    DummyWorker = Struct.new(:queue)
+    DummyWorker = Struct.new(:queue, :weight) do
+      def queue_namespace
+        nil
+      end
+
+      def get_weight
+        weight
+      end
+    end
 
     DEFAULT_WORKERS = [
-      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default'), ee: false),
-      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers'), ee: false)
+      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('default', 1), ee: false),
+      Gitlab::SidekiqConfig::Worker.new(DummyWorker.new('mailers', 2), ee: false)
     ].freeze
 
     class << self
@@ -30,7 +39,7 @@ module Gitlab
 
       def config_queues
         @config_queues ||= begin
-          config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml'))
+          config = YAML.load_file(Rails.root.join(SIDEKIQ_QUEUES_PATH))
           config[:queues].map(&:first)
         end
       end
@@ -65,6 +74,28 @@ module Gitlab
         Gitlab.ee? && ee_workers != YAML.safe_load(File.read(EE_QUEUE_CONFIG_PATH))
       end
 
+      def queues_for_sidekiq_queues_yml
+        namespaces_with_equal_weights =
+          workers
+            .group_by(&:queue_namespace)
+            .map(&:last)
+            .select { |workers| workers.map(&:get_weight).uniq.count == 1 }
+            .map(&:first)
+
+        namespaces = namespaces_with_equal_weights.map(&:queue_namespace).to_set
+        remaining_queues = workers.reject { |worker| namespaces.include?(worker.queue_namespace) }
+
+        (namespaces_with_equal_weights.map(&:namespace_and_weight) +
+         remaining_queues.map(&:queue_and_weight)).sort
+      end
+
+      def sidekiq_queues_yml_outdated?
+        # YAML.load is OK here as we control the file contents
+        config_queues = YAML.load(File.read(SIDEKIQ_QUEUES_PATH))[:queues] # rubocop:disable Security/YAMLLoad
+
+        queues_for_sidekiq_queues_yml != config_queues
+      end
+
       private
 
       def find_workers(root, ee:)
diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb
index 313d9b17b16..ac94bab9a8f 100644
--- a/lib/gitlab/sidekiq_config/worker.rb
+++ b/lib/gitlab/sidekiq_config/worker.rb
@@ -7,8 +7,9 @@ module Gitlab
 
       attr_reader :klass
       delegate :feature_category_not_owned?, :get_feature_category,
-               :get_worker_resource_boundary, :latency_sensitive_worker?,
-               :queue, :worker_has_external_dependencies?,
+               :get_weight, :get_worker_resource_boundary,
+               :latency_sensitive_worker?, :queue, :queue_namespace,
+               :worker_has_external_dependencies?,
                to: :klass
 
       def initialize(klass, ee:)
@@ -35,7 +36,7 @@ module Gitlab
 
       # Put namespaced queues first
       def to_sort
-        [queue.include?(':') ? 0 : 1, queue]
+        [queue_namespace ? 0 : 1, queue]
       end
 
       # YAML representation
@@ -46,6 +47,14 @@ module Gitlab
       def to_yaml
         queue
       end
+
+      def namespace_and_weight
+        [queue_namespace, get_weight]
+      end
+
+      def queue_and_weight
+        [queue, get_weight]
+      end
     end
   end
 end
diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake
index 15c6c930386..eb3de195626 100644
--- a/lib/tasks/gitlab/sidekiq.rake
+++ b/lib/tasks/gitlab/sidekiq.rake
@@ -4,8 +4,13 @@ return if Rails.env.production?
 
 namespace :gitlab do
   namespace :sidekiq do
+    def write_yaml(path, banner, object)
+      File.write(path, banner + YAML.dump(object))
+    end
+
     namespace :all_queues_yml do
-      def write_yaml(path, object)
+      desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions'
+      task generate: :environment do
         banner = <<~BANNER
           # This file is generated automatically by
           #   bin/rake gitlab:sidekiq:all_queues_yml:generate
@@ -13,17 +18,12 @@ namespace :gitlab do
           # Do not edit it manually!
         BANNER
 
-        File.write(path, banner + YAML.dump(object))
-      end
-
-      desc 'GitLab | Sidekiq | Generate all_queues.yml based on worker definitions'
-      task generate: :environment do
         foss_workers, ee_workers = Gitlab::SidekiqConfig.workers_for_all_queues_yml
 
-        write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, foss_workers)
+        write_yaml(Gitlab::SidekiqConfig::FOSS_QUEUE_CONFIG_PATH, banner, foss_workers)
 
         if Gitlab.ee?
-          write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, ee_workers)
+          write_yaml(Gitlab::SidekiqConfig::EE_QUEUE_CONFIG_PATH, banner, ee_workers)
         end
       end
 
@@ -44,5 +44,57 @@ namespace :gitlab do
         end
       end
     end
+
+    namespace :sidekiq_queues_yml do
+      desc 'GitLab | Sidekiq | Generate sidekiq_queues.yml based on worker definitions'
+      task generate: :environment do
+        banner = <<~BANNER
+          # This file is generated automatically by
+          #   bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate
+          #
+          # Do not edit it manually!
+          #
+          # This configuration file should be exclusively used to set queue settings for
+          # Sidekiq. Any other setting should be specified using the Sidekiq CLI or the
+          # Sidekiq Ruby API (see config/initializers/sidekiq.rb).
+          #
+          # All the queues to process and their weights. Every queue _must_ have a weight
+          # defined.
+          #
+          # The available weights are as follows
+          #
+          # 1: low priority
+          # 2: medium priority
+          # 3: high priority
+          # 5: _super_ high priority, this should only be used for _very_ important queues
+          #
+          # As per http://stackoverflow.com/a/21241357/290102 the formula for calculating
+          # the likelihood of a job being popped off a queue (given all queues have work
+          # to perform) is:
+          #
+          #     chance = (queue weight / total weight of all queues) * 100
+        BANNER
+
+        queues_and_weights = Gitlab::SidekiqConfig.queues_for_sidekiq_queues_yml
+
+        write_yaml(Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH, banner, queues: queues_and_weights)
+      end
+
+      desc 'GitLab | Sidekiq | Validate that sidekiq_queues.yml matches worker definitions'
+      task check: :environment do
+        if Gitlab::SidekiqConfig.sidekiq_queues_yml_outdated?
+          raise <<~MSG
+            Changes in worker queues found, please update the metadata by running:
+
+              bin/rake gitlab:sidekiq:sidekiq_queues_yml:generate
+
+            Then commit and push the changes from:
+
+            - #{Gitlab::SidekiqConfig::SIDEKIQ_QUEUES_PATH}
+
+          MSG
+        end
+      end
+    end
   end
 end
diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake
index 91e7031744a..7a4d09bb6d4 100644
--- a/lib/tasks/lint.rake
+++ b/lib/tasks/lint.rake
@@ -38,11 +38,13 @@ unless Rails.env.production?
       ]
 
       if Gitlab.ee?
-        # This task will fail on CE installations (e.g. gitlab-org/gitlab-foss)
-        # since it will detect strings in the locale files that do not exist in
-        # the source files. To work around this we will only enable this task on
-        # EE installations.
+        # These tasks will fail on FOSS installations
+        # (e.g. gitlab-org/gitlab-foss) since they test against a single
+        # file that is generated by an EE installation, which can
+        # contain values that a FOSS installation won't find.  To work
+        # around this we will only enable this task on EE installations.
         tasks << 'gettext:updated_check'
+        tasks << 'gitlab:sidekiq:sidekiq_queues_yml:check'
       end
 
       tasks.each do |task|