Add latest changes from gitlab-org/gitlab@master

3 files changed, 222 insertions, 0 deletions

diff --git a/app/services/ci/queue/build_queue_service.rb b/app/services/ci/queue/build_queue_service.rb
new file mode 100644
index 00000000000..8190599fbb5
--- /dev/null
+++ b/app/services/ci/queue/build_queue_service.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Ci
+  module Queue
+    class BuildQueueService
+      include ::Gitlab::Utils::StrongMemoize
+
+      attr_reader :runner
+
+      def initialize(runner)
+        @runner = runner
+      end
+
+      def new_builds
+        strategy.new_builds
+      end
+
+      ##
+      # This is overridden in EE
+      #
+      def builds_for_shared_runner
+        strategy.builds_for_shared_runner
+      end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def builds_for_group_runner
+        # Workaround for weird Rails bug, that makes `runner.groups.to_sql` to return `runner_id = NULL`
+        groups = ::Group.joins(:runner_namespaces).merge(runner.runner_namespaces)
+
+        hierarchy_groups = Gitlab::ObjectHierarchy
+          .new(groups, options: { use_distinct: ::Feature.enabled?(:use_distinct_in_register_job_object_hierarchy) })
+          .base_and_descendants
+
+        projects = Project.where(namespace_id: hierarchy_groups)
+          .with_group_runners_enabled
+          .with_builds_enabled
+          .without_deleted
+
+        relation = new_builds.where(project: projects)
+
+        order(relation)
+      end
+
+      def builds_for_project_runner
+        relation = new_builds
+          .where(project: runner.projects.without_deleted.with_builds_enabled)
+
+        order(relation)
+      end
+
+      def builds_queued_before(relation, time)
+        relation.queued_before(time)
+      end
+
+      def builds_for_protected_runner(relation)
+        relation.ref_protected
+      end
+
+      def builds_matching_tag_ids(relation, ids)
+        strategy.builds_matching_tag_ids(relation, ids)
+      end
+
+      def builds_with_any_tags(relation)
+        strategy.builds_with_any_tags(relation)
+      end
+
+      def order(relation)
+        strategy.order(relation)
+      end
+
+      def execute(relation)
+        strategy.build_ids(relation)
+      end
+
+      private
+
+      def strategy
+        strong_memoize(:strategy) do
+          if ::Feature.enabled?(:ci_pending_builds_queue_source, runner, default_enabled: :yaml)
+            Queue::PendingBuildsStrategy.new(runner)
+          else
+            Queue::BuildsTableStrategy.new(runner)
+          end
+        end
+      end
+    end
+  end
+end
+
+Ci::Queue::BuildQueueService.prepend_mod_with('Ci::Queue::BuildQueueService')
diff --git a/app/services/ci/queue/builds_table_strategy.rb b/app/services/ci/queue/builds_table_strategy.rb
new file mode 100644
index 00000000000..2039ece8281
--- /dev/null
+++ b/app/services/ci/queue/builds_table_strategy.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Ci
+  module Queue
+    class BuildsTableStrategy
+      attr_reader :runner
+
+      def initialize(runner)
+        @runner = runner
+      end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def builds_for_shared_runner
+        relation = new_builds
+          # don't run projects which have not enabled shared runners and builds
+          .joins('INNER JOIN projects ON ci_builds.project_id = projects.id')
+          .where(projects: { shared_runners_enabled: true, pending_delete: false })
+          .joins('LEFT JOIN project_features ON ci_builds.project_id = project_features.project_id')
+          .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0')
+
+        if Feature.enabled?(:ci_queueing_disaster_recovery, runner, type: :ops, default_enabled: :yaml)
+          # if disaster recovery is enabled, we fallback to FIFO scheduling
+          relation.order('ci_builds.id ASC')
+        else
+          # Implement fair scheduling
+          # this returns builds that are ordered by number of running builds
+          # we prefer projects that don't use shared runners at all
+          relation
+            .joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.project_id = project_builds.project_id")
+            .order(Arel.sql('COALESCE(project_builds.running_builds, 0) ASC'), 'ci_builds.id ASC')
+        end
+      end
+
+      def builds_matching_tag_ids(relation, ids)
+        # pick builds that does not have other tags than runner's one
+        relation.matches_tag_ids(ids)
+      end
+
+      def builds_with_any_tags(relation)
+        # pick builds that have at least one tag
+        relation.with_any_tags
+      end
+
+      def order(relation)
+        relation.order('id ASC')
+      end
+
+      def new_builds
+        ::Ci::Build.pending.unstarted
+      end
+
+      def build_ids(relation)
+        relation.pluck(:id)
+      end
+
+      private
+
+      def running_builds_for_shared_runners
+        ::Ci::Build.running
+          .where(runner: ::Ci::Runner.instance_type)
+          .group(:project_id)
+          .select(:project_id, 'COUNT(*) AS running_builds')
+      end
+      # rubocop:enable CodeReuse/ActiveRecord
+    end
+  end
+end
diff --git a/app/services/ci/queue/pending_builds_strategy.rb b/app/services/ci/queue/pending_builds_strategy.rb
new file mode 100644
index 00000000000..1c6007f0be8
--- /dev/null
+++ b/app/services/ci/queue/pending_builds_strategy.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Ci
+  module Queue
+    class PendingBuildsStrategy
+      attr_reader :runner
+
+      def initialize(runner)
+        @runner = runner
+      end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def builds_for_shared_runner
+        relation = new_builds
+          # don't run projects which have not enabled shared runners and builds
+          .joins('INNER JOIN projects ON ci_pending_builds.project_id = projects.id')
+          .where(projects: { shared_runners_enabled: true, pending_delete: false })
+          .joins('LEFT JOIN project_features ON ci_pending_builds.project_id = project_features.project_id')
+          .where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0')
+
+        if Feature.enabled?(:ci_queueing_disaster_recovery, runner, type: :ops, default_enabled: :yaml)
+          # if disaster recovery is enabled, we fallback to FIFO scheduling
+          relation.order('ci_pending_builds.build_id ASC')
+        else
+          # Implement fair scheduling
+          # this returns builds that are ordered by number of running builds
+          # we prefer projects that don't use shared runners at all
+          relation
+            .joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_pending_builds.project_id=project_builds.project_id")
+            .order(Arel.sql('COALESCE(project_builds.running_builds, 0) ASC'), 'ci_pending_builds.build_id ASC')
+        end
+      end
+
+      def builds_matching_tag_ids(relation, ids)
+        relation.merge(CommitStatus.matches_tag_ids(ids, table: 'ci_pending_builds', column: 'build_id'))
+      end
+
+      def builds_with_any_tags(relation)
+        relation.merge(CommitStatus.with_any_tags(table: 'ci_pending_builds', column: 'build_id'))
+      end
+
+      def order(relation)
+        relation.order('build_id ASC')
+      end
+
+      def new_builds
+        ::Ci::PendingBuild.all
+      end
+
+      def build_ids(relation)
+        relation.pluck(:build_id)
+      end
+
+      private
+
+      def running_builds_for_shared_runners
+        ::Ci::RunningBuild
+          .instance_type
+          .group(:project_id)
+          .select(:project_id, 'COUNT(*) AS running_builds')
+      end
+      # rubocop:enable CodeReuse/ActiveRecord
+    end
+  end
+end