Merge branch '51651-fill-pipeline-source-for-external-pipelines' into 'master'

Resolve "Fill pipeline source for external pipelines"

Closes #51651

See merge request gitlab-org/gitlab-ce!21814

6 files changed, 171 insertions, 0 deletions

diff --git a/changelogs/unreleased/51651-fill-pipeline-source-for-external-pipelines.yml b/changelogs/unreleased/51651-fill-pipeline-source-for-external-pipelines.yml
new file mode 100644
index 00000000000..6720430e5fc
--- /dev/null
+++ b/changelogs/unreleased/51651-fill-pipeline-source-for-external-pipelines.yml
@@ -0,0 +1,5 @@
+---
+title: Retroactively fill pipeline source for external pipelines.
+merge_request: 21814
+author:
+type: other
diff --git a/db/migrate/20180916011959_add_index_pipelines_project_id_source.rb b/db/migrate/20180916011959_add_index_pipelines_project_id_source.rb
new file mode 100644
index 00000000000..b9bebf30cf0
--- /dev/null
+++ b/db/migrate/20180916011959_add_index_pipelines_project_id_source.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class AddIndexPipelinesProjectIdSource < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_pipelines, [:project_id, :source]
+  end
+
+  def down
+    remove_concurrent_index :ci_pipelines, [:project_id, :source]
+  end
+end
diff --git a/db/post_migrate/20180916014356_populate_external_pipeline_source.rb b/db/post_migrate/20180916014356_populate_external_pipeline_source.rb
new file mode 100644
index 00000000000..5577d05cf40
--- /dev/null
+++ b/db/post_migrate/20180916014356_populate_external_pipeline_source.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class PopulateExternalPipelineSource < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+  MIGRATION = 'PopulateExternalPipelineSource'.freeze
+  BATCH_SIZE = 500
+
+  disable_ddl_transaction!
+
+  class Pipeline < ActiveRecord::Base
+    include EachBatch
+    self.table_name = 'ci_pipelines'
+  end
+
+  def up
+    Pipeline.where(source: nil).tap do |relation|
+      queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                            MIGRATION,
+                                                            5.minutes,
+                                                            batch_size: BATCH_SIZE)
+    end
+  end
+
+  def down
+    # noop
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f92d8005dfb..ecb9d4391d7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -475,6 +475,7 @@ ActiveRecord::Schema.define(version: 20180917172041) do
   add_index "ci_pipelines", ["project_id", "iid"], name: "index_ci_pipelines_on_project_id_and_iid", unique: true, where: "(iid IS NOT NULL)", using: :btree
   add_index "ci_pipelines", ["project_id", "ref", "status", "id"], name: "index_ci_pipelines_on_project_id_and_ref_and_status_and_id", using: :btree
   add_index "ci_pipelines", ["project_id", "sha"], name: "index_ci_pipelines_on_project_id_and_sha", using: :btree
+  add_index "ci_pipelines", ["project_id", "source"], name: "index_ci_pipelines_on_project_id_and_source", using: :btree
   add_index "ci_pipelines", ["project_id", "status", "config_source"], name: "index_ci_pipelines_on_project_id_and_status_and_config_source", using: :btree
   add_index "ci_pipelines", ["project_id"], name: "index_ci_pipelines_on_project_id", using: :btree
   add_index "ci_pipelines", ["status"], name: "index_ci_pipelines_on_status", using: :btree
diff --git a/lib/gitlab/background_migration/populate_external_pipeline_source.rb b/lib/gitlab/background_migration/populate_external_pipeline_source.rb
new file mode 100644
index 00000000000..036fe641757
--- /dev/null
+++ b/lib/gitlab/background_migration/populate_external_pipeline_source.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class PopulateExternalPipelineSource
+      module Migratable
+        class Pipeline < ActiveRecord::Base
+          self.table_name = 'ci_pipelines'
+
+          def self.sources
+            {
+              unknown: nil,
+              push: 1,
+              web: 2,
+              trigger: 3,
+              schedule: 4,
+              api: 5,
+              external: 6
+            }
+          end
+        end
+
+        class CommitStatus < ActiveRecord::Base
+          self.table_name = 'ci_builds'
+          self.inheritance_column = :_type_disabled
+
+          scope :has_pipeline, -> { where('ci_builds.commit_id=ci_pipelines.id') }
+          scope :of_type, -> (type) { where('type=?', type) }
+        end
+      end
+
+      def perform(start_id, stop_id)
+        external_pipelines(start_id, stop_id)
+          .update_all(source: Migratable::Pipeline.sources[:external])
+      end
+
+      private
+
+      def external_pipelines(start_id, stop_id)
+        Migratable::Pipeline.where(id: (start_id..stop_id))
+          .where(
+            'EXISTS (?) AND NOT EXISTS (?)',
+            Migratable::CommitStatus.of_type('GenericCommitStatus').has_pipeline.select(1),
+            Migratable::CommitStatus.of_type('Ci::Build').has_pipeline.select(1)
+          )
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/background_migration/populate_external_pipeline_source_spec.rb b/spec/lib/gitlab/background_migration/populate_external_pipeline_source_spec.rb
new file mode 100644
index 00000000000..c7b272cd6ca
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/populate_external_pipeline_source_spec.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::PopulateExternalPipelineSource, :migration, schema: 20180916011959 do
+  let(:migration) { described_class.new }
+
+  let!(:internal_pipeline) { create(:ci_pipeline, source: :web) }
+  let(:pipelines) { [internal_pipeline, unknown_pipeline].map(&:id) }
+
+  let!(:unknown_pipeline) do
+    build(:ci_pipeline, source: :unknown)
+      .tap { |pipeline| pipeline.save(validate: false) }
+  end
+
+  subject { migration.perform(pipelines.min, pipelines.max) }
+
+  shared_examples 'no changes' do
+    it 'does not change the pipeline source' do
+      expect { subject }.not_to change { unknown_pipeline.reload.source }
+    end
+  end
+
+  context 'when unknown pipeline is external' do
+    before do
+      create(:generic_commit_status, pipeline: unknown_pipeline)
+    end
+
+    it 'populates the pipeline source' do
+      subject
+
+      expect(unknown_pipeline.reload.source).to eq('external')
+    end
+
+    it 'can be repeated without effect' do
+      subject
+
+      expect { subject }.not_to change { unknown_pipeline.reload.source }
+    end
+  end
+
+  context 'when unknown pipeline has just a build' do
+    before do
+      create(:ci_build, pipeline: unknown_pipeline)
+    end
+
+    it_behaves_like 'no changes'
+  end
+
+  context 'when unknown pipeline has no statuses' do
+    it_behaves_like 'no changes'
+  end
+
+  context 'when unknown pipeline has a build and a status' do
+    before do
+      create(:generic_commit_status, pipeline: unknown_pipeline)
+      create(:ci_build, pipeline: unknown_pipeline)
+    end
+
+    it_behaves_like 'no changes'
+  end
+end