Merge branch 'mk-10-4-4-fix-no-untracked-upload-files-error' into '10-4-stable-patch-4'10-4-stable-patch-4

[10.4.4 port] Resolve "PrepareUntrackedUploads PostgreSQL syntax error"

See merge request gitlab-org/gitlab-ce!17130

5 files changed, 112 insertions, 46 deletions

diff --git a/changelogs/unreleased/mk-fix-no-untracked-upload-files-error.yml b/changelogs/unreleased/mk-fix-no-untracked-upload-files-error.yml
new file mode 100644
index 00000000000..fddfba94192
--- /dev/null
+++ b/changelogs/unreleased/mk-fix-no-untracked-upload-files-error.yml
@@ -0,0 +1,5 @@
+---
+title: Resolve PrepareUntrackedUploads PostgreSQL syntax error
+merge_request: 17019
+author:
+type: fixed
diff --git a/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb
new file mode 100644
index 00000000000..e46e793d9d2
--- /dev/null
+++ b/db/migrate/20180208183958_schedule_populate_untracked_uploads_if_needed.rb
@@ -0,0 +1,47 @@
+# See http://doc.gitlab.com/ce/development/migration_style_guide.html
+# for more information on how to write migrations for GitLab.
+
+class SchedulePopulateUntrackedUploadsIfNeeded < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze
+
+  class UntrackedFile < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'untracked_files_for_uploads'
+  end
+
+  def up
+    if table_exists?(:untracked_files_for_uploads)
+      process_or_remove_table
+    end
+  end
+
+  def down
+    # nothing
+  end
+
+  private
+
+  def process_or_remove_table
+    if UntrackedFile.all.empty?
+      drop_temp_table
+    else
+      schedule_populate_untracked_uploads_jobs
+    end
+  end
+
+  def drop_temp_table
+    drop_table(:untracked_files_for_uploads, if_exists: true)
+  end
+
+  def schedule_populate_untracked_uploads_jobs
+    say "Scheduling #{FOLLOW_UP_MIGRATION} background migration jobs since there are rows in untracked_files_for_uploads."
+
+    bulk_queue_background_migration_jobs_by_range(
+      UntrackedFile, FOLLOW_UP_MIGRATION)
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f0d94974f41..ac59992fba1 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20180202111106) do
+ActiveRecord::Schema.define(version: 20180208183958) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
index 476c46341ae..9fc46f57319 100644
--- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb
+++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb
@@ -39,7 +39,11 @@ module Gitlab
 
         store_untracked_file_paths
 
-        schedule_populate_untracked_uploads_jobs
+        if UntrackedFile.all.empty?
+          drop_temp_table
+        else
+          schedule_populate_untracked_uploads_jobs
+        end
       end
 
       private
@@ -88,7 +92,7 @@ module Gitlab
           end
         end
 
-        yield(paths)
+        yield(paths) if paths.any?
       end
 
       def build_find_command(search_dir)
@@ -158,6 +162,11 @@ module Gitlab
         bulk_queue_background_migration_jobs_by_range(
           UntrackedFile, FOLLOW_UP_MIGRATION)
       end
+
+      def drop_temp_table
+        UntrackedFile.connection.drop_table(:untracked_files_for_uploads,
+                                            if_exists: true)
+      end
     end
   end
 end
diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb
index 8bb9ebe0419..b21197144a9 100644
--- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb
+++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb
@@ -8,8 +8,6 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
 
   before do
     DatabaseCleaner.clean
-
-    drop_temp_table_if_exists
   end
 
   after do
@@ -23,25 +21,25 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
     end
   end
 
-  it 'ensures the untracked_files_for_uploads table exists' do
-    expect do
-      described_class.new.perform
-    end.to change { ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) }.from(false).to(true)
-  end
+  # E.g. The installation is in use at the time of migration, and someone has
+  # just uploaded a file
+  shared_examples 'does not add files in /uploads/tmp' do
+    let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') }
 
-  it 'has a path field long enough for really long paths' do
-    described_class.new.perform
+    before do
+      FileUtils.mkdir_p(File.dirname(tmp_file))
+      FileUtils.touch(tmp_file)
+    end
 
-    component = 'a' * 255
+    after do
+      FileUtils.rm(tmp_file)
+    end
 
-    long_path = [
-      'uploads',
-      component, # project.full_path
-      component  # filename
-    ].flatten.join('/')
+    it 'does not add files from /uploads/tmp' do
+      described_class.new.perform
 
-    record = untracked_files_for_uploads.create!(path: long_path)
-    expect(record.reload.path.size).to eq(519)
+      expect(untracked_files_for_uploads.count).to eq(5)
+    end
   end
 
   context "test bulk insert with ON CONFLICT DO NOTHING or IGNORE" do
@@ -69,6 +67,21 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
         UploadService.new(project2, uploaded_file, FileUploader).execute
       end
 
+      it 'has a path field long enough for really long paths' do
+        described_class.new.perform
+
+        component = 'a' * 255
+
+        long_path = [
+          'uploads',
+          component, # project.full_path
+          component  # filename
+        ].flatten.join('/')
+
+        record = untracked_files_for_uploads.create!(path: long_path)
+        expect(record.reload.path.size).to eq(519)
+      end
+
       it 'adds unhashed files to the untracked_files_for_uploads table' do
         described_class.new.perform
 
@@ -109,21 +122,17 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
         end
       end
 
-      # E.g. The installation is in use at the time of migration, and someone has
-      # just uploaded a file
       context 'when there are files in /uploads/tmp' do
-        let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') }
-
-        before do
-          FileUtils.touch(tmp_file)
-        end
+        it_behaves_like 'does not add files in /uploads/tmp'
+      end
 
-        after do
-          FileUtils.rm(tmp_file)
-        end
+      context 'when the last batch size exactly matches the max batch size' do
+        it 'does not raise error' do
+          stub_const("#{described_class}::FIND_BATCH_SIZE", 5)
 
-        it 'does not add files from /uploads/tmp' do
-          described_class.new.perform
+          expect do
+            described_class.new.perform
+          end.not_to raise_error
 
           expect(untracked_files_for_uploads.count).to eq(5)
         end
@@ -197,21 +206,17 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
         end
       end
 
-      # E.g. The installation is in use at the time of migration, and someone has
-      # just uploaded a file
       context 'when there are files in /uploads/tmp' do
-        let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') }
-
-        before do
-          FileUtils.touch(tmp_file)
-        end
+        it_behaves_like 'does not add files in /uploads/tmp'
+      end
 
-        after do
-          FileUtils.rm(tmp_file)
-        end
+      context 'when the last batch size exactly matches the max batch size' do
+        it 'does not raise error' do
+          stub_const("#{described_class}::FIND_BATCH_SIZE", 5)
 
-        it 'does not add files from /uploads/tmp' do
-          described_class.new.perform
+          expect do
+            described_class.new.perform
+          end.not_to raise_error
 
           expect(untracked_files_for_uploads.count).to eq(5)
         end
@@ -222,10 +227,10 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do
   # Very new or lightly-used installations that are running this migration
   # may not have an upload directory because they have no uploads.
   context 'when no files were ever uploaded' do
-    it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do
+    it 'deletes the `untracked_files_for_uploads` table (and does not raise error)' do
       described_class.new.perform
 
-      expect(untracked_files_for_uploads.count).to eq(0)
+      expect(untracked_files_for_uploads.connection.table_exists?(:untracked_files_for_uploads)).to be_falsey
     end
   end
 end