From d6435b68c4fc4e0325ec6a3deb807d0e3dd4dec4 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 6 Nov 2017 13:44:30 -0800 Subject: Add TrackUntrackedUploads post-deploy migration To create the table, and schedule the background migration that begins the work. --- .../20171103140253_track_untracked_uploads.rb | 37 +++++++ db/schema.rb | 10 ++ .../prepare_unhashed_uploads.rb | 35 +++++++ spec/migrations/track_untracked_uploads_spec.rb | 113 +++++++++++++++++++++ 4 files changed, 195 insertions(+) create mode 100644 db/post_migrate/20171103140253_track_untracked_uploads.rb create mode 100644 lib/gitlab/background_migration/prepare_unhashed_uploads.rb create mode 100644 spec/migrations/track_untracked_uploads_spec.rb diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb new file mode 100644 index 00000000000..90d530d4011 --- /dev/null +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -0,0 +1,37 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class TrackUntrackedUploads < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + DOWNTIME = false + MIGRATION = 'PrepareUnhashedUploads' + + def up + unless table_exists?(:unhashed_upload_files) + create_table :unhashed_upload_files do |t| + t.string :path, null: false + t.boolean :tracked, default: false, null: false + t.timestamps_with_timezone null: false + end + end + + unless index_exists?(:unhashed_upload_files, :path) + add_index :unhashed_upload_files, :path, unique: true + end + + unless index_exists?(:unhashed_upload_files, :tracked) + add_index :unhashed_upload_files, :tracked + end + + BackgroundMigrationWorker.perform_async(MIGRATION) + end + + def down + if table_exists?(:unhashed_upload_files) + drop_table :unhashed_upload_files + end + end +end diff --git a/db/schema.rb b/db/schema.rb index effb2604af2..2b7e12b45f1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1719,6 +1719,16 @@ ActiveRecord::Schema.define(version: 20171124150326) do add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree + create_table "unhashed_upload_files", force: :cascade do |t| + t.string "path", null: false + t.boolean "tracked", default: false, null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + end + + add_index "unhashed_upload_files", ["path"], name: "index_unhashed_upload_files_on_path", unique: true, using: :btree + add_index "unhashed_upload_files", ["tracked"], name: "index_unhashed_upload_files_on_tracked", using: :btree + create_table "uploads", force: :cascade do |t| t.integer "size", limit: 8, null: false t.string "path", null: false diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb new file mode 100644 index 00000000000..3efc604fa86 --- /dev/null +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -0,0 +1,35 @@ +module Gitlab + module BackgroundMigration + class PrepareUnhashedUploads + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + end + + def perform + return unless migrate? + + clear_unhashed_upload_files + store_unhashed_upload_files + schedule_populate_untracked_uploads_jobs + end + + private + + def migrate? + UnhashedUploadFile.table_exists? + end + + def clear_unhashed_upload_files + # TODO + end + + def store_unhashed_upload_files + # TODO + end + + def schedule_populate_untracked_uploads_jobs + # TODO + end + end + end +end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb new file mode 100644 index 00000000000..5c1113a5e47 --- /dev/null +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -0,0 +1,113 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') + +describe TrackUntrackedUploads, :migration, :sidekiq do + matcher :be_scheduled_migration do + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration] + end + end + + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" + end + end + + it 'correctly schedules the follow-up background migration' do + Sidekiq::Testing.fake! do + migrate! + + expect(described_class::MIGRATION).to be_scheduled_migration + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end + end + + it 'ensures the unhashed_upload_files table exists' do + expect do + migrate! + end.to change { table_exists?(:unhashed_upload_files) }.from(false).to(true) + end + + it 'has a path field long enough for really long paths' do + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + end + + migrate! + + max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255 + long_path = "./uploads#{("/#{max_length_namespace_path}") * Namespace::NUMBER_OF_ANCESTORS_ALLOWED}/#{max_length_project_path}/#{max_length_filename}" + unhashed_upload_file = UnhashedUploadFile.new(path: long_path) + unhashed_upload_file.save! + expect(UnhashedUploadFile.first.path.size).to eq(5641) + end + + context 'with tracked and untracked uploads' do + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(:project1) { create(:project) } + let(:project2) { create(:project) } + let(:appearance) { create(:appearance) } + + before do + fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + + # Tracked, by doing normal file upload + uploaded_file = fixture_file_upload(fixture) + user1.update(avatar: uploaded_file) + project1.update(avatar: uploaded_file) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + appearance.update(logo: uploaded_file) + + # Untracked, by doing normal file upload then deleting records from DB + uploaded_file = fixture_file_upload(fixture) + user2.update(avatar: uploaded_file) + user2.uploads.delete_all + project2.update(avatar: uploaded_file) + UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + project2.uploads.delete_all + appearance.update(header_logo: uploaded_file) + appearance.uploads.last.destroy + end + + it 'schedules background migrations' do + Sidekiq::Testing.inline! do + migrate! + + # Tracked uploads still exist + expect(user1.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/1/rails_sample.jpg", + "uploader" => "AvatarUploader" + }) + expect(project1.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/1/rails_sample.jpg", + "uploader" => "AvatarUploader" + }) + expect(appearance.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/logo/1/rails_sample.jpg", + "uploader" => "AttachmentUploader" + }) + expect(project1.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) + expect(project1.uploads.last.uploader).to eq('FileUploader') + + # Untracked uploads are now tracked + expect(user2.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/2/rails_sample.jpg", + "uploader" => "AvatarUploader" + }) + expect(project2.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/2/rails_sample.jpg", + "uploader" => "AvatarUploader" + }) + expect(appearance.uploads.count).to eq(2) + expect(appearance.uploads.last.attributes).to include({ + "path" => "uploads/-/system/appearance/header_logo/1/rails_sample.jpg", + "uploader" => "AttachmentUploader" + }) + expect(project2.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) + expect(project2.uploads.last.uploader).to eq('FileUploader') + end + end + end +end -- cgit v1.2.1 From ab814e4dd3cf06559a95ca5dd19722431314f6fa Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 12:53:10 -0800 Subject: Backport `which` from EE --- lib/gitlab/utils.rb | 17 +++++++++++++++++ spec/lib/gitlab/utils_spec.rb | 10 +++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index abb3d3a02c3..b3baaf036d8 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -46,5 +46,22 @@ module Gitlab def random_string Random.rand(Float::MAX.to_i).to_s(36) end + + # See: http://stackoverflow.com/questions/2108727/which-in-ruby-checking-if-program-exists-in-path-from-ruby + # Cross-platform way of finding an executable in the $PATH. + # + # which('ruby') #=> /usr/bin/ruby + def which(cmd, env = ENV) + exts = env['PATHEXT'] ? env['PATHEXT'].split(';') : [''] + + env['PATH'].split(File::PATH_SEPARATOR).each do |path| + exts.each do |ext| + exe = File.join(path, "#{cmd}#{ext}") + return exe if File.executable?(exe) && !File.directory?(exe) + end + end + + nil + end end end diff --git a/spec/lib/gitlab/utils_spec.rb b/spec/lib/gitlab/utils_spec.rb index 3137a72fdc4..e872a5290c5 100644 --- a/spec/lib/gitlab/utils_spec.rb +++ b/spec/lib/gitlab/utils_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::Utils do - delegate :to_boolean, :boolean_to_yes_no, :slugify, :random_string, to: :described_class + delegate :to_boolean, :boolean_to_yes_no, :slugify, :random_string, :which, to: :described_class describe '.slugify' do { @@ -59,4 +59,12 @@ describe Gitlab::Utils do expect(random_string).to be_kind_of(String) end end + + describe '.which' do + it 'finds the full path to an executable binary' do + expect(File).to receive(:executable?).with('/bin/sh').and_return(true) + + expect(which('sh', 'PATH' => '/bin')).to eq('/bin/sh') + end + end end -- cgit v1.2.1 From b6ea41d13073ce8b4d16b2edb602c82aae10ea0b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 6 Nov 2017 17:07:35 -0800 Subject: Find and store unhashed upload file paths --- .../prepare_unhashed_uploads.rb | 58 ++++++++++++++-- .../prepare_unhashed_uploads_spec.rb | 80 ++++++++++++++++++++++ 2 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 3efc604fa86..11f43044829 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -1,6 +1,9 @@ module Gitlab module BackgroundMigration class PrepareUnhashedUploads + FILE_PATH_BATCH_SIZE = 500 + UPLOAD_DIR = "#{CarrierWave.root}/uploads" + class UnhashedUploadFile < ActiveRecord::Base self.table_name = 'unhashed_upload_files' end @@ -8,8 +11,8 @@ module Gitlab def perform return unless migrate? - clear_unhashed_upload_files - store_unhashed_upload_files + clear_unhashed_upload_file_paths + store_unhashed_upload_file_paths schedule_populate_untracked_uploads_jobs end @@ -19,12 +22,55 @@ module Gitlab UnhashedUploadFile.table_exists? end - def clear_unhashed_upload_files - # TODO + def clear_unhashed_upload_file_paths + UnhashedUploadFile.delete_all end - def store_unhashed_upload_files - # TODO + def store_unhashed_upload_file_paths + return unless Dir.exists?(UPLOAD_DIR) + + file_paths = [] + each_file_path(UPLOAD_DIR) do |file_path| + file_paths << file_path + + if file_paths.size >= FILE_PATH_BATCH_SIZE + insert_file_paths(file_paths) + file_paths = [] + end + end + + insert_file_paths(file_paths) if file_paths.any? + end + + def each_file_path(search_dir, &block) + cmd = build_find_command(search_dir) + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + stdout.each_line("\0") do |line| + yield(line.chomp("\0")) + end + raise "Find command failed" unless status_thread.value.success? + end + end + + def build_find_command(search_dir) + cmd = ['find', search_dir, '-type', 'f', '!', '-path', "#{UPLOAD_DIR}/@hashed/*", '!', '-path', "#{UPLOAD_DIR}/tmp/*", '-print0'] + + ['ionice', '-c', 'Idle'] + cmd if ionice_is_available? + + cmd + end + + def ionice_is_available? + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, even though it isn't very nice + false + end + + def insert_file_paths(file_paths) + file_paths.each do |file_path| + UnhashedUploadFile.create!(path: file_path) + end end def schedule_populate_untracked_uploads_jobs diff --git a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb new file mode 100644 index 00000000000..2f641a5deed --- /dev/null +++ b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb @@ -0,0 +1,80 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema: 20171103140253 do + let!(:unhashed_upload_files) { table(:unhashed_upload_files) } + + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(:project1) { create(:project) } + let(:project2) { create(:project) } + let(:appearance) { create(:appearance) } + + context 'when files were uploaded before and after hashed storage was enabled' do + before do + fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + uploaded_file = fixture_file_upload(fixture) + + user1.update(avatar: uploaded_file) + project1.update(avatar: uploaded_file) + appearance.update(logo: uploaded_file, header_logo: uploaded_file) + uploaded_file = fixture_file_upload(fixture) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + + stub_application_setting(hashed_storage_enabled: true) + + # Hashed files + uploaded_file = fixture_file_upload(fixture) + UploadService.new(project2, uploaded_file, FileUploader).execute + end + + it 'adds unhashed files to the unhashed_upload_files table' do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end + + it 'does not add hashed files to the unhashed_upload_files table' do + described_class.new.perform + + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end + + # E.g. from a previous failed run of this background migration + context 'when there is existing data in unhashed_upload_files' do + before do + unhashed_upload_files.create(path: '/foo/bar.jpg') + end + + it 'clears existing data before adding new data' do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(1).to(5) + end + end + + # E.g. The installation is in use at the time of migration, and someone has + # just uploaded a file + context 'when there are files in /uploads/tmp' do + before do + FileUtils.touch(Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg')) + end + + it 'does not add files from /uploads/tmp' do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end + end + end + + # Very new or lightly-used installations that are running this migration + # may not have an upload directory because they have no uploads. + context 'when no files were ever uploaded' do + it 'does not add to the unhashed_upload_files table (and does not raise error)' do + expect do + described_class.new.perform + end.not_to change { unhashed_upload_files.count }.from(0) + end + end +end -- cgit v1.2.1 From 8315c66a569bbc1b4806762e4da49c22813fc523 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 12:53:24 -0800 Subject: Kick off follow up background migration jobs To process the unhashed_upload_files table. --- .../populate_untracked_uploads.rb | 51 ++++++++++++++++++ .../prepare_unhashed_uploads.rb | 8 ++- .../prepare_unhashed_uploads_spec.rb | 63 ++++++++++++++++------ 3 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 lib/gitlab/background_migration/populate_untracked_uploads.rb diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb new file mode 100644 index 00000000000..6dbef41cff8 --- /dev/null +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -0,0 +1,51 @@ +module Gitlab + module BackgroundMigration + class PopulateUntrackedUploads + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + + scope :untracked, -> { where(tracked: false) } + + def ensure_tracked! + # TODO + # unless unhashed_upload_file.in_uploads? + # unhashed_upload_file.add_to_uploads + # end + # + # unhashed_upload_file.mark_as_tracked + end + + def model_id + # TODO + end + + def model_type + # TODO + end + + def uploader + # TODO + end + end + + class Upload < ActiveRecord::Base + self.table_name = 'uploads' + end + + def perform(start_id, end_id) + return unless migrate? + + files = UnhashedUploadFile.untracked.where(id: start_id..end_id) + files.each do |unhashed_upload_file| + unhashed_upload_file.ensure_tracked! + end + end + + private + + def migrate? + UnhashedUploadFile.table_exists? && Upload.table_exists? + end + end + end +end diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 11f43044829..556457039fa 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -1,10 +1,16 @@ module Gitlab module BackgroundMigration class PrepareUnhashedUploads + # For bulk_queue_background_migration_jobs_by_range + include Database::MigrationHelpers + FILE_PATH_BATCH_SIZE = 500 UPLOAD_DIR = "#{CarrierWave.root}/uploads" + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads' class UnhashedUploadFile < ActiveRecord::Base + include EachBatch + self.table_name = 'unhashed_upload_files' end @@ -74,7 +80,7 @@ module Gitlab end def schedule_populate_untracked_uploads_jobs - # TODO + bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) end end end diff --git a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb index 2f641a5deed..76d126e8f00 100644 --- a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidekiq, schema: 20171103140253 do let!(:unhashed_upload_files) { table(:unhashed_upload_files) } let(:user1) { create(:user) } @@ -9,6 +9,18 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema let(:project2) { create(:project) } let(:appearance) { create(:appearance) } + matcher :be_scheduled_migration do |*expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration, expected] + end + end + + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" + end + end + context 'when files were uploaded before and after hashed storage was enabled' do before do fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') @@ -28,16 +40,29 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema end it 'adds unhashed files to the unhashed_upload_files table' do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end end it 'does not add hashed files to the unhashed_upload_files table' do - described_class.new.perform + Sidekiq::Testing.fake! do + described_class.new.perform - hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path - expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end + end + + it 'correctly schedules the follow-up background migration jobs' do + Sidekiq::Testing.fake! do + described_class.new.perform + + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end end # E.g. from a previous failed run of this background migration @@ -47,9 +72,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema end it 'clears existing data before adding new data' do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(1).to(5) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(1).to(5) + end end end @@ -61,9 +88,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema end it 'does not add files from /uploads/tmp' do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { unhashed_upload_files.count }.from(0).to(5) + end end end end @@ -72,9 +101,11 @@ describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, schema # may not have an upload directory because they have no uploads. context 'when no files were ever uploaded' do it 'does not add to the unhashed_upload_files table (and does not raise error)' do - expect do - described_class.new.perform - end.not_to change { unhashed_upload_files.count }.from(0) + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.not_to change { unhashed_upload_files.count }.from(0) + end end end end -- cgit v1.2.1 From 3a0ad99d59506592e8d5c6abf0de0fc2104f0bf2 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 19:08:02 -0800 Subject: Add untracked files to uploads --- .../populate_untracked_uploads.rb | 144 ++++- .../populate_untracked_uploads_spec.rb | 581 +++++++++++++++++++++ spec/migrations/track_untracked_uploads_spec.rb | 61 ++- 3 files changed, 752 insertions(+), 34 deletions(-) create mode 100644 spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 6dbef41cff8..acd424f4558 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -4,27 +4,149 @@ module Gitlab class UnhashedUploadFile < ActiveRecord::Base self.table_name = 'unhashed_upload_files' + # Ends with /:random_hex/:filename + FILE_UPLOADER_PATH_PATTERN = /\/\h+\/[^\/]+\z/ + + # These regex patterns are tested against a relative path, relative to + # the upload directory. + # For convenience, if there exists a capture group in the pattern, then + # it indicates the model_id. + PATH_PATTERNS = [ + { + pattern: /\A-\/system\/appearance\/logo\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Appearance', + }, + { + pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Appearance', + }, + { + pattern: /\A-\/system\/note\/attachment\/(\d+)/, + uploader: 'AttachmentUploader', + model_type: 'Note', + }, + { + pattern: /\A-\/system\/user\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'User', + }, + { + pattern: /\A-\/system\/group\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'Group', + }, + { + pattern: /\A-\/system\/project\/avatar\/(\d+)/, + uploader: 'AvatarUploader', + model_type: 'Project', + }, + { + pattern: FILE_UPLOADER_PATH_PATTERN, + uploader: 'FileUploader', + model_type: 'Project' + }, + ] + scope :untracked, -> { where(tracked: false) } def ensure_tracked! - # TODO - # unless unhashed_upload_file.in_uploads? - # unhashed_upload_file.add_to_uploads - # end - # - # unhashed_upload_file.mark_as_tracked + return if persisted? && tracked? + + unless in_uploads? + add_to_uploads + end + + mark_as_tracked end - def model_id - # TODO + def in_uploads? + # Even though we are checking relative paths, path is enough to + # uniquely identify uploads. There is no ambiguity between + # FileUploader paths and other Uploader paths because we use the /-/ + # separator kind of like an escape character. Project full_path will + # never conflict with an upload path starting with "uploads/-/". + Upload.exists?(path: upload_path) end - def model_type - # TODO + def add_to_uploads + Upload.create!( + path: upload_path, + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + ) + end + + def mark_as_tracked + self.tracked = true + self.save! + end + + def upload_path + # UnhashedUploadFile#path is absolute, but Upload#path depends on uploader + if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) + matchd[0].sub(/\A\//, '') # remove leading slash + else + path_relative_to_carrierwave_root + end end def uploader - # TODO + PATH_PATTERNS.each do |path_pattern_map| + if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + return path_pattern_map[:uploader] + end + end + end + + def model_type + PATH_PATTERNS.each do |path_pattern_map| + if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + return path_pattern_map[:model_type] + end + end + end + + def model_id + PATH_PATTERNS.each do |path_pattern_map| + matchd = path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + + # If something is captured (matchd[1] is not nil), it is a model_id + return matchd[1] if matchd && matchd[1] + end + + # Only the FileUploader pattern will not match an ID + file_uploader_model_id + end + + def file_size + File.size(path) + end + + # Not including a leading slash + def path_relative_to_upload_dir + @path_relative_to_upload_dir ||= path.sub(/\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}\//, '') + end + + # Not including a leading slash + def path_relative_to_carrierwave_root + "uploads/#{path_relative_to_upload_dir}" + end + + private + + def file_uploader_model_id + pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ + matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) + raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + full_path = matchd[1] + project = Project.find_by_full_path(full_path) + project.id.to_s end end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb new file mode 100644 index 00000000000..ae6a712f2ee --- /dev/null +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -0,0 +1,581 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do + let!(:unhashed_upload_files) { table(:unhashed_upload_files) } + let!(:uploads) { table(:uploads) } + + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(:project1) { create(:project) } + let(:project2) { create(:project) } + let(:appearance) { create(:appearance) } + + context 'with untracked files and tracked files in unhashed_upload_files' do + before do + fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + + # Tracked, by doing normal file upload + uploaded_file = fixture_file_upload(fixture) + user1.update!(avatar: uploaded_file) + project1.update!(avatar: uploaded_file) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + appearance.update!(logo: uploaded_file) + + # Untracked, by doing normal file upload then later deleting records from DB + uploaded_file = fixture_file_upload(fixture) + user2.update!(avatar: uploaded_file) + project2.update!(avatar: uploaded_file) + UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + appearance.update!(header_logo: uploaded_file) + + # Unhashed upload files created by PrepareUnhashedUploads + unhashed_upload_files.create!(path: appearance.logo.file.file) + unhashed_upload_files.create!(path: appearance.header_logo.file.file) + unhashed_upload_files.create!(path: user1.avatar.file.file) + unhashed_upload_files.create!(path: user2.avatar.file.file) + unhashed_upload_files.create!(path: project1.avatar.file.file) + unhashed_upload_files.create!(path: project2.avatar.file.file) + unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") + unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") + + user2.uploads.delete_all + project2.uploads.delete_all + appearance.uploads.last.destroy + end + + it 'adds untracked files to the uploads table' do + expect do + described_class.new.perform(1, 1000) + end.to change { uploads.count }.from(4).to(8) + + expect(user2.uploads.count).to eq(1) + expect(project2.uploads.count).to eq(2) + expect(appearance.uploads.count).to eq(2) + end + + it 'does not create duplicate uploads of already tracked files' do + described_class.new.perform(1, 1000) + + expect(user1.uploads.count).to eq(1) + expect(project1.uploads.count).to eq(2) + expect(appearance.uploads.count).to eq(2) + end + end + + context 'with no untracked files' do + it 'does not add to the uploads table (and does not raise error)' do + expect do + described_class.new.perform(1, 1000) + end.not_to change { uploads.count }.from(0) + end + end +end + +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFile do + let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } + + describe '#ensure_tracked!' do + let(:user1) { create(:user) } + + context 'when the file is already in the uploads table' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") } + + before do + upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234) + end + + it 'does not add an upload' do + expect do + unhashed_upload_file.ensure_tracked! + end.to_not change { upload_class.count }.from(1) + end + end + end + + describe '#add_to_uploads' do + let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } + let(:uploaded_file) { fixture_file_upload(fixture) } + + context 'for an appearance logo file path' do + let(:appearance) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + + before do + appearance.update!(logo: uploaded_file) + appearance.uploads.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => appearance.id, + "model_type" => "Appearance", + "uploader" => "AttachmentUploader" + }) + end + end + + context 'for an appearance header_logo file path' do + let(:appearance) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: appearance.header_logo.file.file) } + + before do + appearance.update!(header_logo: uploaded_file) + appearance.uploads.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => appearance.id, + "model_type" => "Appearance", + "uploader" => "AttachmentUploader" + }) + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:note) { create(:note) } + let(:unhashed_upload_file) { described_class.create!(path: note.attachment.file.file) } + + before do + note.update!(attachment: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/note/attachment/#{note.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => note.id, + "model_type" => "Note", + "uploader" => "AttachmentUploader" + }) + end + end + + context 'for a user avatar file path' do + let(:user) { create(:user) } + let(:unhashed_upload_file) { described_class.create!(path: user.avatar.file.file) } + + before do + user.update!(avatar: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/user/avatar/#{user.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => user.id, + "model_type" => "User", + "uploader" => "AvatarUploader" + }) + end + end + + context 'for a group avatar file path' do + let(:group) { create(:group) } + let(:unhashed_upload_file) { described_class.create!(path: group.avatar.file.file) } + + before do + group.update!(avatar: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/group/avatar/#{group.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => group.id, + "model_type" => "Group", + "uploader" => "AvatarUploader" + }) + end + end + + context 'for a project avatar file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + + before do + project.update!(avatar: uploaded_file) + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "uploads/-/system/project/avatar/#{project.id}/rails_sample.jpg", + "checksum" => nil, + "model_id" => project.id, + "model_type" => "Project", + "uploader" => "AvatarUploader" + }) + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + + before do + UploadService.new(project, uploaded_file, FileUploader).execute # Markdown upload + unhashed_upload_file.save! + upload_class.delete_all + end + + it 'creates an Upload record' do + expect do + unhashed_upload_file.add_to_uploads + end.to change { upload_class.count }.from(0).to(1) + + random_hex = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] + expect(upload_class.first.attributes).to include({ + "size" => 35255, + "path" => "#{random_hex}/rails_sample.jpg", + "checksum" => nil, + "model_id" => project.id, + "model_type" => "Project", + "uploader" => "FileUploader" + }) + end + end + end + + describe '#mark_as_tracked' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'saves the record with tracked set to true' do + expect do + unhashed_upload_file.mark_as_tracked + end.to change { unhashed_upload_file.tracked }.from(false).to(true) + + expect(unhashed_upload_file.persisted?).to be_truthy + end + end + + describe '#upload_path' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns the file path relative to the CarrierWave root' do + expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:random_hex) { SecureRandom.hex } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } + + it 'returns the file path relative to the project directory in uploads' do + expect(unhashed_upload_file.upload_path).to eq("#{random_hex}/Some file.jpg") + end + end + end + + describe '#uploader' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns AttachmentUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns AttachmentUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns AttachmentUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns AvatarUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns AvatarUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns AvatarUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + + it 'returns FileUploader as a string' do + expect(unhashed_upload_file.uploader).to eq('FileUploader') + end + end + end + + describe '#model_type' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns Appearance as a string' do + expect(unhashed_upload_file.model_type).to eq('Appearance') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns Appearance as a string' do + expect(unhashed_upload_file.model_type).to eq('Appearance') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns Note as a string' do + expect(unhashed_upload_file.model_type).to eq('Note') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns User as a string' do + expect(unhashed_upload_file.model_type).to eq('User') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns Group as a string' do + expect(unhashed_upload_file.model_type).to eq('Group') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns Project as a string' do + expect(unhashed_upload_file.model_type).to eq('Project') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + + it 'returns Project as a string' do + expect(unhashed_upload_file.model_type).to eq('Project') + end + end + end + + describe '#model_id' do + context 'for an appearance logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1') + end + end + + context 'for an appearance header_logo file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1') + end + end + + context 'for a pre-Markdown Note attachment file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a user avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a group avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a project avatar file path' do + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq('1234') + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + + it 'returns the ID as a string' do + expect(unhashed_upload_file.model_id).to eq(project.id.to_s) + end + end + end + + describe '#file_size' do + let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } + let(:uploaded_file) { fixture_file_upload(fixture) } + + context 'for an appearance logo file path' do + let(:appearance) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + + before do + appearance.update!(logo: uploaded_file) + end + + it 'returns the file size' do + expect(unhashed_upload_file.file_size).to eq(35255) + end + + it 'returns the same thing that CarrierWave would return' do + expect(unhashed_upload_file.file_size).to eq(appearance.logo.size) + end + end + + context 'for a project avatar file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + + before do + project.update!(avatar: uploaded_file) + end + + it 'returns the file size' do + expect(unhashed_upload_file.file_size).to eq(35255) + end + + it 'returns the same thing that CarrierWave would return' do + expect(unhashed_upload_file.file_size).to eq(project.avatar.size) + end + end + + context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do + let(:project) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + + before do + UploadService.new(project, uploaded_file, FileUploader).execute + end + + it 'returns the file size' do + expect(unhashed_upload_file.file_size).to eq(35255) + end + + it 'returns the same thing that CarrierWave would return' do + expect(unhashed_upload_file.file_size).to eq(project.uploads.first.size) + end + end + end +end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 5c1113a5e47..8db41786397 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -2,6 +2,10 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') describe TrackUntrackedUploads, :migration, :sidekiq do + class UnhashedUploadFile < ActiveRecord::Base + self.table_name = 'unhashed_upload_files' + end + matcher :be_scheduled_migration do match do |migration| BackgroundMigrationWorker.jobs.any? do |job| @@ -30,10 +34,6 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end it 'has a path field long enough for really long paths' do - class UnhashedUploadFile < ActiveRecord::Base - self.table_name = 'unhashed_upload_files' - end - migrate! max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255 @@ -57,7 +57,8 @@ describe TrackUntrackedUploads, :migration, :sidekiq do uploaded_file = fixture_file_upload(fixture) user1.update(avatar: uploaded_file) project1.update(avatar: uploaded_file) - UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + upload_result = UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + @project1_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') appearance.update(logo: uploaded_file) # Untracked, by doing normal file upload then deleting records from DB @@ -65,48 +66,62 @@ describe TrackUntrackedUploads, :migration, :sidekiq do user2.update(avatar: uploaded_file) user2.uploads.delete_all project2.update(avatar: uploaded_file) - UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + upload_result = UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + @project2_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') project2.uploads.delete_all appearance.update(header_logo: uploaded_file) appearance.uploads.last.destroy end - it 'schedules background migrations' do + it 'tracks untracked migrations' do Sidekiq::Testing.inline! do migrate! # Tracked uploads still exist - expect(user1.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/1/rails_sample.jpg", + expect(user1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(project1.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/1/rails_sample.jpg", + expect(project1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(appearance.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/logo/1/rails_sample.jpg", + expect(appearance.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" }) - expect(project1.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) - expect(project1.uploads.last.uploader).to eq('FileUploader') + expect(project1.uploads.last.attributes).to include({ + "path" => @project1_markdown_upload_path, + "uploader" => "FileUploader" + }) # Untracked uploads are now tracked - expect(user2.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/2/rails_sample.jpg", + expect(user2.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{user2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(project2.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/2/rails_sample.jpg", + expect(project2.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{project2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" }) - expect(appearance.uploads.count).to eq(2) + expect(appearance.reload.uploads.count).to eq(2) expect(appearance.uploads.last.attributes).to include({ - "path" => "uploads/-/system/appearance/header_logo/1/rails_sample.jpg", + "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" }) - expect(project2.uploads.last.path).to match(/\w+\/rails_sample\.jpg/) - expect(project2.uploads.last.uploader).to eq('FileUploader') + expect(project2.uploads.last.attributes).to include({ + "path" => @project2_markdown_upload_path, + "uploader" => "FileUploader" + }) + end + end + + it 'all UnhashedUploadFile records are marked as tracked' do + Sidekiq::Testing.inline! do + migrate! + + expect(UnhashedUploadFile.count).to eq(8) + expect(UnhashedUploadFile.count).to eq(UnhashedUploadFile.where(tracked: true).count) end end end -- cgit v1.2.1 From 1bae010b63f0bcff79f32ce99190f2d5b6d9fbcd Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 19:54:28 -0800 Subject: Calculate checksums by copy-pasting in the whole `Upload` class. Also, fix `Namespace` `model_type` (it should not be `Group`). --- .../populate_untracked_uploads.rb | 67 ++++++++- .../populate_untracked_uploads_spec.rb | 149 +++++++++------------ spec/migrations/track_untracked_uploads_spec.rb | 64 +++++---- 3 files changed, 170 insertions(+), 110 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index acd424f4558..1773b53bd68 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -35,7 +35,7 @@ module Gitlab { pattern: /\A-\/system\/group\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'Group', + model_type: 'Namespace', }, { pattern: /\A-\/system\/project\/avatar\/(\d+)/, @@ -150,8 +150,71 @@ module Gitlab end end + # Copy-pasted class for less fragile migration class Upload < ActiveRecord::Base - self.table_name = 'uploads' + self.table_name = 'uploads' # This is the only line different from copy-paste + + # Upper limit for foreground checksum processing + CHECKSUM_THRESHOLD = 100.megabytes + + belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations + + validates :size, presence: true + validates :path, presence: true + validates :model, presence: true + validates :uploader, presence: true + + before_save :calculate_checksum, if: :foreground_checksum? + after_commit :schedule_checksum, unless: :foreground_checksum? + + def self.remove_path(path) + where(path: path).destroy_all + end + + def self.record(uploader) + remove_path(uploader.relative_path) + + create( + size: uploader.file.size, + path: uploader.relative_path, + model: uploader.model, + uploader: uploader.class.to_s + ) + end + + def absolute_path + return path unless relative_path? + + uploader_class.absolute_path(self) + end + + def calculate_checksum + return unless exist? + + self.checksum = Digest::SHA256.file(absolute_path).hexdigest + end + + def exist? + File.exist?(absolute_path) + end + + private + + def foreground_checksum? + size <= CHECKSUM_THRESHOLD + end + + def schedule_checksum + UploadChecksumWorker.perform_async(id) + end + + def relative_path? + !path.start_with?('/') + end + + def uploader_class + Object.const_get(uploader) + end end def perform(start_id, end_id) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index ae6a712f2ee..c61a207d012 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -97,61 +97,53 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi let(:uploaded_file) { fixture_file_upload(fixture) } context 'for an appearance logo file path' do - let(:appearance) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + let(:model) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: model.logo.file.file) } before do - appearance.update!(logo: uploaded_file) - appearance.uploads.delete_all + model.update!(logo: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => appearance.id, - "model_type" => "Appearance", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/logo/#{model.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for an appearance header_logo file path' do - let(:appearance) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: appearance.header_logo.file.file) } + let(:model) { create(:appearance) } + let(:unhashed_upload_file) { described_class.create!(path: model.header_logo.file.file) } before do - appearance.update!(header_logo: uploaded_file) - appearance.uploads.delete_all + model.update!(header_logo: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => appearance.id, - "model_type" => "Appearance", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/header_logo/#{model.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a pre-Markdown Note attachment file path' do - let(:note) { create(:note) } - let(:unhashed_upload_file) { described_class.create!(path: note.attachment.file.file) } + let(:model) { create(:note) } + let(:unhashed_upload_file) { described_class.create!(path: model.attachment.file.file) } before do - note.update!(attachment: uploaded_file) + model.update!(attachment: uploaded_file) upload_class.delete_all end @@ -161,115 +153,99 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi end.to change { upload_class.count }.from(0).to(1) expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/note/attachment/#{note.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => note.id, + "path" => "uploads/-/system/note/attachment/#{model.id}/rails_sample.jpg", + "model_id" => model.id, "model_type" => "Note", "uploader" => "AttachmentUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a user avatar file path' do - let(:user) { create(:user) } - let(:unhashed_upload_file) { described_class.create!(path: user.avatar.file.file) } + let(:model) { create(:user) } + let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } before do - user.update!(avatar: uploaded_file) - upload_class.delete_all + model.update!(avatar: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/user/avatar/#{user.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => user.id, - "model_type" => "User", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{model.id}/rails_sample.jpg", "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a group avatar file path' do - let(:group) { create(:group) } - let(:unhashed_upload_file) { described_class.create!(path: group.avatar.file.file) } + let(:model) { create(:group) } + let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } before do - group.update!(avatar: uploaded_file) - upload_class.delete_all + model.update!(avatar: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/group/avatar/#{group.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => group.id, - "model_type" => "Group", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/group/avatar/#{model.id}/rails_sample.jpg", + "model_id" => model.id, + "model_type" => "Namespace", # Explicitly calling this out because it was unexpected to me (I assumed it should be "Group") "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a project avatar file path' do - let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + let(:model) { create(:project) } + let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } before do - project.update!(avatar: uploaded_file) - upload_class.delete_all + model.update!(avatar: uploaded_file) + model.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "uploads/-/system/project/avatar/#{project.id}/rails_sample.jpg", - "checksum" => nil, - "model_id" => project.id, - "model_type" => "Project", + expect(model.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{model.id}/rails_sample.jpg", "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + let(:model) { create(:project) } + let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } before do - UploadService.new(project, uploaded_file, FileUploader).execute # Markdown upload + UploadService.new(model, uploaded_file, FileUploader).execute # Markdown upload unhashed_upload_file.save! - upload_class.delete_all + model.reload.uploads.delete_all end it 'creates an Upload record' do expect do unhashed_upload_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + end.to change { model.reload.uploads.count }.from(0).to(1) - random_hex = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] - expect(upload_class.first.attributes).to include({ - "size" => 35255, - "path" => "#{random_hex}/rails_sample.jpg", - "checksum" => nil, - "model_id" => project.id, - "model_type" => "Project", + hex_secret = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] + expect(model.uploads.first.attributes).to include({ + "path" => "#{hex_secret}/rails_sample.jpg", "uploader" => "FileUploader" - }) + }.merge(rails_sample_jpg_attrs)) end end end @@ -441,8 +417,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a group avatar file path' do let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns Group as a string' do - expect(unhashed_upload_file.model_type).to eq('Group') + it 'returns Namespace as a string' do + expect(unhashed_upload_file.model_type).to eq('Namespace') end end @@ -578,4 +554,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi end end end + + def rails_sample_jpg_attrs + { + "size" => 35255, + "checksum" => 'f2d1fd9d8d8a3368d468fa067888605d74a66f41c16f55979ceaf2af77375844' + } + end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 8db41786397..9539993be31 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -49,6 +49,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do let(:project1) { create(:project) } let(:project2) { create(:project) } let(:appearance) { create(:appearance) } + let(:uploads) { table(:uploads) } before do fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') @@ -73,46 +74,52 @@ describe TrackUntrackedUploads, :migration, :sidekiq do appearance.uploads.last.destroy end - it 'tracks untracked migrations' do + it 'tracks untracked uploads' do Sidekiq::Testing.inline! do - migrate! + expect do + migrate! + end.to change { uploads.count }.from(4).to(8) - # Tracked uploads still exist - expect(user1.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }) - expect(project1.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }) - expect(appearance.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }) - expect(project1.uploads.last.attributes).to include({ - "path" => @project1_markdown_upload_path, - "uploader" => "FileUploader" - }) - - # Untracked uploads are now tracked expect(user2.reload.uploads.first.attributes).to include({ "path" => "uploads/-/system/user/avatar/#{user2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) expect(project2.reload.uploads.first.attributes).to include({ "path" => "uploads/-/system/project/avatar/#{project2.id}/rails_sample.jpg", "uploader" => "AvatarUploader" - }) + }.merge(rails_sample_jpg_attrs)) expect(appearance.reload.uploads.count).to eq(2) expect(appearance.uploads.last.attributes).to include({ "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", "uploader" => "AttachmentUploader" - }) + }.merge(rails_sample_jpg_attrs)) expect(project2.uploads.last.attributes).to include({ "path" => @project2_markdown_upload_path, "uploader" => "FileUploader" - }) + }.merge(rails_sample_jpg_attrs)) + end + end + + it 'ignores already-tracked uploads' do + Sidekiq::Testing.inline! do + migrate! + + expect(user1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", + "uploader" => "AvatarUploader", + }.merge(rails_sample_jpg_attrs)) + expect(project1.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", + "uploader" => "AvatarUploader" + }.merge(rails_sample_jpg_attrs)) + expect(appearance.reload.uploads.first.attributes).to include({ + "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", + "uploader" => "AttachmentUploader" + }.merge(rails_sample_jpg_attrs)) + expect(project1.uploads.last.attributes).to include({ + "path" => @project1_markdown_upload_path, + "uploader" => "FileUploader" + }.merge(rails_sample_jpg_attrs)) end end @@ -125,4 +132,11 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end end + + def rails_sample_jpg_attrs + { + "size" => 35255, + "checksum" => 'f2d1fd9d8d8a3368d468fa067888605d74a66f41c16f55979ceaf2af77375844' + } + end end -- cgit v1.2.1 From 3dc74378ec54afb4b15d9f1bdf3781ddd50f83e3 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 20:15:28 -0800 Subject: Allow individual failures --- .../populate_untracked_uploads.rb | 40 +++++++++++++--------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 1773b53bd68..934431ccddd 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -97,28 +97,18 @@ module Gitlab end def uploader - PATH_PATTERNS.each do |path_pattern_map| - if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) - return path_pattern_map[:uploader] - end - end + matching_pattern_map[:uploader] end def model_type - PATH_PATTERNS.each do |path_pattern_map| - if path_relative_to_upload_dir.match(path_pattern_map[:pattern]) - return path_pattern_map[:model_type] - end - end + matching_pattern_map[:model_type] end def model_id - PATH_PATTERNS.each do |path_pattern_map| - matchd = path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern]) - # If something is captured (matchd[1] is not nil), it is a model_id - return matchd[1] if matchd && matchd[1] - end + # If something is captured (matchd[1] is not nil), it is a model_id + return matchd[1] if matchd[1] # Only the FileUploader pattern will not match an ID file_uploader_model_id @@ -140,6 +130,16 @@ module Gitlab private + def matching_pattern_map + @matching_pattern_map ||= PATH_PATTERNS.find do |path_pattern_map| + path_relative_to_upload_dir.match(path_pattern_map[:pattern]) + end + + raise "Unknown upload path pattern \"#{path}\"" unless @matching_pattern_map + + @matching_pattern_map + end + def file_uploader_model_id pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) @@ -222,7 +222,15 @@ module Gitlab files = UnhashedUploadFile.untracked.where(id: start_id..end_id) files.each do |unhashed_upload_file| - unhashed_upload_file.ensure_tracked! + begin + unhashed_upload_file.ensure_tracked! + rescue StandardError => e + Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}" + + # The untracked rows will remain in the DB. We will be able to see + # which ones failed to become tracked, and then we can decide what + # to do. + end end end -- cgit v1.2.1 From 13e0ee373573c41d4c1d095fbeec2a133ae626bb Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 20:38:17 -0800 Subject: Test batch processing --- .../populate_untracked_uploads_spec.rb | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index c61a207d012..5446edae06f 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -53,6 +53,12 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(appearance.uploads.count).to eq(2) end + it 'sets all added or confirmed tracked files to tracked' do + expect do + described_class.new.perform(1, 1000) + end.to change { unhashed_upload_files.where(tracked: true).count }.from(0).to(8) + end + it 'does not create duplicate uploads of already tracked files' do described_class.new.perform(1, 1000) @@ -60,6 +66,42 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project1.uploads.count).to eq(2) expect(appearance.uploads.count).to eq(2) end + + it 'uses the start and end batch ids [only 1st half]' do + start_id = unhashed_upload_files.all.to_a[0].id + end_id = unhashed_upload_files.all.to_a[3].id + + expect do + described_class.new.perform(start_id, end_id) + end.to change { uploads.count }.from(4).to(6) + + expect(user1.uploads.count).to eq(1) + expect(user2.uploads.count).to eq(1) + expect(appearance.uploads.count).to eq(2) + expect(project1.uploads.count).to eq(2) + expect(project2.uploads.count).to eq(0) + + # Only 4 have been either confirmed or added to uploads + expect(unhashed_upload_files.where(tracked: true).count).to eq(4) + end + + it 'uses the start and end batch ids [only 2nd half]' do + start_id = unhashed_upload_files.all.to_a[4].id + end_id = unhashed_upload_files.all.to_a[7].id + + expect do + described_class.new.perform(start_id, end_id) + end.to change { uploads.count }.from(4).to(6) + + expect(user1.uploads.count).to eq(1) + expect(user2.uploads.count).to eq(0) + expect(appearance.uploads.count).to eq(1) + expect(project1.uploads.count).to eq(2) + expect(project2.uploads.count).to eq(2) + + # Only 4 have been either confirmed or added to uploads + expect(unhashed_upload_files.where(tracked: true).count).to eq(4) + end end context 'with no untracked files' do -- cgit v1.2.1 From ffbaf19fe8a83f651f45380749ccc12cd38ec29f Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 20:54:54 -0800 Subject: Fix Rubocop offenses --- .../background_migration/populate_untracked_uploads.rb | 16 ++++++++-------- .../background_migration/prepare_unhashed_uploads.rb | 6 +++--- .../populate_untracked_uploads_spec.rb | 2 +- spec/migrations/track_untracked_uploads_spec.rb | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 934431ccddd..ef0f1209ef5 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -15,39 +15,39 @@ module Gitlab { pattern: /\A-\/system\/appearance\/logo\/(\d+)/, uploader: 'AttachmentUploader', - model_type: 'Appearance', + model_type: 'Appearance' }, { pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, uploader: 'AttachmentUploader', - model_type: 'Appearance', + model_type: 'Appearance' }, { pattern: /\A-\/system\/note\/attachment\/(\d+)/, uploader: 'AttachmentUploader', - model_type: 'Note', + model_type: 'Note' }, { pattern: /\A-\/system\/user\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'User', + model_type: 'User' }, { pattern: /\A-\/system\/group\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'Namespace', + model_type: 'Namespace' }, { pattern: /\A-\/system\/project\/avatar\/(\d+)/, uploader: 'AvatarUploader', - model_type: 'Project', + model_type: 'Project' }, { pattern: FILE_UPLOADER_PATH_PATTERN, uploader: 'FileUploader', model_type: 'Project' - }, - ] + } + ].freeze scope :untracked, -> { where(tracked: false) } diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 556457039fa..7c426022304 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -5,8 +5,8 @@ module Gitlab include Database::MigrationHelpers FILE_PATH_BATCH_SIZE = 500 - UPLOAD_DIR = "#{CarrierWave.root}/uploads" - FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads' + UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze class UnhashedUploadFile < ActiveRecord::Base include EachBatch @@ -33,7 +33,7 @@ module Gitlab end def store_unhashed_upload_file_paths - return unless Dir.exists?(UPLOAD_DIR) + return unless Dir.exist?(UPLOAD_DIR) file_paths = [] each_file_path(UPLOAD_DIR) do |file_path| diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 5446edae06f..82d5a588a56 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -129,7 +129,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'does not add an upload' do expect do unhashed_upload_file.ensure_tracked! - end.to_not change { upload_class.count }.from(1) + end.not_to change { upload_class.count }.from(1) end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 9539993be31..d896a3e7d54 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -37,7 +37,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do migrate! max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255 - long_path = "./uploads#{("/#{max_length_namespace_path}") * Namespace::NUMBER_OF_ANCESTORS_ALLOWED}/#{max_length_project_path}/#{max_length_filename}" + long_path = "./uploads#{"/#{max_length_namespace_path}" * Namespace::NUMBER_OF_ANCESTORS_ALLOWED}/#{max_length_project_path}/#{max_length_filename}" unhashed_upload_file = UnhashedUploadFile.new(path: long_path) unhashed_upload_file.save! expect(UnhashedUploadFile.first.path.size).to eq(5641) @@ -106,7 +106,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(user1.reload.uploads.first.attributes).to include({ "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader", + "uploader" => "AvatarUploader" }.merge(rails_sample_jpg_attrs)) expect(project1.reload.uploads.first.attributes).to include({ "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", -- cgit v1.2.1 From 36611773e920ebaa1c1c8d603107f47200fb8e00 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 7 Nov 2017 21:05:54 -0800 Subject: Add changelog entry --- changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml diff --git a/changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml b/changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml new file mode 100644 index 00000000000..499543ef883 --- /dev/null +++ b/changelogs/unreleased/mk-add-old-attachments-to-uploads-table.yml @@ -0,0 +1,5 @@ +--- +title: Add untracked files to uploads table +merge_request: 15270 +author: +type: other -- cgit v1.2.1 From 41412fec5b9aea8a6104320c5b554ffdabe52506 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 12:31:51 -0800 Subject: Avoid instantiating an AR object and ignore dupes --- .../background_migration/prepare_unhashed_uploads.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 7c426022304..982c0ff5320 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -75,10 +75,24 @@ module Gitlab def insert_file_paths(file_paths) file_paths.each do |file_path| - UnhashedUploadFile.create!(path: file_path) + insert_file_path(file_path) end end + def insert_file_path(file_path) + table_columns_and_values = 'unhashed_upload_files (path, created_at, updated_at) VALUES (?, ?, ?)' + + sql = if Gitlab::Database.postgresql? + "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" + else + "INSERT IGNORE INTO #{table_columns_and_values};" + end + + timestamp = Time.now.utc.iso8601 + sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) + ActiveRecord::Base.connection.execute(sql) + end + def schedule_populate_untracked_uploads_jobs bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) end -- cgit v1.2.1 From 7c43692f68dd62773b0a7b094453f582a4242ef7 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 12:44:49 -0800 Subject: Make regexes more readable --- .../populate_untracked_uploads.rb | 22 +++++++++++----------- spec/migrations/track_untracked_uploads_spec.rb | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index ef0f1209ef5..42ad28400f4 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -5,7 +5,8 @@ module Gitlab self.table_name = 'unhashed_upload_files' # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH_PATTERN = /\/\h+\/[^\/]+\z/ + FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z} + FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN = %r{\A(.+)#{FILE_UPLOADER_PATH_PATTERN}} # These regex patterns are tested against a relative path, relative to # the upload directory. @@ -13,32 +14,32 @@ module Gitlab # it indicates the model_id. PATH_PATTERNS = [ { - pattern: /\A-\/system\/appearance\/logo\/(\d+)/, + pattern: %r{\A-/system/appearance/logo/(\d+)/}, uploader: 'AttachmentUploader', model_type: 'Appearance' }, { - pattern: /\A-\/system\/appearance\/header_logo\/(\d+)/, + pattern: %r{\A-/system/appearance/header_logo/(\d+)/}, uploader: 'AttachmentUploader', model_type: 'Appearance' }, { - pattern: /\A-\/system\/note\/attachment\/(\d+)/, + pattern: %r{\A-/system/note/attachment/(\d+)/}, uploader: 'AttachmentUploader', model_type: 'Note' }, { - pattern: /\A-\/system\/user\/avatar\/(\d+)/, + pattern: %r{\A-/system/user/avatar/(\d+)/}, uploader: 'AvatarUploader', model_type: 'User' }, { - pattern: /\A-\/system\/group\/avatar\/(\d+)/, + pattern: %r{\A-/system/group/avatar/(\d+)/}, uploader: 'AvatarUploader', model_type: 'Namespace' }, { - pattern: /\A-\/system\/project\/avatar\/(\d+)/, + pattern: %r{\A-/system/project/avatar/(\d+)/}, uploader: 'AvatarUploader', model_type: 'Project' }, @@ -90,7 +91,7 @@ module Gitlab if uploader == 'FileUploader' # Path relative to project directory in uploads matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) - matchd[0].sub(/\A\//, '') # remove leading slash + matchd[0].sub(%r{\A/}, '') # remove leading slash else path_relative_to_carrierwave_root end @@ -120,7 +121,7 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - @path_relative_to_upload_dir ||= path.sub(/\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}\//, '') + @path_relative_to_upload_dir ||= path.sub(%r{\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/}, '') end # Not including a leading slash @@ -141,8 +142,7 @@ module Gitlab end def file_uploader_model_id - pattern_to_capture_full_path = /\A(.+)#{FILE_UPLOADER_PATH_PATTERN}/ - matchd = path_relative_to_upload_dir.match(pattern_to_capture_full_path) + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN) raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd full_path = matchd[1] project = Project.find_by_full_path(full_path) diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index d896a3e7d54..308d8924f19 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -59,7 +59,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do user1.update(avatar: uploaded_file) project1.update(avatar: uploaded_file) upload_result = UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - @project1_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') + @project1_markdown_upload_path = upload_result[:url].sub(%r{\A/uploads/}, '') appearance.update(logo: uploaded_file) # Untracked, by doing normal file upload then deleting records from DB @@ -68,7 +68,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do user2.uploads.delete_all project2.update(avatar: uploaded_file) upload_result = UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload - @project2_markdown_upload_path = upload_result[:url].sub(/\A\/uploads\//, '') + @project2_markdown_upload_path = upload_result[:url].sub(%r{\A/uploads/}, '') project2.uploads.delete_all appearance.update(header_logo: uploaded_file) appearance.uploads.last.destroy -- cgit v1.2.1 From 0e9efa74a7ea653a969436cb98c0a4ba80e8dd71 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 13:29:03 -0800 Subject: Use `find` `-prune` option for performance --- lib/gitlab/background_migration/prepare_unhashed_uploads.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index 982c0ff5320..ce488542df9 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -59,9 +59,11 @@ module Gitlab end def build_find_command(search_dir) - cmd = ['find', search_dir, '-type', 'f', '!', '-path', "#{UPLOAD_DIR}/@hashed/*", '!', '-path', "#{UPLOAD_DIR}/tmp/*", '-print0'] + hashed_path = "#{UPLOAD_DIR}/@hashed/*" + tmp_path = "#{UPLOAD_DIR}/tmp/*" + cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] - ['ionice', '-c', 'Idle'] + cmd if ionice_is_available? + %w[ionice -c Idle] + cmd if ionice_is_available? cmd end -- cgit v1.2.1 From 2ab3031bd35213802e508fef6eebceaaf40cee9b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 8 Nov 2017 15:05:08 -0800 Subject: Refactor, no change in behavior --- .../populate_untracked_uploads.rb | 10 +++---- .../prepare_unhashed_uploads.rb | 35 +++++++++++++--------- .../populate_untracked_uploads_spec.rb | 9 ++---- spec/migrations/track_untracked_uploads_spec.rb | 26 ++++++++-------- spec/support/track_untracked_uploads_helpers.rb | 12 ++++++++ 5 files changed, 53 insertions(+), 39 deletions(-) create mode 100644 spec/support/track_untracked_uploads_helpers.rb diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 42ad28400f4..e63220c8001 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -55,9 +55,7 @@ module Gitlab def ensure_tracked! return if persisted? && tracked? - unless in_uploads? - add_to_uploads - end + add_to_uploads unless in_uploads? mark_as_tracked end @@ -82,8 +80,7 @@ module Gitlab end def mark_as_tracked - self.tracked = true - self.save! + update!(tracked: true) end def upload_path @@ -121,7 +118,8 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - @path_relative_to_upload_dir ||= path.sub(%r{\A#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/}, '') + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR)}/} + @path_relative_to_upload_dir ||= path.sub(base, '') end # Not including a leading slash diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb index ce488542df9..8033f994959 100644 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb @@ -35,29 +35,36 @@ module Gitlab def store_unhashed_upload_file_paths return unless Dir.exist?(UPLOAD_DIR) - file_paths = [] - each_file_path(UPLOAD_DIR) do |file_path| - file_paths << file_path - - if file_paths.size >= FILE_PATH_BATCH_SIZE - insert_file_paths(file_paths) - file_paths = [] - end + each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + insert_file_paths(file_paths) end - - insert_file_paths(file_paths) if file_paths.any? end - def each_file_path(search_dir, &block) + def each_file_batch(search_dir, batch_size, &block) cmd = build_find_command(search_dir) + Open3.popen2(*cmd) do |stdin, stdout, status_thread| - stdout.each_line("\0") do |line| - yield(line.chomp("\0")) - end + yield_paths_in_batches(stdout, batch_size, &block) + raise "Find command failed" unless status_thread.value.success? end end + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0") + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) + end + def build_find_command(search_dir) hashed_path = "#{UPLOAD_DIR}/@hashed/*" tmp_path = "#{UPLOAD_DIR}/tmp/*" diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 82d5a588a56..28a9ee73470 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -114,6 +114,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFile do + include TrackUntrackedUploadsHelpers + let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } describe '#ensure_tracked!' do @@ -596,11 +598,4 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi end end end - - def rails_sample_jpg_attrs - { - "size" => 35255, - "checksum" => 'f2d1fd9d8d8a3368d468fa067888605d74a66f41c16f55979ceaf2af77375844' - } - end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 308d8924f19..49758ede1a4 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -2,6 +2,8 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') describe TrackUntrackedUploads, :migration, :sidekiq do + include TrackUntrackedUploadsHelpers + class UnhashedUploadFile < ActiveRecord::Base self.table_name = 'unhashed_upload_files' end @@ -36,11 +38,18 @@ describe TrackUntrackedUploads, :migration, :sidekiq do it 'has a path field long enough for really long paths' do migrate! - max_length_namespace_path = max_length_project_path = max_length_filename = 'a' * 255 - long_path = "./uploads#{"/#{max_length_namespace_path}" * Namespace::NUMBER_OF_ANCESTORS_ALLOWED}/#{max_length_project_path}/#{max_length_filename}" - unhashed_upload_file = UnhashedUploadFile.new(path: long_path) - unhashed_upload_file.save! - expect(UnhashedUploadFile.first.path.size).to eq(5641) + component = 'a'*255 + + long_path = [ + CarrierWave.root, + 'uploads', + [component] * Namespace::NUMBER_OF_ANCESTORS_ALLOWED, # namespaces + component, # project + component # filename + ].flatten.join('/') + + record = UnhashedUploadFile.create!(path: long_path) + expect(record.reload.path.size).to eq(5711) end context 'with tracked and untracked uploads' do @@ -132,11 +141,4 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end end - - def rails_sample_jpg_attrs - { - "size" => 35255, - "checksum" => 'f2d1fd9d8d8a3368d468fa067888605d74a66f41c16f55979ceaf2af77375844' - } - end end diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb new file mode 100644 index 00000000000..749c5775bb0 --- /dev/null +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -0,0 +1,12 @@ +module TrackUntrackedUploadsHelpers + def rails_sample_jpg_attrs + @rails_sample_jpg_attrs ||= { + "size" => File.size(rails_sample_file_path), + "checksum" => Digest::SHA256.file(rails_sample_file_path).hexdigest + } + end + + def rails_sample_file_path + Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + end +end -- cgit v1.2.1 From a210cb6b827d9d918788578fc4ae956471de3b12 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:17:56 -0800 Subject: Rename table to untracked_files_for_uploads --- .../20171103140253_track_untracked_uploads.rb | 18 +- db/schema.rb | 6 +- .../populate_untracked_uploads.rb | 16 +- .../prepare_unhashed_uploads.rb | 110 ----------- .../prepare_untracked_uploads.rb | 110 +++++++++++ .../populate_untracked_uploads_spec.rb | 212 ++++++++++----------- .../prepare_unhashed_uploads_spec.rb | 111 ----------- .../prepare_untracked_uploads_spec.rb | 111 +++++++++++ spec/migrations/track_untracked_uploads_spec.rb | 16 +- 9 files changed, 355 insertions(+), 355 deletions(-) delete mode 100644 lib/gitlab/background_migration/prepare_unhashed_uploads.rb create mode 100644 lib/gitlab/background_migration/prepare_untracked_uploads.rb delete mode 100644 spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb create mode 100644 spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb index 90d530d4011..5e4e357b8bc 100644 --- a/db/post_migrate/20171103140253_track_untracked_uploads.rb +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -7,31 +7,31 @@ class TrackUntrackedUploads < ActiveRecord::Migration disable_ddl_transaction! DOWNTIME = false - MIGRATION = 'PrepareUnhashedUploads' + MIGRATION = 'PrepareUntrackedUploads' def up - unless table_exists?(:unhashed_upload_files) - create_table :unhashed_upload_files do |t| + unless table_exists?(:untracked_files_for_uploads) + create_table :untracked_files_for_uploads do |t| t.string :path, null: false t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false end end - unless index_exists?(:unhashed_upload_files, :path) - add_index :unhashed_upload_files, :path, unique: true + unless index_exists?(:untracked_files_for_uploads, :path) + add_index :untracked_files_for_uploads, :path, unique: true end - unless index_exists?(:unhashed_upload_files, :tracked) - add_index :unhashed_upload_files, :tracked + unless index_exists?(:untracked_files_for_uploads, :tracked) + add_index :untracked_files_for_uploads, :tracked end BackgroundMigrationWorker.perform_async(MIGRATION) end def down - if table_exists?(:unhashed_upload_files) - drop_table :unhashed_upload_files + if table_exists?(:untracked_files_for_uploads) + drop_table :untracked_files_for_uploads end end end diff --git a/db/schema.rb b/db/schema.rb index 2b7e12b45f1..e193d569739 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1719,15 +1719,15 @@ ActiveRecord::Schema.define(version: 20171124150326) do add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree - create_table "unhashed_upload_files", force: :cascade do |t| + create_table "untracked_files_for_uploads", force: :cascade do |t| t.string "path", null: false t.boolean "tracked", default: false, null: false t.datetime_with_timezone "created_at", null: false t.datetime_with_timezone "updated_at", null: false end - add_index "unhashed_upload_files", ["path"], name: "index_unhashed_upload_files_on_path", unique: true, using: :btree - add_index "unhashed_upload_files", ["tracked"], name: "index_unhashed_upload_files_on_tracked", using: :btree + add_index "untracked_files_for_uploads", ["path"], name: "index_untracked_files_for_uploads_on_path", unique: true, using: :btree + add_index "untracked_files_for_uploads", ["tracked"], name: "index_untracked_files_for_uploads_on_tracked", using: :btree create_table "uploads", force: :cascade do |t| t.integer "size", limit: 8, null: false diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index e63220c8001..bd0f2f591a4 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -1,8 +1,8 @@ module Gitlab module BackgroundMigration class PopulateUntrackedUploads - class UnhashedUploadFile < ActiveRecord::Base - self.table_name = 'unhashed_upload_files' + class UntrackedFile < ActiveRecord::Base + self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z} @@ -84,7 +84,7 @@ module Gitlab end def upload_path - # UnhashedUploadFile#path is absolute, but Upload#path depends on uploader + # UntrackedFile#path is absolute, but Upload#path depends on uploader if uploader == 'FileUploader' # Path relative to project directory in uploads matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) @@ -118,7 +118,7 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR)}/} + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end @@ -218,10 +218,10 @@ module Gitlab def perform(start_id, end_id) return unless migrate? - files = UnhashedUploadFile.untracked.where(id: start_id..end_id) - files.each do |unhashed_upload_file| + files = UntrackedFile.untracked.where(id: start_id..end_id) + files.each do |untracked_file| begin - unhashed_upload_file.ensure_tracked! + untracked_file.ensure_tracked! rescue StandardError => e Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}" @@ -235,7 +235,7 @@ module Gitlab private def migrate? - UnhashedUploadFile.table_exists? && Upload.table_exists? + UntrackedFile.table_exists? && Upload.table_exists? end end end diff --git a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb b/lib/gitlab/background_migration/prepare_unhashed_uploads.rb deleted file mode 100644 index 8033f994959..00000000000 --- a/lib/gitlab/background_migration/prepare_unhashed_uploads.rb +++ /dev/null @@ -1,110 +0,0 @@ -module Gitlab - module BackgroundMigration - class PrepareUnhashedUploads - # For bulk_queue_background_migration_jobs_by_range - include Database::MigrationHelpers - - FILE_PATH_BATCH_SIZE = 500 - UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze - FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze - - class UnhashedUploadFile < ActiveRecord::Base - include EachBatch - - self.table_name = 'unhashed_upload_files' - end - - def perform - return unless migrate? - - clear_unhashed_upload_file_paths - store_unhashed_upload_file_paths - schedule_populate_untracked_uploads_jobs - end - - private - - def migrate? - UnhashedUploadFile.table_exists? - end - - def clear_unhashed_upload_file_paths - UnhashedUploadFile.delete_all - end - - def store_unhashed_upload_file_paths - return unless Dir.exist?(UPLOAD_DIR) - - each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| - insert_file_paths(file_paths) - end - end - - def each_file_batch(search_dir, batch_size, &block) - cmd = build_find_command(search_dir) - - Open3.popen2(*cmd) do |stdin, stdout, status_thread| - yield_paths_in_batches(stdout, batch_size, &block) - - raise "Find command failed" unless status_thread.value.success? - end - end - - def yield_paths_in_batches(stdout, batch_size, &block) - paths = [] - - stdout.each_line("\0") do |line| - paths << line.chomp("\0") - - if paths.size >= batch_size - yield(paths) - paths = [] - end - end - - yield(paths) - end - - def build_find_command(search_dir) - hashed_path = "#{UPLOAD_DIR}/@hashed/*" - tmp_path = "#{UPLOAD_DIR}/tmp/*" - cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] - - %w[ionice -c Idle] + cmd if ionice_is_available? - - cmd - end - - def ionice_is_available? - Gitlab::Utils.which('ionice') - rescue StandardError - # In this case, returning false is relatively safe, even though it isn't very nice - false - end - - def insert_file_paths(file_paths) - file_paths.each do |file_path| - insert_file_path(file_path) - end - end - - def insert_file_path(file_path) - table_columns_and_values = 'unhashed_upload_files (path, created_at, updated_at) VALUES (?, ?, ?)' - - sql = if Gitlab::Database.postgresql? - "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" - else - "INSERT IGNORE INTO #{table_columns_and_values};" - end - - timestamp = Time.now.utc.iso8601 - sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) - ActiveRecord::Base.connection.execute(sql) - end - - def schedule_populate_untracked_uploads_jobs - bulk_queue_background_migration_jobs_by_range(UnhashedUploadFile, FOLLOW_UP_MIGRATION) - end - end - end -end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb new file mode 100644 index 00000000000..6a7fef18e53 --- /dev/null +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -0,0 +1,110 @@ +module Gitlab + module BackgroundMigration + class PrepareUntrackedUploads + # For bulk_queue_background_migration_jobs_by_range + include Database::MigrationHelpers + + FILE_PATH_BATCH_SIZE = 500 + UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze + FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze + + class UntrackedFile < ActiveRecord::Base + include EachBatch + + self.table_name = 'untracked_files_for_uploads' + end + + def perform + return unless migrate? + + clear_untracked_file_paths + store_untracked_file_paths + schedule_populate_untracked_uploads_jobs + end + + private + + def migrate? + UntrackedFile.table_exists? + end + + def clear_untracked_file_paths + UntrackedFile.delete_all + end + + def store_untracked_file_paths + return unless Dir.exist?(UPLOAD_DIR) + + each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + insert_file_paths(file_paths) + end + end + + def each_file_batch(search_dir, batch_size, &block) + cmd = build_find_command(search_dir) + + Open3.popen2(*cmd) do |stdin, stdout, status_thread| + yield_paths_in_batches(stdout, batch_size, &block) + + raise "Find command failed" unless status_thread.value.success? + end + end + + def yield_paths_in_batches(stdout, batch_size, &block) + paths = [] + + stdout.each_line("\0") do |line| + paths << line.chomp("\0") + + if paths.size >= batch_size + yield(paths) + paths = [] + end + end + + yield(paths) + end + + def build_find_command(search_dir) + hashed_path = "#{UPLOAD_DIR}/@hashed/*" + tmp_path = "#{UPLOAD_DIR}/tmp/*" + cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] + + %w[ionice -c Idle] + cmd if ionice_is_available? + + cmd + end + + def ionice_is_available? + Gitlab::Utils.which('ionice') + rescue StandardError + # In this case, returning false is relatively safe, even though it isn't very nice + false + end + + def insert_file_paths(file_paths) + file_paths.each do |file_path| + insert_file_path(file_path) + end + end + + def insert_file_path(file_path) + table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' + + sql = if Gitlab::Database.postgresql? + "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" + else + "INSERT IGNORE INTO #{table_columns_and_values};" + end + + timestamp = Time.now.utc.iso8601 + sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) + ActiveRecord::Base.connection.execute(sql) + end + + def schedule_populate_untracked_uploads_jobs + bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION) + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 28a9ee73470..b3122e90c83 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do - let!(:unhashed_upload_files) { table(:unhashed_upload_files) } + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } let!(:uploads) { table(:uploads) } let(:user1) { create(:user) } @@ -10,7 +10,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid let(:project2) { create(:project) } let(:appearance) { create(:appearance) } - context 'with untracked files and tracked files in unhashed_upload_files' do + context 'with untracked files and tracked files in untracked_files_for_uploads' do before do fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') @@ -28,15 +28,15 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload appearance.update!(header_logo: uploaded_file) - # Unhashed upload files created by PrepareUnhashedUploads - unhashed_upload_files.create!(path: appearance.logo.file.file) - unhashed_upload_files.create!(path: appearance.header_logo.file.file) - unhashed_upload_files.create!(path: user1.avatar.file.file) - unhashed_upload_files.create!(path: user2.avatar.file.file) - unhashed_upload_files.create!(path: project1.avatar.file.file) - unhashed_upload_files.create!(path: project2.avatar.file.file) - unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") - unhashed_upload_files.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") + # File records created by PrepareUntrackedUploads + untracked_files_for_uploads.create!(path: appearance.logo.file.file) + untracked_files_for_uploads.create!(path: appearance.header_logo.file.file) + untracked_files_for_uploads.create!(path: user1.avatar.file.file) + untracked_files_for_uploads.create!(path: user2.avatar.file.file) + untracked_files_for_uploads.create!(path: project1.avatar.file.file) + untracked_files_for_uploads.create!(path: project2.avatar.file.file) + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") user2.uploads.delete_all project2.uploads.delete_all @@ -56,7 +56,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'sets all added or confirmed tracked files to tracked' do expect do described_class.new.perform(1, 1000) - end.to change { unhashed_upload_files.where(tracked: true).count }.from(0).to(8) + end.to change { untracked_files_for_uploads.where(tracked: true).count }.from(0).to(8) end it 'does not create duplicate uploads of already tracked files' do @@ -68,8 +68,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end it 'uses the start and end batch ids [only 1st half]' do - start_id = unhashed_upload_files.all.to_a[0].id - end_id = unhashed_upload_files.all.to_a[3].id + start_id = untracked_files_for_uploads.all.to_a[0].id + end_id = untracked_files_for_uploads.all.to_a[3].id expect do described_class.new.perform(start_id, end_id) @@ -82,12 +82,12 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project2.uploads.count).to eq(0) # Only 4 have been either confirmed or added to uploads - expect(unhashed_upload_files.where(tracked: true).count).to eq(4) + expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) end it 'uses the start and end batch ids [only 2nd half]' do - start_id = unhashed_upload_files.all.to_a[4].id - end_id = unhashed_upload_files.all.to_a[7].id + start_id = untracked_files_for_uploads.all.to_a[4].id + end_id = untracked_files_for_uploads.all.to_a[7].id expect do described_class.new.perform(start_id, end_id) @@ -100,7 +100,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project2.uploads.count).to eq(2) # Only 4 have been either confirmed or added to uploads - expect(unhashed_upload_files.where(tracked: true).count).to eq(4) + expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) end end @@ -113,7 +113,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end end -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFile do +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do include TrackUntrackedUploadsHelpers let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } @@ -122,7 +122,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi let(:user1) { create(:user) } context 'when the file is already in the uploads table' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") } before do upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234) @@ -130,7 +130,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'does not add an upload' do expect do - unhashed_upload_file.ensure_tracked! + untracked_file.ensure_tracked! end.not_to change { upload_class.count }.from(1) end end @@ -142,7 +142,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for an appearance logo file path' do let(:model) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: model.logo.file.file) } + let(:untracked_file) { described_class.create!(path: model.logo.file.file) } before do model.update!(logo: uploaded_file) @@ -151,7 +151,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -163,7 +163,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for an appearance header_logo file path' do let(:model) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: model.header_logo.file.file) } + let(:untracked_file) { described_class.create!(path: model.header_logo.file.file) } before do model.update!(header_logo: uploaded_file) @@ -172,7 +172,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -184,7 +184,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a pre-Markdown Note attachment file path' do let(:model) { create(:note) } - let(:unhashed_upload_file) { described_class.create!(path: model.attachment.file.file) } + let(:untracked_file) { described_class.create!(path: model.attachment.file.file) } before do model.update!(attachment: uploaded_file) @@ -193,7 +193,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { upload_class.count }.from(0).to(1) expect(upload_class.first.attributes).to include({ @@ -207,7 +207,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a user avatar file path' do let(:model) { create(:user) } - let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } before do model.update!(avatar: uploaded_file) @@ -216,7 +216,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -228,7 +228,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a group avatar file path' do let(:model) { create(:group) } - let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } before do model.update!(avatar: uploaded_file) @@ -237,7 +237,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -251,7 +251,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a project avatar file path' do let(:model) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } before do model.update!(avatar: uploaded_file) @@ -260,7 +260,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include({ @@ -272,20 +272,20 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - let(:unhashed_upload_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } + let(:untracked_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } before do UploadService.new(model, uploaded_file, FileUploader).execute # Markdown upload - unhashed_upload_file.save! + untracked_file.save! model.reload.uploads.delete_all end it 'creates an Upload record' do expect do - unhashed_upload_file.add_to_uploads + untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) - hex_secret = unhashed_upload_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] + hex_secret = untracked_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] expect(model.uploads.first.attributes).to include({ "path" => "#{hex_secret}/rails_sample.jpg", "uploader" => "FileUploader" @@ -295,250 +295,250 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi end describe '#mark_as_tracked' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'saves the record with tracked set to true' do expect do - unhashed_upload_file.mark_as_tracked - end.to change { unhashed_upload_file.tracked }.from(false).to(true) + untracked_file.mark_as_tracked + end.to change { untracked_file.tracked }.from(false).to(true) - expect(unhashed_upload_file.persisted?).to be_truthy + expect(untracked_file.persisted?).to be_truthy end end describe '#upload_path' do context 'for an appearance logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') end end context 'for an appearance header_logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') end end context 'for a pre-Markdown Note attachment file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') + expect(untracked_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') end end context 'for a user avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') end end context 'for a group avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') end end context 'for a project avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do - expect(unhashed_upload_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') + expect(untracked_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } let(:random_hex) { SecureRandom.hex } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } it 'returns the file path relative to the project directory in uploads' do - expect(unhashed_upload_file.upload_path).to eq("#{random_hex}/Some file.jpg") + expect(untracked_file.upload_path).to eq("#{random_hex}/Some file.jpg") end end end describe '#uploader' do context 'for an appearance logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns AttachmentUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + expect(untracked_file.uploader).to eq('AttachmentUploader') end end context 'for an appearance header_logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns AttachmentUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + expect(untracked_file.uploader).to eq('AttachmentUploader') end end context 'for a pre-Markdown Note attachment file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns AttachmentUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AttachmentUploader') + expect(untracked_file.uploader).to eq('AttachmentUploader') end end context 'for a user avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + expect(untracked_file.uploader).to eq('AvatarUploader') end end context 'for a group avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + expect(untracked_file.uploader).to eq('AvatarUploader') end end context 'for a project avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('AvatarUploader') + expect(untracked_file.uploader).to eq('AvatarUploader') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns FileUploader as a string' do - expect(unhashed_upload_file.uploader).to eq('FileUploader') + expect(untracked_file.uploader).to eq('FileUploader') end end end describe '#model_type' do context 'for an appearance logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns Appearance as a string' do - expect(unhashed_upload_file.model_type).to eq('Appearance') + expect(untracked_file.model_type).to eq('Appearance') end end context 'for an appearance header_logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns Appearance as a string' do - expect(unhashed_upload_file.model_type).to eq('Appearance') + expect(untracked_file.model_type).to eq('Appearance') end end context 'for a pre-Markdown Note attachment file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns Note as a string' do - expect(unhashed_upload_file.model_type).to eq('Note') + expect(untracked_file.model_type).to eq('Note') end end context 'for a user avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns User as a string' do - expect(unhashed_upload_file.model_type).to eq('User') + expect(untracked_file.model_type).to eq('User') end end context 'for a group avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns Namespace as a string' do - expect(unhashed_upload_file.model_type).to eq('Namespace') + expect(untracked_file.model_type).to eq('Namespace') end end context 'for a project avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns Project as a string' do - expect(unhashed_upload_file.model_type).to eq('Project') + expect(untracked_file.model_type).to eq('Project') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns Project as a string' do - expect(unhashed_upload_file.model_type).to eq('Project') + expect(untracked_file.model_type).to eq('Project') end end end describe '#model_id' do context 'for an appearance logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1') + expect(untracked_file.model_id).to eq('1') end end context 'for an appearance header_logo file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1') + expect(untracked_file.model_id).to eq('1') end end context 'for a pre-Markdown Note attachment file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1234') + expect(untracked_file.model_id).to eq('1234') end end context 'for a user avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1234') + expect(untracked_file.model_id).to eq('1234') end end context 'for a group avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1234') + expect(untracked_file.model_id).to eq('1234') end end context 'for a project avatar file path' do - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq('1234') + expect(untracked_file.model_id).to eq('1234') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns the ID as a string' do - expect(unhashed_upload_file.model_id).to eq(project.id.to_s) + expect(untracked_file.model_id).to eq(project.id.to_s) end end end @@ -549,52 +549,52 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UnhashedUploadFi context 'for an appearance logo file path' do let(:appearance) { create(:appearance) } - let(:unhashed_upload_file) { described_class.create!(path: appearance.logo.file.file) } + let(:untracked_file) { described_class.create!(path: appearance.logo.file.file) } before do appearance.update!(logo: uploaded_file) end it 'returns the file size' do - expect(unhashed_upload_file.file_size).to eq(35255) + expect(untracked_file.file_size).to eq(35255) end it 'returns the same thing that CarrierWave would return' do - expect(unhashed_upload_file.file_size).to eq(appearance.logo.size) + expect(untracked_file.file_size).to eq(appearance.logo.size) end end context 'for a project avatar file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: project.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: project.avatar.file.file) } before do project.update!(avatar: uploaded_file) end it 'returns the file size' do - expect(unhashed_upload_file.file_size).to eq(35255) + expect(untracked_file.file_size).to eq(35255) end it 'returns the same thing that CarrierWave would return' do - expect(unhashed_upload_file.file_size).to eq(project.avatar.size) + expect(untracked_file.file_size).to eq(project.avatar.size) end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:unhashed_upload_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUnhashedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } before do UploadService.new(project, uploaded_file, FileUploader).execute end it 'returns the file size' do - expect(unhashed_upload_file.file_size).to eq(35255) + expect(untracked_file.file_size).to eq(35255) end it 'returns the same thing that CarrierWave would return' do - expect(unhashed_upload_file.file_size).to eq(project.uploads.first.size) + expect(untracked_file.file_size).to eq(project.uploads.first.size) end end end diff --git a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb deleted file mode 100644 index 76d126e8f00..00000000000 --- a/spec/lib/gitlab/background_migration/prepare_unhashed_uploads_spec.rb +++ /dev/null @@ -1,111 +0,0 @@ -require 'spec_helper' - -describe Gitlab::BackgroundMigration::PrepareUnhashedUploads, :migration, :sidekiq, schema: 20171103140253 do - let!(:unhashed_upload_files) { table(:unhashed_upload_files) } - - let(:user1) { create(:user) } - let(:user2) { create(:user) } - let(:project1) { create(:project) } - let(:project2) { create(:project) } - let(:appearance) { create(:appearance) } - - matcher :be_scheduled_migration do |*expected| - match do |migration| - BackgroundMigrationWorker.jobs.any? do |job| - job['args'] == [migration, expected] - end - end - - failure_message do |migration| - "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" - end - end - - context 'when files were uploaded before and after hashed storage was enabled' do - before do - fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') - uploaded_file = fixture_file_upload(fixture) - - user1.update(avatar: uploaded_file) - project1.update(avatar: uploaded_file) - appearance.update(logo: uploaded_file, header_logo: uploaded_file) - uploaded_file = fixture_file_upload(fixture) - UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - - stub_application_setting(hashed_storage_enabled: true) - - # Hashed files - uploaded_file = fixture_file_upload(fixture) - UploadService.new(project2, uploaded_file, FileUploader).execute - end - - it 'adds unhashed files to the unhashed_upload_files table' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) - end - end - - it 'does not add hashed files to the unhashed_upload_files table' do - Sidekiq::Testing.fake! do - described_class.new.perform - - hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path - expect(unhashed_upload_files.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey - end - end - - it 'correctly schedules the follow-up background migration jobs' do - Sidekiq::Testing.fake! do - described_class.new.perform - - expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) - expect(BackgroundMigrationWorker.jobs.size).to eq(1) - end - end - - # E.g. from a previous failed run of this background migration - context 'when there is existing data in unhashed_upload_files' do - before do - unhashed_upload_files.create(path: '/foo/bar.jpg') - end - - it 'clears existing data before adding new data' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(1).to(5) - end - end - end - - # E.g. The installation is in use at the time of migration, and someone has - # just uploaded a file - context 'when there are files in /uploads/tmp' do - before do - FileUtils.touch(Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg')) - end - - it 'does not add files from /uploads/tmp' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { unhashed_upload_files.count }.from(0).to(5) - end - end - end - end - - # Very new or lightly-used installations that are running this migration - # may not have an upload directory because they have no uploads. - context 'when no files were ever uploaded' do - it 'does not add to the unhashed_upload_files table (and does not raise error)' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.not_to change { unhashed_upload_files.count }.from(0) - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb new file mode 100644 index 00000000000..087fa35dd15 --- /dev/null +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -0,0 +1,111 @@ +require 'spec_helper' + +describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } + + let(:user1) { create(:user) } + let(:user2) { create(:user) } + let(:project1) { create(:project) } + let(:project2) { create(:project) } + let(:appearance) { create(:appearance) } + + matcher :be_scheduled_migration do |*expected| + match do |migration| + BackgroundMigrationWorker.jobs.any? do |job| + job['args'] == [migration, expected] + end + end + + failure_message do |migration| + "Migration `#{migration}` with args `#{expected.inspect}` not scheduled!" + end + end + + context 'when files were uploaded before and after hashed storage was enabled' do + before do + fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + uploaded_file = fixture_file_upload(fixture) + + user1.update(avatar: uploaded_file) + project1.update(avatar: uploaded_file) + appearance.update(logo: uploaded_file, header_logo: uploaded_file) + uploaded_file = fixture_file_upload(fixture) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + + stub_application_setting(hashed_storage_enabled: true) + + # Hashed files + uploaded_file = fixture_file_upload(fixture) + UploadService.new(project2, uploaded_file, FileUploader).execute + end + + it 'adds unhashed files to the untracked_files_for_uploads table' do + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(0).to(5) + end + end + + it 'does not add hashed files to the untracked_files_for_uploads table' do + Sidekiq::Testing.fake! do + described_class.new.perform + + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end + end + + it 'correctly schedules the follow-up background migration jobs' do + Sidekiq::Testing.fake! do + described_class.new.perform + + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end + end + + # E.g. from a previous failed run of this background migration + context 'when there is existing data in untracked_files_for_uploads' do + before do + untracked_files_for_uploads.create(path: '/foo/bar.jpg') + end + + it 'clears existing data before adding new data' do + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(1).to(5) + end + end + end + + # E.g. The installation is in use at the time of migration, and someone has + # just uploaded a file + context 'when there are files in /uploads/tmp' do + before do + FileUtils.touch(Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg')) + end + + it 'does not add files from /uploads/tmp' do + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(0).to(5) + end + end + end + end + + # Very new or lightly-used installations that are running this migration + # may not have an upload directory because they have no uploads. + context 'when no files were ever uploaded' do + it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do + Sidekiq::Testing.fake! do + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(0) + end + end + end +end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 49758ede1a4..95496696bc6 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -4,8 +4,8 @@ require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_up describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers - class UnhashedUploadFile < ActiveRecord::Base - self.table_name = 'unhashed_upload_files' + class UntrackedFile < ActiveRecord::Base + self.table_name = 'untracked_files_for_uploads' end matcher :be_scheduled_migration do @@ -29,10 +29,10 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end - it 'ensures the unhashed_upload_files table exists' do + it 'ensures the untracked_files_for_uploads table exists' do expect do migrate! - end.to change { table_exists?(:unhashed_upload_files) }.from(false).to(true) + end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) end it 'has a path field long enough for really long paths' do @@ -48,7 +48,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do component # filename ].flatten.join('/') - record = UnhashedUploadFile.create!(path: long_path) + record = UntrackedFile.create!(path: long_path) expect(record.reload.path.size).to eq(5711) end @@ -132,12 +132,12 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end - it 'all UnhashedUploadFile records are marked as tracked' do + it 'all UntrackedFile records are marked as tracked' do Sidekiq::Testing.inline! do migrate! - expect(UnhashedUploadFile.count).to eq(8) - expect(UnhashedUploadFile.count).to eq(UnhashedUploadFile.where(tracked: true).count) + expect(UntrackedFile.count).to eq(8) + expect(UntrackedFile.count).to eq(UntrackedFile.where(tracked: true).count) end end end -- cgit v1.2.1 From 1807bc16466a37684c5e1de6b498230dfbd3be06 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:20:00 -0800 Subject: Reword test --- spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 087fa35dd15..3d2504f84a1 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -71,7 +71,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side untracked_files_for_uploads.create(path: '/foo/bar.jpg') end - it 'clears existing data before adding new data' do + it 'does not error or produce duplicates of existing data' do Sidekiq::Testing.fake! do expect do described_class.new.perform -- cgit v1.2.1 From c77a353dca25c2e08c409f1f3fc5a61a3eea69dc Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:21:32 -0800 Subject: Remove unnecessary clearing Since duplicate inserts are now ignored. --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 5 ----- .../gitlab/background_migration/prepare_untracked_uploads_spec.rb | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 6a7fef18e53..11978b73a8a 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -17,7 +17,6 @@ module Gitlab def perform return unless migrate? - clear_untracked_file_paths store_untracked_file_paths schedule_populate_untracked_uploads_jobs end @@ -28,10 +27,6 @@ module Gitlab UntrackedFile.table_exists? end - def clear_untracked_file_paths - UntrackedFile.delete_all - end - def store_untracked_file_paths return unless Dir.exist?(UPLOAD_DIR) diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 3d2504f84a1..cf1cad15c9a 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -68,14 +68,14 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # E.g. from a previous failed run of this background migration context 'when there is existing data in untracked_files_for_uploads' do before do - untracked_files_for_uploads.create(path: '/foo/bar.jpg') + described_class.new.perform end it 'does not error or produce duplicates of existing data' do Sidekiq::Testing.fake! do expect do described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(1).to(5) + end.not_to change { untracked_files_for_uploads.count }.from(5) end end end -- cgit v1.2.1 From 1ba4d417fe7bec333c410cff861d43f0bd1e1c03 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 9 Nov 2017 17:26:15 -0800 Subject: Clean up after test --- .../gitlab/background_migration/prepare_untracked_uploads_spec.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index cf1cad15c9a..913622cdb95 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -83,8 +83,14 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # E.g. The installation is in use at the time of migration, and someone has # just uploaded a file context 'when there are files in /uploads/tmp' do + let(:tmp_file) { Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg') } + before do - FileUtils.touch(Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg')) + FileUtils.touch(tmp_file) + end + + after do + FileUtils.rm(tmp_file) end it 'does not add files from /uploads/tmp' do -- cgit v1.2.1 From 2b481e7b28114982e95af5ab17cdab2dc832bf78 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 13 Nov 2017 15:39:26 -0800 Subject: Exclude `untracked_files_for_uploads` from schema Because it is a temporary table meant only to facilitate a migration of data. It is referenced only by the post-deploy migration and 2 related background migrations. It should be dropped when the data migration is finished. --- db/schema.rb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index e193d569739..effb2604af2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1719,16 +1719,6 @@ ActiveRecord::Schema.define(version: 20171124150326) do add_index "u2f_registrations", ["key_handle"], name: "index_u2f_registrations_on_key_handle", using: :btree add_index "u2f_registrations", ["user_id"], name: "index_u2f_registrations_on_user_id", using: :btree - create_table "untracked_files_for_uploads", force: :cascade do |t| - t.string "path", null: false - t.boolean "tracked", default: false, null: false - t.datetime_with_timezone "created_at", null: false - t.datetime_with_timezone "updated_at", null: false - end - - add_index "untracked_files_for_uploads", ["path"], name: "index_untracked_files_for_uploads_on_path", unique: true, using: :btree - add_index "untracked_files_for_uploads", ["tracked"], name: "index_untracked_files_for_uploads_on_tracked", using: :btree - create_table "uploads", force: :cascade do |t| t.integer "size", limit: 8, null: false t.string "path", null: false -- cgit v1.2.1 From 81f061d5a4ebefefe0efbc3c1f28b86b665a2b92 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 14:47:32 -0800 Subject: Fix `ionice` prepend --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 11978b73a8a..6aba3011720 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -65,7 +65,7 @@ module Gitlab tmp_path = "#{UPLOAD_DIR}/tmp/*" cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] - %w[ionice -c Idle] + cmd if ionice_is_available? + cmd = %w[ionice -c Idle] + cmd if ionice_is_available? cmd end -- cgit v1.2.1 From 3dc0b118eca3716c0cda841232e0789739627957 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 16:11:53 -0800 Subject: Store paths relative to CarrierWave.root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So the path on source installs cannot be too long for our column. And fix the column length test since Route.path is limited to 255 chars, it doesn’t matter how many nested groups there are. --- .../populate_untracked_uploads.rb | 12 +-- .../prepare_untracked_uploads.rb | 16 ++-- .../populate_untracked_uploads_spec.rb | 102 +++++++++++---------- .../prepare_untracked_uploads_spec.rb | 11 ++- spec/migrations/track_untracked_uploads_spec.rb | 6 +- 5 files changed, 79 insertions(+), 68 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index bd0f2f591a4..c5f9d998d9e 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -90,7 +90,7 @@ module Gitlab matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) matchd[0].sub(%r{\A/}, '') # remove leading slash else - path_relative_to_carrierwave_root + path end end @@ -113,20 +113,16 @@ module Gitlab end def file_size - File.size(path) + absolute_path = File.join(CarrierWave.root, path) + File.size(absolute_path) end # Not including a leading slash def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR)}/} + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end - # Not including a leading slash - def path_relative_to_carrierwave_root - "uploads/#{path_relative_to_upload_dir}" - end - private def matching_pattern_map diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 6aba3011720..e9d162661d1 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -5,8 +5,12 @@ module Gitlab include Database::MigrationHelpers FILE_PATH_BATCH_SIZE = 500 - UPLOAD_DIR = "#{CarrierWave.root}/uploads".freeze + RELATIVE_UPLOAD_DIR = "uploads".freeze + ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze + START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/} + EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze + EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze class UntrackedFile < ActiveRecord::Base include EachBatch @@ -28,9 +32,9 @@ module Gitlab end def store_untracked_file_paths - return unless Dir.exist?(UPLOAD_DIR) + return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) - each_file_batch(UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + each_file_batch(ABSOLUTE_UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| insert_file_paths(file_paths) end end @@ -49,7 +53,7 @@ module Gitlab paths = [] stdout.each_line("\0") do |line| - paths << line.chomp("\0") + paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '') if paths.size >= batch_size yield(paths) @@ -61,9 +65,7 @@ module Gitlab end def build_find_command(search_dir) - hashed_path = "#{UPLOAD_DIR}/@hashed/*" - tmp_path = "#{UPLOAD_DIR}/tmp/*" - cmd = %W[find #{search_dir} -type f ! ( -path #{hashed_path} -prune ) ! ( -path #{tmp_path} -prune ) -print0] + cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0] cmd = %w[ionice -c Idle] + cmd if ionice_is_available? diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index b3122e90c83..953793a353c 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -29,14 +29,14 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid appearance.update!(header_logo: uploaded_file) # File records created by PrepareUntrackedUploads - untracked_files_for_uploads.create!(path: appearance.logo.file.file) - untracked_files_for_uploads.create!(path: appearance.header_logo.file.file) - untracked_files_for_uploads.create!(path: user1.avatar.file.file) - untracked_files_for_uploads.create!(path: user2.avatar.file.file) - untracked_files_for_uploads.create!(path: project1.avatar.file.file) - untracked_files_for_uploads.create!(path: project2.avatar.file.file) - untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") - untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") + untracked_files_for_uploads.create!(path: found_path(appearance.logo.file.file)) + untracked_files_for_uploads.create!(path: found_path(appearance.header_logo.file.file)) + untracked_files_for_uploads.create!(path: found_path(user1.avatar.file.file)) + untracked_files_for_uploads.create!(path: found_path(user2.avatar.file.file)) + untracked_files_for_uploads.create!(path: found_path(project1.avatar.file.file)) + untracked_files_for_uploads.create!(path: found_path(project2.avatar.file.file)) + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") user2.uploads.delete_all project2.uploads.delete_all @@ -122,7 +122,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do let(:user1) { create(:user) } context 'when the file is already in the uploads table' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/#{user1.id}/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg") } before do upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234) @@ -142,7 +142,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance logo file path' do let(:model) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: model.logo.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.logo.file.file)) } before do model.update!(logo: uploaded_file) @@ -163,7 +163,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance header_logo file path' do let(:model) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: model.header_logo.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.header_logo.file.file)) } before do model.update!(header_logo: uploaded_file) @@ -184,7 +184,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a pre-Markdown Note attachment file path' do let(:model) { create(:note) } - let(:untracked_file) { described_class.create!(path: model.attachment.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.attachment.file.file)) } before do model.update!(attachment: uploaded_file) @@ -207,7 +207,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a user avatar file path' do let(:model) { create(:user) } - let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } before do model.update!(avatar: uploaded_file) @@ -228,7 +228,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a group avatar file path' do let(:model) { create(:group) } - let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } before do model.update!(avatar: uploaded_file) @@ -251,7 +251,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project avatar file path' do let(:model) { create(:project) } - let(:untracked_file) { described_class.create!(path: model.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } before do model.update!(avatar: uploaded_file) @@ -272,7 +272,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - let(:untracked_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } + let(:untracked_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } before do UploadService.new(model, uploaded_file, FileUploader).execute # Markdown upload @@ -295,7 +295,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end describe '#mark_as_tracked' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'saves the record with tracked set to true' do expect do @@ -308,7 +308,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#upload_path' do context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') @@ -316,7 +316,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') @@ -324,7 +324,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') @@ -332,7 +332,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') @@ -340,7 +340,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') @@ -348,7 +348,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns the file path relative to the CarrierWave root' do expect(untracked_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') @@ -358,7 +358,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } let(:random_hex) { SecureRandom.hex } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } it 'returns the file path relative to the project directory in uploads' do expect(untracked_file.upload_path).to eq("#{random_hex}/Some file.jpg") @@ -368,7 +368,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#uploader' do context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns AttachmentUploader as a string' do expect(untracked_file.uploader).to eq('AttachmentUploader') @@ -376,7 +376,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns AttachmentUploader as a string' do expect(untracked_file.uploader).to eq('AttachmentUploader') @@ -384,7 +384,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns AttachmentUploader as a string' do expect(untracked_file.uploader).to eq('AttachmentUploader') @@ -392,7 +392,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do expect(untracked_file.uploader).to eq('AvatarUploader') @@ -400,7 +400,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do expect(untracked_file.uploader).to eq('AvatarUploader') @@ -408,7 +408,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns AvatarUploader as a string' do expect(untracked_file.uploader).to eq('AvatarUploader') @@ -417,7 +417,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns FileUploader as a string' do expect(untracked_file.uploader).to eq('FileUploader') @@ -427,7 +427,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#model_type' do context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns Appearance as a string' do expect(untracked_file.model_type).to eq('Appearance') @@ -435,7 +435,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns Appearance as a string' do expect(untracked_file.model_type).to eq('Appearance') @@ -443,7 +443,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns Note as a string' do expect(untracked_file.model_type).to eq('Note') @@ -451,7 +451,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns User as a string' do expect(untracked_file.model_type).to eq('User') @@ -459,7 +459,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns Namespace as a string' do expect(untracked_file.model_type).to eq('Namespace') @@ -467,7 +467,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns Project as a string' do expect(untracked_file.model_type).to eq('Project') @@ -476,7 +476,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns Project as a string' do expect(untracked_file.model_type).to eq('Project') @@ -486,7 +486,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#model_id' do context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1') @@ -494,7 +494,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1') @@ -502,7 +502,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1234') @@ -510,7 +510,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1234') @@ -518,7 +518,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1234') @@ -526,7 +526,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq('1234') @@ -535,7 +535,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } it 'returns the ID as a string' do expect(untracked_file.model_id).to eq(project.id.to_s) @@ -549,7 +549,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance logo file path' do let(:appearance) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: appearance.logo.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(appearance.logo.file.file)) } before do appearance.update!(logo: uploaded_file) @@ -566,7 +566,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project avatar file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: project.avatar.file.file) } + let(:untracked_file) { described_class.create!(path: found_path(project.avatar.file.file)) } before do project.update!(avatar: uploaded_file) @@ -583,7 +583,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } before do UploadService.new(project, uploaded_file, FileUploader).execute @@ -599,3 +599,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end end end + +# The path returned by the find command in PrepareUntrackedUploads +# AKA the path relative to CarrierWave.root, without a leading slash. +def found_path(absolute_path) + absolute_path.sub(%r{\A#{CarrierWave.root}/}, '') +end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 913622cdb95..d0dd6b7c157 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -47,6 +47,15 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end end + it 'adds files with paths relative to CarrierWave.root' do + Sidekiq::Testing.fake! do + described_class.new.perform + untracked_files_for_uploads.all.each do |file| + expect(file.path.start_with?('uploads/')).to be_truthy + end + end + end + it 'does not add hashed files to the untracked_files_for_uploads table' do Sidekiq::Testing.fake! do described_class.new.perform @@ -83,7 +92,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # E.g. The installation is in use at the time of migration, and someone has # just uploaded a file context 'when there are files in /uploads/tmp' do - let(:tmp_file) { Rails.root.join(described_class::UPLOAD_DIR, 'tmp', 'some_file.jpg') } + let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } before do FileUtils.touch(tmp_file) diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 95496696bc6..83eb6bbd537 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -41,15 +41,13 @@ describe TrackUntrackedUploads, :migration, :sidekiq do component = 'a'*255 long_path = [ - CarrierWave.root, 'uploads', - [component] * Namespace::NUMBER_OF_ANCESTORS_ALLOWED, # namespaces - component, # project + component, # project.full_path component # filename ].flatten.join('/') record = UntrackedFile.create!(path: long_path) - expect(record.reload.path.size).to eq(5711) + expect(record.reload.path.size).to eq(519) end context 'with tracked and untracked uploads' do -- cgit v1.2.1 From 0e97e3089fba60c3e405f907cbc937cbc1b662ad Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 16:15:33 -0800 Subject: Fix MySQL path field length MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I believe the field only needs to fit 519 at the moment but I’m rounding up to be a little safer. See the migration spec for more detail on the magic number 519. --- db/post_migrate/20171103140253_track_untracked_uploads.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb index 5e4e357b8bc..09ff21b103f 100644 --- a/db/post_migrate/20171103140253_track_untracked_uploads.rb +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -12,7 +12,7 @@ class TrackUntrackedUploads < ActiveRecord::Migration def up unless table_exists?(:untracked_files_for_uploads) create_table :untracked_files_for_uploads do |t| - t.string :path, null: false + t.string :path, limit: 600, null: false t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false end -- cgit v1.2.1 From dd3ba1f200f0d753a1fafc058b4423f230398dce Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:18:05 -0800 Subject: Fix uploads.path length for long filenames This will prevent our other migration for adding old files to the uploads table from breaking. --- ...171103000000_set_uploads_path_size_for_mysql.rb | 25 ++++++++++++++++++++++ db/schema.rb | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20171103000000_set_uploads_path_size_for_mysql.rb diff --git a/db/migrate/20171103000000_set_uploads_path_size_for_mysql.rb b/db/migrate/20171103000000_set_uploads_path_size_for_mysql.rb new file mode 100644 index 00000000000..1fbe505f804 --- /dev/null +++ b/db/migrate/20171103000000_set_uploads_path_size_for_mysql.rb @@ -0,0 +1,25 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class SetUploadsPathSizeForMysql < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def up + # We need at least 297 at the moment. For more detail on that number, see: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/40168#what-is-the-expected-correct-behavior + # + # Rails + PostgreSQL `string` is equivalent to a `text` field, but + # Rails + MySQL `string` is `varchar(255)` by default. Also, note that we + # have an upper limit because with a unique index, MySQL has a max key + # length of 3072 bytes which seems to correspond to `varchar(1024)`. + change_column :uploads, :path, :string, limit: 511 + end + + def down + # It was unspecified, which is varchar(255) by default in Rails for MySQL. + change_column :uploads, :path, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index effb2604af2..fed7284a1a1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1721,7 +1721,7 @@ ActiveRecord::Schema.define(version: 20171124150326) do create_table "uploads", force: :cascade do |t| t.integer "size", limit: 8, null: false - t.string "path", null: false + t.string "path", limit: 511, null: false t.string "checksum", limit: 64 t.integer "model_id" t.string "model_type" -- cgit v1.2.1 From 0715034805a8f68c66118cf78777bca92ad7cef1 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:20:15 -0800 Subject: Remove irrelevant copy-pasted code --- .../populate_untracked_uploads.rb | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index c5f9d998d9e..f2207fa4e72 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -153,29 +153,9 @@ module Gitlab belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations - validates :size, presence: true - validates :path, presence: true - validates :model, presence: true - validates :uploader, presence: true - before_save :calculate_checksum, if: :foreground_checksum? after_commit :schedule_checksum, unless: :foreground_checksum? - def self.remove_path(path) - where(path: path).destroy_all - end - - def self.record(uploader) - remove_path(uploader.relative_path) - - create( - size: uploader.file.size, - path: uploader.relative_path, - model: uploader.model, - uploader: uploader.class.to_s - ) - end - def absolute_path return path unless relative_path? -- cgit v1.2.1 From b63e8f4adfda2f907280824e6acf69bbaa56de3a Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:49:10 -0800 Subject: Fallback on checksum jobs Since `calculate_checksum` depends on `Uploader` classes which are not defined in this background migration and may change at any time. --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index f2207fa4e72..8529e8d1d0b 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -166,6 +166,8 @@ module Gitlab return unless exist? self.checksum = Digest::SHA256.file(absolute_path).hexdigest + rescue StandardError + schedule_checksum end def exist? -- cgit v1.2.1 From c25b7c0e3f6d43b5fb77e53bbd0dd4495b8e0c69 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 22:49:24 -0800 Subject: Speed up inserts --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index e9d162661d1..983e63143e0 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -80,8 +80,10 @@ module Gitlab end def insert_file_paths(file_paths) - file_paths.each do |file_path| - insert_file_path(file_path) + ActiveRecord::Base.transaction do + file_paths.each do |file_path| + insert_file_path(file_path) + end end end -- cgit v1.2.1 From d530085685105e2d7cd6d87ba866756683f0488d Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 14 Nov 2017 23:15:30 -0800 Subject: Refactor specs --- .../populate_untracked_uploads_spec.rb | 384 ++++++++------------- .../prepare_untracked_uploads_spec.rb | 26 +- spec/migrations/track_untracked_uploads_spec.rb | 98 ++---- spec/support/track_untracked_uploads_helpers.rb | 12 +- 4 files changed, 183 insertions(+), 337 deletions(-) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 953793a353c..c794a2f152b 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,46 +1,36 @@ require 'spec_helper' describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do + include TrackUntrackedUploadsHelpers + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } let!(:uploads) { table(:uploads) } - let(:user1) { create(:user) } - let(:user2) { create(:user) } - let(:project1) { create(:project) } - let(:project2) { create(:project) } - let(:appearance) { create(:appearance) } - context 'with untracked files and tracked files in untracked_files_for_uploads' do - before do - fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user1) { create(:user, :with_avatar) } + let!(:user2) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let!(:project2) { create(:project, :with_avatar) } - # Tracked, by doing normal file upload - uploaded_file = fixture_file_upload(fixture) - user1.update!(avatar: uploaded_file) - project1.update!(avatar: uploaded_file) + before do UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - appearance.update!(logo: uploaded_file) - - # Untracked, by doing normal file upload then later deleting records from DB - uploaded_file = fixture_file_upload(fixture) - user2.update!(avatar: uploaded_file) - project2.update!(avatar: uploaded_file) UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload - appearance.update!(header_logo: uploaded_file) # File records created by PrepareUntrackedUploads - untracked_files_for_uploads.create!(path: found_path(appearance.logo.file.file)) - untracked_files_for_uploads.create!(path: found_path(appearance.header_logo.file.file)) - untracked_files_for_uploads.create!(path: found_path(user1.avatar.file.file)) - untracked_files_for_uploads.create!(path: found_path(user2.avatar.file.file)) - untracked_files_for_uploads.create!(path: found_path(project1.avatar.file.file)) - untracked_files_for_uploads.create!(path: found_path(project2.avatar.file.file)) + untracked_files_for_uploads.create!(path: appearance.uploads.first.path) + untracked_files_for_uploads.create!(path: appearance.uploads.last.path) + untracked_files_for_uploads.create!(path: user1.uploads.first.path) + untracked_files_for_uploads.create!(path: user2.uploads.first.path) + untracked_files_for_uploads.create!(path: project1.uploads.first.path) + untracked_files_for_uploads.create!(path: project2.uploads.first.path) untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}") untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}") + # Untrack 4 files user2.uploads.delete_all - project2.uploads.delete_all - appearance.uploads.last.destroy + project2.uploads.delete_all # 2 files: avatar and a Markdown upload + appearance.uploads.where("path like '%header_logo%'").delete_all end it 'adds untracked files to the uploads table' do @@ -119,33 +109,36 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } describe '#ensure_tracked!' do - let(:user1) { create(:user) } + let!(:user1) { create(:user, :with_avatar) } + let!(:untracked_file) { described_class.create!(path: user1.uploads.first.path) } context 'when the file is already in the uploads table' do - let(:untracked_file) { described_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg") } + it 'does not add an upload' do + expect do + untracked_file.ensure_tracked! + end.not_to change { upload_class.count }.from(1) + end + end + context 'when the file is not already in the uploads table' do before do - upload_class.create!(path: "uploads/-/system/user/avatar/#{user1.id}/avatar.jpg", uploader: 'AvatarUploader', model_type: 'User', model_id: user1.id, size: 1234) + user1.uploads.delete_all end - it 'does not add an upload' do + it 'adds an upload' do expect do untracked_file.ensure_tracked! - end.not_to change { upload_class.count }.from(1) + end.to change { upload_class.count }.from(0).to(1) end end end describe '#add_to_uploads' do - let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } - let(:uploaded_file) { fixture_file_upload(fixture) } - - context 'for an appearance logo file path' do - let(:model) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: found_path(model.logo.file.file)) } + shared_examples_for 'add_to_uploads_non_markdown_files' do + let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') } + let!(:untracked_file) { described_class.create!(path: expected_upload_attrs['path']) } before do - model.update!(logo: uploaded_file) model.uploads.delete_all end @@ -154,129 +147,68 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/logo/#{model.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) + expect(model.uploads.first.attributes).to include(expected_upload_attrs) end end - context 'for an appearance header_logo file path' do - let(:model) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: found_path(model.header_logo.file.file)) } + context 'for an appearance logo file path' do + let(:model) { create(:appearance, logo: uploaded_file) } - before do - model.update!(header_logo: uploaded_file) - model.uploads.delete_all - end + it_behaves_like 'add_to_uploads_non_markdown_files' + end - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { model.reload.uploads.count }.from(0).to(1) + context 'for an appearance header_logo file path' do + let(:model) { create(:appearance, header_logo: uploaded_file) } - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/header_logo/#{model.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a pre-Markdown Note attachment file path' do - let(:model) { create(:note) } - let(:untracked_file) { described_class.create!(path: found_path(model.attachment.file.file)) } - - before do - model.update!(attachment: uploaded_file) - upload_class.delete_all + class Note < ActiveRecord::Base + has_many :uploads, as: :model, dependent: :destroy end - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { upload_class.count }.from(0).to(1) + let(:model) { create(:note, :with_attachment) } - expect(upload_class.first.attributes).to include({ - "path" => "uploads/-/system/note/attachment/#{model.id}/rails_sample.jpg", - "model_id" => model.id, - "model_type" => "Note", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a user avatar file path' do - let(:model) { create(:user) } - let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } + let(:model) { create(:user, :with_avatar) } - before do - model.update!(avatar: uploaded_file) - model.uploads.delete_all - end - - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { model.reload.uploads.count }.from(0).to(1) - - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/#{model.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a group avatar file path' do - let(:model) { create(:group) } - let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } - - before do - model.update!(avatar: uploaded_file) - model.uploads.delete_all - end + let(:model) { create(:group, :with_avatar) } - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { model.reload.uploads.count }.from(0).to(1) - - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/group/avatar/#{model.id}/rails_sample.jpg", - "model_id" => model.id, - "model_type" => "Namespace", # Explicitly calling this out because it was unexpected to me (I assumed it should be "Group") - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a project avatar file path' do - let(:model) { create(:project) } - let(:untracked_file) { described_class.create!(path: found_path(model.avatar.file.file)) } - - before do - model.update!(avatar: uploaded_file) - model.uploads.delete_all - end - - it 'creates an Upload record' do - expect do - untracked_file.add_to_uploads - end.to change { model.reload.uploads.count }.from(0).to(1) + let(:model) { create(:project, :with_avatar) } - expect(model.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/#{model.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - end + it_behaves_like 'add_to_uploads_non_markdown_files' end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - let(:untracked_file) { described_class.new(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") } + let(:expected_upload_attrs) { {} } + + # UntrackedFile.path is different than Upload.path + let(:untracked_file) { create_untracked_file("/#{model.full_path}/#{model.uploads.first.path}") } before do - UploadService.new(model, uploaded_file, FileUploader).execute # Markdown upload - untracked_file.save! + # Upload the file + UploadService.new(model, uploaded_file, FileUploader).execute + + # Create the untracked_files_for_uploads record + untracked_file + + # Save the expected upload attributes + expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + + # Untrack the file model.reload.uploads.delete_all end @@ -286,18 +218,15 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end.to change { model.reload.uploads.count }.from(0).to(1) hex_secret = untracked_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] - expect(model.uploads.first.attributes).to include({ - "path" => "#{hex_secret}/rails_sample.jpg", - "uploader" => "FileUploader" - }.merge(rails_sample_jpg_attrs)) + expect(model.uploads.first.attributes).to include(expected_upload_attrs) end end end describe '#mark_as_tracked' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } - it 'saves the record with tracked set to true' do + untracked_file = create_untracked_file("/-/system/appearance/logo/1/some_logo.jpg") + expect do untracked_file.mark_as_tracked end.to change { untracked_file.tracked }.from(false).to(true) @@ -307,253 +236,218 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end describe '#upload_path' do - context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + def assert_upload_path(file_path, expected_upload_path) + untracked_file = create_untracked_file(file_path) + expect(untracked_file.upload_path).to eq(expected_upload_path) + end + + context 'for an appearance logo file path' do it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/logo/1/some_logo.jpg') + assert_upload_path('/-/system/appearance/logo/1/some_logo.jpg', 'uploads/-/system/appearance/logo/1/some_logo.jpg') end end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/appearance/header_logo/1/some_logo.jpg') + assert_upload_path('/-/system/appearance/header_logo/1/some_logo.jpg', 'uploads/-/system/appearance/header_logo/1/some_logo.jpg') end end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/note/attachment/1234/some_attachment.pdf') + assert_upload_path('/-/system/note/attachment/1234/some_attachment.pdf', 'uploads/-/system/note/attachment/1234/some_attachment.pdf') end end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/user/avatar/1234/avatar.jpg') + assert_upload_path('/-/system/user/avatar/1234/avatar.jpg', 'uploads/-/system/user/avatar/1234/avatar.jpg') end end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/group/avatar/1234/avatar.jpg') + assert_upload_path('/-/system/group/avatar/1234/avatar.jpg', 'uploads/-/system/group/avatar/1234/avatar.jpg') end end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } - it 'returns the file path relative to the CarrierWave root' do - expect(untracked_file.upload_path).to eq('uploads/-/system/project/avatar/1234/avatar.jpg') + assert_upload_path('/-/system/project/avatar/1234/avatar.jpg', 'uploads/-/system/project/avatar/1234/avatar.jpg') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:random_hex) { SecureRandom.hex } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{random_hex}/Some file.jpg") } - it 'returns the file path relative to the project directory in uploads' do - expect(untracked_file.upload_path).to eq("#{random_hex}/Some file.jpg") + project = create(:project) + random_hex = SecureRandom.hex + + assert_upload_path("/#{project.full_path}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg") end end end describe '#uploader' do - context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + def assert_uploader(file_path, expected_uploader) + untracked_file = create_untracked_file(file_path) + + expect(untracked_file.uploader).to eq(expected_uploader) + end + context 'for an appearance logo file path' do it 'returns AttachmentUploader as a string' do - expect(untracked_file.uploader).to eq('AttachmentUploader') + assert_uploader('/-/system/appearance/logo/1/some_logo.jpg', 'AttachmentUploader') end end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } - it 'returns AttachmentUploader as a string' do - expect(untracked_file.uploader).to eq('AttachmentUploader') + assert_uploader('/-/system/appearance/header_logo/1/some_logo.jpg', 'AttachmentUploader') end end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } - it 'returns AttachmentUploader as a string' do - expect(untracked_file.uploader).to eq('AttachmentUploader') + assert_uploader('/-/system/note/attachment/1234/some_attachment.pdf', 'AttachmentUploader') end end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } - it 'returns AvatarUploader as a string' do - expect(untracked_file.uploader).to eq('AvatarUploader') + assert_uploader('/-/system/user/avatar/1234/avatar.jpg', 'AvatarUploader') end end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns AvatarUploader as a string' do - expect(untracked_file.uploader).to eq('AvatarUploader') + assert_uploader('/-/system/group/avatar/1234/avatar.jpg', 'AvatarUploader') end end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } - it 'returns AvatarUploader as a string' do - expect(untracked_file.uploader).to eq('AvatarUploader') + assert_uploader('/-/system/project/avatar/1234/avatar.jpg', 'AvatarUploader') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } - it 'returns FileUploader as a string' do - expect(untracked_file.uploader).to eq('FileUploader') + project = create(:project) + + assert_uploader("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader') end end end describe '#model_type' do - context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + def assert_model_type(file_path, expected_model_type) + untracked_file = create_untracked_file(file_path) + expect(untracked_file.model_type).to eq(expected_model_type) + end + + context 'for an appearance logo file path' do it 'returns Appearance as a string' do - expect(untracked_file.model_type).to eq('Appearance') + assert_model_type('/-/system/appearance/logo/1/some_logo.jpg', 'Appearance') end end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } - it 'returns Appearance as a string' do - expect(untracked_file.model_type).to eq('Appearance') + assert_model_type('/-/system/appearance/header_logo/1/some_logo.jpg', 'Appearance') end end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } - it 'returns Note as a string' do - expect(untracked_file.model_type).to eq('Note') + assert_model_type('/-/system/note/attachment/1234/some_attachment.pdf', 'Note') end end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } - it 'returns User as a string' do - expect(untracked_file.model_type).to eq('User') + assert_model_type('/-/system/user/avatar/1234/avatar.jpg', 'User') end end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns Namespace as a string' do - expect(untracked_file.model_type).to eq('Namespace') + assert_model_type('/-/system/group/avatar/1234/avatar.jpg', 'Namespace') end end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } - it 'returns Project as a string' do - expect(untracked_file.model_type).to eq('Project') + assert_model_type('/-/system/project/avatar/1234/avatar.jpg', 'Project') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } - it 'returns Project as a string' do - expect(untracked_file.model_type).to eq('Project') + project = create(:project) + + assert_model_type("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'Project') end end end describe '#model_id' do - context 'for an appearance logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/logo/1/some_logo.jpg") } + def assert_model_id(file_path, expected_model_id) + untracked_file = create_untracked_file(file_path) + + expect(untracked_file.model_id).to eq(expected_model_id) + end + context 'for an appearance logo file path' do it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1') + assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', '1') end end context 'for an appearance header_logo file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/appearance/header_logo/1/some_logo.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1') + assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', '1') end end context 'for a pre-Markdown Note attachment file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/note/attachment/1234/some_attachment.pdf") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1234') + assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', '1234') end end context 'for a user avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/user/avatar/1234/avatar.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1234') + assert_model_id('/-/system/user/avatar/1234/avatar.jpg', '1234') end end context 'for a group avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/group/avatar/1234/avatar.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1234') + assert_model_id('/-/system/group/avatar/1234/avatar.jpg', '1234') end end context 'for a project avatar file path' do - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/-/system/project/avatar/1234/avatar.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq('1234') + assert_model_id('/-/system/project/avatar/1234/avatar.jpg', '1234') end end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do - let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg") } - it 'returns the ID as a string' do - expect(untracked_file.model_id).to eq(project.id.to_s) + project = create(:project) + + assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id.to_s) end end end describe '#file_size' do - let(:fixture) { Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') } - let(:uploaded_file) { fixture_file_upload(fixture) } - context 'for an appearance logo file path' do - let(:appearance) { create(:appearance) } - let(:untracked_file) { described_class.create!(path: found_path(appearance.logo.file.file)) } - - before do - appearance.update!(logo: uploaded_file) - end + let(:appearance) { create(:appearance, logo: uploaded_file) } + let(:untracked_file) { described_class.create!(path: appearance.uploads.first.path) } it 'returns the file size' do expect(untracked_file.file_size).to eq(35255) @@ -565,12 +459,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end context 'for a project avatar file path' do - let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: found_path(project.avatar.file.file)) } - - before do - project.update!(avatar: uploaded_file) - end + let(:project) { create(:project, avatar: uploaded_file) } + let(:untracked_file) { described_class.create!(path: project.uploads.first.path) } it 'returns the file size' do expect(untracked_file.file_size).to eq(35255) @@ -583,7 +473,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:project) { create(:project) } - let(:untracked_file) { described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project.full_path}/#{project.uploads.first.path}") } + let(:untracked_file) { create_untracked_file("/#{project.full_path}/#{project.uploads.first.path}") } before do UploadService.new(project, uploaded_file, FileUploader).execute @@ -598,10 +488,8 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end end end -end -# The path returned by the find command in PrepareUntrackedUploads -# AKA the path relative to CarrierWave.root, without a leading slash. -def found_path(absolute_path) - absolute_path.sub(%r{\A#{CarrierWave.root}/}, '') + def create_untracked_file(path_relative_to_upload_dir) + described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}#{path_relative_to_upload_dir}") + end end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index d0dd6b7c157..d61135912dd 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -1,13 +1,9 @@ require 'spec_helper' describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do - let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } + include TrackUntrackedUploadsHelpers - let(:user1) { create(:user) } - let(:user2) { create(:user) } - let(:project1) { create(:project) } - let(:project2) { create(:project) } - let(:appearance) { create(:appearance) } + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } matcher :be_scheduled_migration do |*expected| match do |migration| @@ -22,20 +18,18 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end context 'when files were uploaded before and after hashed storage was enabled' do - before do - fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') - uploaded_file = fixture_file_upload(fixture) + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let(:project2) { create(:project) } # instantiate after enabling hashed_storage - user1.update(avatar: uploaded_file) - project1.update(avatar: uploaded_file) - appearance.update(logo: uploaded_file, header_logo: uploaded_file) - uploaded_file = fixture_file_upload(fixture) - UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + before do + # Markdown upload before enabling hashed_storage + UploadService.new(project1, uploaded_file, FileUploader).execute stub_application_setting(hashed_storage_enabled: true) - # Hashed files - uploaded_file = fixture_file_upload(fixture) + # Markdown upload after enabling hashed_storage UploadService.new(project2, uploaded_file, FileUploader).execute end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 83eb6bbd537..a6e880279b6 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -4,9 +4,7 @@ require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_up describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers - class UntrackedFile < ActiveRecord::Base - self.table_name = 'untracked_files_for_uploads' - end + let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } matcher :be_scheduled_migration do match do |migration| @@ -46,39 +44,36 @@ describe TrackUntrackedUploads, :migration, :sidekiq do component # filename ].flatten.join('/') - record = UntrackedFile.create!(path: long_path) + record = untracked_files_for_uploads.create!(path: long_path) expect(record.reload.path.size).to eq(519) end context 'with tracked and untracked uploads' do - let(:user1) { create(:user) } - let(:user2) { create(:user) } - let(:project1) { create(:project) } - let(:project2) { create(:project) } - let(:appearance) { create(:appearance) } + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user1) { create(:user, :with_avatar) } + let!(:user2) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let!(:project2) { create(:project, :with_avatar) } let(:uploads) { table(:uploads) } before do - fixture = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') - - # Tracked, by doing normal file upload - uploaded_file = fixture_file_upload(fixture) - user1.update(avatar: uploaded_file) - project1.update(avatar: uploaded_file) - upload_result = UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - @project1_markdown_upload_path = upload_result[:url].sub(%r{\A/uploads/}, '') - appearance.update(logo: uploaded_file) - - # Untracked, by doing normal file upload then deleting records from DB - uploaded_file = fixture_file_upload(fixture) - user2.update(avatar: uploaded_file) + UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload + UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload + + # Save expected Upload attributes + @appearance_logo_attributes = appearance.uploads.where("path like '%/logo/%'").first.attributes.slice('path', 'uploader', 'size', 'checksum') + @appearance_header_logo_attributes = appearance.uploads.where("path like '%/header_logo/%'").first.attributes.slice('path', 'uploader', 'size', 'checksum') + @user1_avatar_attributes = user1.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @user2_avatar_attributes = user2.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @project1_avatar_attributes = project1.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @project2_avatar_attributes = project2.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @project1_markdown_attributes = project1.uploads.last.attributes.slice('path', 'uploader', 'size', 'checksum') + @project2_markdown_attributes = project2.uploads.last.attributes.slice('path', 'uploader', 'size', 'checksum') + + # Untrack 4 files user2.uploads.delete_all - project2.update(avatar: uploaded_file) - upload_result = UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload - @project2_markdown_upload_path = upload_result[:url].sub(%r{\A/uploads/}, '') - project2.uploads.delete_all - appearance.update(header_logo: uploaded_file) - appearance.uploads.last.destroy + project2.uploads.delete_all # 2 files: avatar and a Markdown upload + appearance.uploads.where("path like '%header_logo%'").delete_all end it 'tracks untracked uploads' do @@ -87,23 +82,10 @@ describe TrackUntrackedUploads, :migration, :sidekiq do migrate! end.to change { uploads.count }.from(4).to(8) - expect(user2.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/#{user2.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - expect(project2.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/#{project2.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - expect(appearance.reload.uploads.count).to eq(2) - expect(appearance.uploads.last.attributes).to include({ - "path" => "uploads/-/system/appearance/header_logo/#{appearance.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) - expect(project2.uploads.last.attributes).to include({ - "path" => @project2_markdown_upload_path, - "uploader" => "FileUploader" - }.merge(rails_sample_jpg_attrs)) + expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) + expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) + expect(project2.reload.uploads.first.attributes).to include(@project2_avatar_attributes) + expect(project2.uploads.last.attributes).to include(@project2_markdown_attributes) end end @@ -111,31 +93,19 @@ describe TrackUntrackedUploads, :migration, :sidekiq do Sidekiq::Testing.inline! do migrate! - expect(user1.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/user/avatar/#{user1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - expect(project1.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/project/avatar/#{project1.id}/rails_sample.jpg", - "uploader" => "AvatarUploader" - }.merge(rails_sample_jpg_attrs)) - expect(appearance.reload.uploads.first.attributes).to include({ - "path" => "uploads/-/system/appearance/logo/#{appearance.id}/rails_sample.jpg", - "uploader" => "AttachmentUploader" - }.merge(rails_sample_jpg_attrs)) - expect(project1.uploads.last.attributes).to include({ - "path" => @project1_markdown_upload_path, - "uploader" => "FileUploader" - }.merge(rails_sample_jpg_attrs)) + expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) + expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) + expect(project1.reload.uploads.first.attributes).to include(@project1_avatar_attributes) + expect(project1.uploads.last.attributes).to include(@project1_markdown_attributes) end end - it 'all UntrackedFile records are marked as tracked' do + it 'all untracked_files_for_uploads records are marked as tracked' do Sidekiq::Testing.inline! do migrate! - expect(UntrackedFile.count).to eq(8) - expect(UntrackedFile.count).to eq(UntrackedFile.where(tracked: true).count) + expect(untracked_files_for_uploads.count).to eq(8) + expect(untracked_files_for_uploads.count).to eq(untracked_files_for_uploads.where(tracked: true).count) end end end diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index 749c5775bb0..5b832929602 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -1,12 +1,6 @@ module TrackUntrackedUploadsHelpers - def rails_sample_jpg_attrs - @rails_sample_jpg_attrs ||= { - "size" => File.size(rails_sample_file_path), - "checksum" => Digest::SHA256.file(rails_sample_file_path).hexdigest - } - end - - def rails_sample_file_path - Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + def uploaded_file + fixture_path = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') + fixture_file_upload(fixture_path) end end -- cgit v1.2.1 From dd8680a7ae4be279ae1d90f0889317a1e6ee0d95 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 02:36:25 -0800 Subject: Drop temporary tracking table when finished --- .../20171103140253_track_untracked_uploads.rb | 20 +++--- .../populate_untracked_uploads.rb | 6 ++ .../populate_untracked_uploads_spec.rb | 36 ++++++++-- .../prepare_untracked_uploads_spec.rb | 63 ++++++++-------- spec/migrations/track_untracked_uploads_spec.rb | 84 +++++++++++----------- spec/support/track_untracked_uploads_helpers.rb | 14 ++++ 6 files changed, 131 insertions(+), 92 deletions(-) diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb index 09ff21b103f..7a34abc85ee 100644 --- a/db/post_migrate/20171103140253_track_untracked_uploads.rb +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -10,6 +10,18 @@ class TrackUntrackedUploads < ActiveRecord::Migration MIGRATION = 'PrepareUntrackedUploads' def up + ensure_temporary_tracking_table_exists + + BackgroundMigrationWorker.perform_async(MIGRATION) + end + + def down + if table_exists?(:untracked_files_for_uploads) + drop_table :untracked_files_for_uploads + end + end + + def ensure_temporary_tracking_table_exists unless table_exists?(:untracked_files_for_uploads) create_table :untracked_files_for_uploads do |t| t.string :path, limit: 600, null: false @@ -25,13 +37,5 @@ class TrackUntrackedUploads < ActiveRecord::Migration unless index_exists?(:untracked_files_for_uploads, :tracked) add_index :untracked_files_for_uploads, :tracked end - - BackgroundMigrationWorker.perform_async(MIGRATION) - end - - def down - if table_exists?(:untracked_files_for_uploads) - drop_table :untracked_files_for_uploads - end end end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 8529e8d1d0b..f28892174bb 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -208,6 +208,8 @@ module Gitlab # to do. end end + + drop_temp_table_if_finished end private @@ -215,6 +217,10 @@ module Gitlab def migrate? UntrackedFile.table_exists? && Upload.table_exists? end + + def drop_temp_table_if_finished + UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.untracked.empty? + end end end end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index c794a2f152b..52f57408bfa 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,11 +1,19 @@ require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop, schema: 20171103140253 do include TrackUntrackedUploadsHelpers + subject { described_class.new } + let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } let!(:uploads) { table(:uploads) } + before do + # Prevent the TrackUntrackedUploads migration from running PrepareUntrackedUploads job + allow(BackgroundMigrationWorker).to receive(:perform_async).and_return(true) + end + context 'with untracked files and tracked files in untracked_files_for_uploads' do let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } let!(:user1) { create(:user, :with_avatar) } @@ -35,7 +43,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'adds untracked files to the uploads table' do expect do - described_class.new.perform(1, 1000) + subject.perform(1, 1000) end.to change { uploads.count }.from(4).to(8) expect(user2.uploads.count).to eq(1) @@ -44,13 +52,15 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end it 'sets all added or confirmed tracked files to tracked' do + expect(subject).to receive(:drop_temp_table_if_finished) # Don't drop the table so we can look at it + expect do - described_class.new.perform(1, 1000) + subject.perform(1, 1000) end.to change { untracked_files_for_uploads.where(tracked: true).count }.from(0).to(8) end it 'does not create duplicate uploads of already tracked files' do - described_class.new.perform(1, 1000) + subject.perform(1, 1000) expect(user1.uploads.count).to eq(1) expect(project1.uploads.count).to eq(2) @@ -62,7 +72,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end_id = untracked_files_for_uploads.all.to_a[3].id expect do - described_class.new.perform(start_id, end_id) + subject.perform(start_id, end_id) end.to change { uploads.count }.from(4).to(6) expect(user1.uploads.count).to eq(1) @@ -80,7 +90,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end_id = untracked_files_for_uploads.all.to_a[7].id expect do - described_class.new.perform(start_id, end_id) + subject.perform(start_id, end_id) end.to change { uploads.count }.from(4).to(6) expect(user1.uploads.count).to eq(1) @@ -92,12 +102,24 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid # Only 4 have been either confirmed or added to uploads expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) end + + it 'does not drop the temporary tracking table after processing the batch, if there are still untracked rows' do + subject.perform(1, untracked_files_for_uploads.last.id - 1) + + expect(table_exists?(:untracked_files_for_uploads)).to be_truthy + end + + it 'drops the temporary tracking table after processing the batch, if there are no untracked rows left' do + subject.perform(1, untracked_files_for_uploads.last.id) + + expect(table_exists?(:untracked_files_for_uploads)).to be_falsey + end end context 'with no untracked files' do it 'does not add to the uploads table (and does not raise error)' do expect do - described_class.new.perform(1, 1000) + subject.perform(1, 1000) end.not_to change { uploads.count }.from(0) end end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index d61135912dd..8fd20fd0bb3 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -17,6 +17,13 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end end + around do |example| + # Especially important so the follow-up migration does not get run + Sidekiq::Testing.fake! do + example.run + end + end + context 'when files were uploaded before and after hashed storage was enabled' do let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } let!(:user) { create(:user, :with_avatar) } @@ -34,38 +41,30 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'adds unhashed files to the untracked_files_for_uploads table' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(0).to(5) - end + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(0).to(5) end it 'adds files with paths relative to CarrierWave.root' do - Sidekiq::Testing.fake! do - described_class.new.perform - untracked_files_for_uploads.all.each do |file| - expect(file.path.start_with?('uploads/')).to be_truthy - end + described_class.new.perform + untracked_files_for_uploads.all.each do |file| + expect(file.path.start_with?('uploads/')).to be_truthy end end it 'does not add hashed files to the untracked_files_for_uploads table' do - Sidekiq::Testing.fake! do - described_class.new.perform + described_class.new.perform - hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path - expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey - end + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey end it 'correctly schedules the follow-up background migration jobs' do - Sidekiq::Testing.fake! do - described_class.new.perform + described_class.new.perform - expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) - expect(BackgroundMigrationWorker.jobs.size).to eq(1) - end + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) end # E.g. from a previous failed run of this background migration @@ -75,11 +74,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'does not error or produce duplicates of existing data' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.not_to change { untracked_files_for_uploads.count }.from(5) - end + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(5) end end @@ -97,11 +94,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'does not add files from /uploads/tmp' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(0).to(5) - end + expect do + described_class.new.perform + end.to change { untracked_files_for_uploads.count }.from(0).to(5) end end end @@ -110,11 +105,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # may not have an upload directory because they have no uploads. context 'when no files were ever uploaded' do it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do - Sidekiq::Testing.fake! do - expect do - described_class.new.perform - end.not_to change { untracked_files_for_uploads.count }.from(0) - end + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(0) end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index a6e880279b6..a17251ba052 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe TrackUntrackedUploads, :migration, :sidekiq do +describe TrackUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop do include TrackUntrackedUploadsHelpers let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } @@ -18,34 +18,41 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end end - it 'correctly schedules the follow-up background migration' do - Sidekiq::Testing.fake! do + context 'before running the background migration' do + around do |example| + # Especially important so the follow-up migration does not get run + Sidekiq::Testing.fake! do + example.run + end + end + + it 'correctly schedules the follow-up background migration' do migrate! expect(described_class::MIGRATION).to be_scheduled_migration expect(BackgroundMigrationWorker.jobs.size).to eq(1) end - end - it 'ensures the untracked_files_for_uploads table exists' do - expect do - migrate! - end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) - end + it 'ensures the untracked_files_for_uploads table exists' do + expect do + migrate! + end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) + end - it 'has a path field long enough for really long paths' do - migrate! + it 'has a path field long enough for really long paths' do + migrate! - component = 'a'*255 + component = 'a'*255 - long_path = [ - 'uploads', - component, # project.full_path - component # filename - ].flatten.join('/') + long_path = [ + 'uploads', + component, # project.full_path + component # filename + ].flatten.join('/') - record = untracked_files_for_uploads.create!(path: long_path) - expect(record.reload.path.size).to eq(519) + record = untracked_files_for_uploads.create!(path: long_path) + expect(record.reload.path.size).to eq(519) + end end context 'with tracked and untracked uploads' do @@ -77,36 +84,29 @@ describe TrackUntrackedUploads, :migration, :sidekiq do end it 'tracks untracked uploads' do - Sidekiq::Testing.inline! do - expect do - migrate! - end.to change { uploads.count }.from(4).to(8) - - expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) - expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) - expect(project2.reload.uploads.first.attributes).to include(@project2_avatar_attributes) - expect(project2.uploads.last.attributes).to include(@project2_markdown_attributes) - end + expect do + migrate! + end.to change { uploads.count }.from(4).to(8) + + expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) + expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) + expect(project2.reload.uploads.first.attributes).to include(@project2_avatar_attributes) + expect(project2.uploads.last.attributes).to include(@project2_markdown_attributes) end it 'ignores already-tracked uploads' do - Sidekiq::Testing.inline! do - migrate! + migrate! - expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) - expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) - expect(project1.reload.uploads.first.attributes).to include(@project1_avatar_attributes) - expect(project1.uploads.last.attributes).to include(@project1_markdown_attributes) - end + expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) + expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) + expect(project1.reload.uploads.first.attributes).to include(@project1_avatar_attributes) + expect(project1.uploads.last.attributes).to include(@project1_markdown_attributes) end - it 'all untracked_files_for_uploads records are marked as tracked' do - Sidekiq::Testing.inline! do - migrate! + it 'the temporary table untracked_files_for_uploads no longer exists' do + migrate! - expect(untracked_files_for_uploads.count).to eq(8) - expect(untracked_files_for_uploads.count).to eq(untracked_files_for_uploads.where(tracked: true).count) - end + expect(table_exists?(:untracked_files_for_uploads)).to be_falsey end end end diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index 5b832929602..bb700bc53f1 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -3,4 +3,18 @@ module TrackUntrackedUploadsHelpers fixture_path = Rails.root.join('spec', 'fixtures', 'rails_sample.jpg') fixture_file_upload(fixture_path) end + + def recreate_temp_table_if_dropped + TrackUntrackedUploads.new.ensure_temporary_tracking_table_exists + end + + RSpec.configure do |config| + config.after(:each, :temp_table_may_drop) do + recreate_temp_table_if_dropped + end + + config.after(:context, :temp_table_may_drop) do + recreate_temp_table_if_dropped + end + end end -- cgit v1.2.1 From 7fd26434196df01091b18747f91f54c0701bb292 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 03:01:35 -0800 Subject: Fix Rubocop offenses --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 10 +++++----- .../background_migration/populate_untracked_uploads_spec.rb | 6 ++---- .../background_migration/prepare_untracked_uploads_spec.rb | 2 +- spec/migrations/track_untracked_uploads_spec.rb | 2 +- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 983e63143e0..9c40cf8aee2 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -91,13 +91,13 @@ module Gitlab table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' sql = if Gitlab::Database.postgresql? - "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" - else - "INSERT IGNORE INTO #{table_columns_and_values};" - end + "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" + else + "INSERT IGNORE INTO #{table_columns_and_values};" + end timestamp = Time.now.utc.iso8601 - sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) + sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend ActiveRecord::Base.connection.execute(sql) end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 52f57408bfa..7f5c7b99742 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -215,7 +215,6 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - let(:expected_upload_attrs) { {} } # UntrackedFile.path is different than Upload.path let(:untracked_file) { create_untracked_file("/#{model.full_path}/#{model.uploads.first.path}") } @@ -228,7 +227,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do untracked_file # Save the expected upload attributes - expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') + @expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') # Untrack the file model.reload.uploads.delete_all @@ -239,8 +238,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do untracked_file.add_to_uploads end.to change { model.reload.uploads.count }.from(0).to(1) - hex_secret = untracked_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1] - expect(model.uploads.first.attributes).to include(expected_upload_attrs) + expect(model.uploads.first.attributes).to include(@expected_upload_attrs) end end end diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 8fd20fd0bb3..81af3f78307 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -36,7 +36,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side stub_application_setting(hashed_storage_enabled: true) - # Markdown upload after enabling hashed_storage + # Markdown upload after enabling hashed_storage UploadService.new(project2, uploaded_file, FileUploader).execute end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index a17251ba052..5632c81f231 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -42,7 +42,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop do it 'has a path field long enough for really long paths' do migrate! - component = 'a'*255 + component = 'a' * 255 long_path = [ 'uploads', -- cgit v1.2.1 From 10c660be007406533e48d5e3c6485ecf210e051b Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 04:51:28 -0800 Subject: Fix migration for pre-Postgres 9.5 --- .../background_migration/prepare_untracked_uploads.rb | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 9c40cf8aee2..8772092da64 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -88,9 +88,14 @@ module Gitlab end def insert_file_path(file_path) + if postgresql_pre_9_5? + # No easy way to do ON CONFLICT DO NOTHING before Postgres 9.5 so just use Rails + return UntrackedFile.where(path: file_path).first_or_create + end + table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' - sql = if Gitlab::Database.postgresql? + sql = if postgresql? "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" else "INSERT IGNORE INTO #{table_columns_and_values};" @@ -101,6 +106,15 @@ module Gitlab ActiveRecord::Base.connection.execute(sql) end + def postgresql? + @postgresql ||= Gitlab::Database.postgresql? + end + + def postgresql_pre_9_5? + @postgresql_pre_9_5 ||= postgresql? && + ActiveRecord::Base.connection.select_value('SHOW server_version_num').to_i < 90500 + end + def schedule_populate_untracked_uploads_jobs bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION) end -- cgit v1.2.1 From 87529ce5823036d4b9dd9ca412643befc8e490c3 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 15 Nov 2017 05:19:07 -0800 Subject: Move temp table creation into the prepare job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Hopefully fixes spec failures in which the table doesn’t exist * Decouples the background migration from the post-deploy migration, e.g. we could easily run it again even though the table is dropped when finished. --- .../20171103140253_track_untracked_uploads.rb | 20 --------- .../prepare_untracked_uploads.rb | 21 ++++++++-- .../populate_untracked_uploads_spec.rb | 17 ++++++-- .../prepare_untracked_uploads_spec.rb | 47 +++++++++++++++++----- spec/migrations/track_untracked_uploads_spec.rb | 34 ++-------------- spec/support/track_untracked_uploads_helpers.rb | 14 ++----- 6 files changed, 76 insertions(+), 77 deletions(-) diff --git a/db/post_migrate/20171103140253_track_untracked_uploads.rb b/db/post_migrate/20171103140253_track_untracked_uploads.rb index 7a34abc85ee..548a94d2d38 100644 --- a/db/post_migrate/20171103140253_track_untracked_uploads.rb +++ b/db/post_migrate/20171103140253_track_untracked_uploads.rb @@ -10,8 +10,6 @@ class TrackUntrackedUploads < ActiveRecord::Migration MIGRATION = 'PrepareUntrackedUploads' def up - ensure_temporary_tracking_table_exists - BackgroundMigrationWorker.perform_async(MIGRATION) end @@ -20,22 +18,4 @@ class TrackUntrackedUploads < ActiveRecord::Migration drop_table :untracked_files_for_uploads end end - - def ensure_temporary_tracking_table_exists - unless table_exists?(:untracked_files_for_uploads) - create_table :untracked_files_for_uploads do |t| - t.string :path, limit: 600, null: false - t.boolean :tracked, default: false, null: false - t.timestamps_with_timezone null: false - end - end - - unless index_exists?(:untracked_files_for_uploads, :path) - add_index :untracked_files_for_uploads, :path, unique: true - end - - unless index_exists?(:untracked_files_for_uploads, :tracked) - add_index :untracked_files_for_uploads, :tracked - end - end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 8772092da64..e0c1daaccf7 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -19,16 +19,29 @@ module Gitlab end def perform - return unless migrate? - + ensure_temporary_tracking_table_exists store_untracked_file_paths schedule_populate_untracked_uploads_jobs end private - def migrate? - UntrackedFile.table_exists? + def ensure_temporary_tracking_table_exists + unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) + UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| + t.string :path, limit: 600, null: false + t.boolean :tracked, default: false, null: false + t.timestamps_with_timezone null: false + end + end + + unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :path) + UntrackedFile.connection.add_index :untracked_files_for_uploads, :path, unique: true + end + + unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :tracked) + UntrackedFile.connection.add_index :untracked_files_for_uploads, :tracked + end end def store_untracked_file_paths diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 7f5c7b99742..317890bd854 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do include TrackUntrackedUploadsHelpers subject { described_class.new } @@ -10,8 +10,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid let!(:uploads) { table(:uploads) } before do - # Prevent the TrackUntrackedUploads migration from running PrepareUntrackedUploads job - allow(BackgroundMigrationWorker).to receive(:perform_async).and_return(true) + ensure_temporary_tracking_table_exists + end + + after(:all) do + drop_temp_table_if_exists end context 'with untracked files and tracked files in untracked_files_for_uploads' do @@ -130,6 +133,14 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } + before(:all) do + ensure_temporary_tracking_table_exists + end + + after(:all) do + drop_temp_table_if_exists + end + describe '#ensure_tracked!' do let!(:user1) { create(:user, :with_avatar) } let!(:untracked_file) { described_class.create!(path: user1.uploads.first.path) } diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 81af3f78307..042fdead281 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -17,6 +17,14 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end end + before do + drop_temp_table_if_exists + end + + after(:all) do + drop_temp_table_if_exists + end + around do |example| # Especially important so the follow-up migration does not get run Sidekiq::Testing.fake! do @@ -24,6 +32,27 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end end + it 'ensures the untracked_files_for_uploads table exists' do + expect do + described_class.new.perform + end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) + end + + it 'has a path field long enough for really long paths' do + described_class.new.perform + + component = 'a' * 255 + + long_path = [ + 'uploads', + component, # project.full_path + component # filename + ].flatten.join('/') + + record = untracked_files_for_uploads.create!(path: long_path) + expect(record.reload.path.size).to eq(519) + end + context 'when files were uploaded before and after hashed storage was enabled' do let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } let!(:user) { create(:user, :with_avatar) } @@ -41,9 +70,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'adds unhashed files to the untracked_files_for_uploads table' do - expect do - described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(0).to(5) + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(5) end it 'adds files with paths relative to CarrierWave.root' do @@ -94,9 +123,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end it 'does not add files from /uploads/tmp' do - expect do - described_class.new.perform - end.to change { untracked_files_for_uploads.count }.from(0).to(5) + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(5) end end end @@ -105,9 +134,9 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side # may not have an upload directory because they have no uploads. context 'when no files were ever uploaded' do it 'does not add to the untracked_files_for_uploads table (and does not raise error)' do - expect do - described_class.new.perform - end.not_to change { untracked_files_for_uploads.count }.from(0) + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(0) end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 5632c81f231..11824bebb91 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe TrackUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop do +describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } @@ -18,41 +18,13 @@ describe TrackUntrackedUploads, :migration, :sidekiq, :temp_table_may_drop do end end - context 'before running the background migration' do - around do |example| - # Especially important so the follow-up migration does not get run - Sidekiq::Testing.fake! do - example.run - end - end - - it 'correctly schedules the follow-up background migration' do + it 'correctly schedules the follow-up background migration' do + Sidekiq::Testing.fake! do migrate! expect(described_class::MIGRATION).to be_scheduled_migration expect(BackgroundMigrationWorker.jobs.size).to eq(1) end - - it 'ensures the untracked_files_for_uploads table exists' do - expect do - migrate! - end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) - end - - it 'has a path field long enough for really long paths' do - migrate! - - component = 'a' * 255 - - long_path = [ - 'uploads', - component, # project.full_path - component # filename - ].flatten.join('/') - - record = untracked_files_for_uploads.create!(path: long_path) - expect(record.reload.path.size).to eq(519) - end end context 'with tracked and untracked uploads' do diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index bb700bc53f1..4d4745fd7f4 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -4,17 +4,11 @@ module TrackUntrackedUploadsHelpers fixture_file_upload(fixture_path) end - def recreate_temp_table_if_dropped - TrackUntrackedUploads.new.ensure_temporary_tracking_table_exists + def ensure_temporary_tracking_table_exists + Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists) end - RSpec.configure do |config| - config.after(:each, :temp_table_may_drop) do - recreate_temp_table_if_dropped - end - - config.after(:context, :temp_table_may_drop) do - recreate_temp_table_if_dropped - end + def drop_temp_table_if_exists + ActiveRecord::Base.connection.drop_table(:untracked_files_for_uploads) if ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) end end -- cgit v1.2.1 From 5552d1adf4489b71190c6b59ab66b300fd5a0604 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 16 Nov 2017 16:24:42 -0800 Subject: Log the find command used --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index e0c1daaccf7..853c9810359 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -82,6 +82,8 @@ module Gitlab cmd = %w[ionice -c Idle] + cmd if ionice_is_available? + Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + cmd end -- cgit v1.2.1 From f5fa39872402c7f27307ffe9bd769c4f120a0f2e Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 16 Nov 2017 16:26:40 -0800 Subject: Attempt to fix spec in CI --- spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index 042fdead281..b6b046ec3aa 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -21,7 +21,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side drop_temp_table_if_exists end - after(:all) do + after do drop_temp_table_if_exists end -- cgit v1.2.1 From 17ce21d74eab4d2973d372cb3f97258eb3b81de9 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 17 Nov 2017 13:49:25 -0800 Subject: Use ionice absolute path --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 853c9810359..c3f5dddb07d 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -80,14 +80,15 @@ module Gitlab def build_find_command(search_dir) cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0] - cmd = %w[ionice -c Idle] + cmd if ionice_is_available? + ionice = which_ionice + cmd = %W[#{ionice} -c Idle] + cmd if ionice Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" cmd end - def ionice_is_available? + def which_ionice Gitlab::Utils.which('ionice') rescue StandardError # In this case, returning false is relatively safe, even though it isn't very nice -- cgit v1.2.1 From edb5cac46c1cba1029fb3e67d4853027590584f6 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 20 Nov 2017 16:27:24 -0800 Subject: Use bulk inserts --- .../prepare_untracked_uploads.rb | 55 +++--- .../prepare_untracked_uploads_spec.rb | 192 ++++++++++++++++----- spec/migrations/track_untracked_uploads_spec.rb | 8 +- 3 files changed, 184 insertions(+), 71 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index c3f5dddb07d..022b2f41393 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -20,7 +20,19 @@ module Gitlab def perform ensure_temporary_tracking_table_exists + + # Since Postgres < 9.5 does not have ON CONFLICT DO NOTHING, and since + # doing inserts-if-not-exists without ON CONFLICT DO NOTHING would be + # slow, start with an empty table for Postgres < 9.5. + # That way we can do bulk inserts at ~30x the speed of individual + # inserts (~20 minutes worth of inserts at GitLab.com scale instead of + # ~10 hours). + # In all other cases, installations will get both bulk inserts and the + # ability for these jobs to retry without having to clear and reinsert. + clear_untracked_file_paths unless can_bulk_insert_and_ignore_duplicates? + store_untracked_file_paths + schedule_populate_untracked_uploads_jobs end @@ -44,6 +56,10 @@ module Gitlab end end + def clear_untracked_file_paths + UntrackedFile.delete_all + end + def store_untracked_file_paths return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) @@ -96,36 +112,35 @@ module Gitlab end def insert_file_paths(file_paths) - ActiveRecord::Base.transaction do - file_paths.each do |file_path| - insert_file_path(file_path) - end - end - end + sql = if postgresql_pre_9_5? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" + elsif postgresql? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)} ON CONFLICT DO NOTHING;" + else # MySQL + "INSERT IGNORE INTO #{table_columns_and_values_for_insert(file_paths)};" + end - def insert_file_path(file_path) - if postgresql_pre_9_5? - # No easy way to do ON CONFLICT DO NOTHING before Postgres 9.5 so just use Rails - return UntrackedFile.where(path: file_path).first_or_create - end + ActiveRecord::Base.connection.execute(sql) + end - table_columns_and_values = 'untracked_files_for_uploads (path, created_at, updated_at) VALUES (?, ?, ?)' + def table_columns_and_values_for_insert(file_paths) + timestamp = Time.now.utc.iso8601 - sql = if postgresql? - "INSERT INTO #{table_columns_and_values} ON CONFLICT DO NOTHING;" - else - "INSERT IGNORE INTO #{table_columns_and_values};" - end + values = file_paths.map do |file_path| + ActiveRecord::Base.send(:sanitize_sql_array, ['(?, ?, ?)', file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend + end.join(', ') - timestamp = Time.now.utc.iso8601 - sql = ActiveRecord::Base.send(:sanitize_sql_array, [sql, file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend - ActiveRecord::Base.connection.execute(sql) + "#{UntrackedFile.table_name} (path, created_at, updated_at) VALUES #{values}" end def postgresql? @postgresql ||= Gitlab::Database.postgresql? end + def can_bulk_insert_and_ignore_duplicates? + !postgresql_pre_9_5? + end + def postgresql_pre_9_5? @postgresql_pre_9_5 ||= postgresql? && ActiveRecord::Base.connection.select_value('SHOW server_version_num').to_i < 90500 diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index b6b046ec3aa..f1eb7173717 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -53,80 +53,178 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side expect(record.reload.path.size).to eq(519) end - context 'when files were uploaded before and after hashed storage was enabled' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } - let!(:user) { create(:user, :with_avatar) } - let!(:project1) { create(:project, :with_avatar) } - let(:project2) { create(:project) } # instantiate after enabling hashed_storage + context "test bulk insert with ON CONFLICT DO NOTHING or IGNORE" do + around do |example| + # If this is CI, we use Postgres 9.2 so this whole context should be + # skipped since we're unable to use ON CONFLICT DO NOTHING or IGNORE. + if described_class.new.send(:can_bulk_insert_and_ignore_duplicates?) + example.run + end + end - before do - # Markdown upload before enabling hashed_storage - UploadService.new(project1, uploaded_file, FileUploader).execute + context 'when files were uploaded before and after hashed storage was enabled' do + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let(:project2) { create(:project) } # instantiate after enabling hashed_storage - stub_application_setting(hashed_storage_enabled: true) + before do + # Markdown upload before enabling hashed_storage + UploadService.new(project1, uploaded_file, FileUploader).execute - # Markdown upload after enabling hashed_storage - UploadService.new(project2, uploaded_file, FileUploader).execute - end + stub_application_setting(hashed_storage_enabled: true) - it 'adds unhashed files to the untracked_files_for_uploads table' do - described_class.new.perform + # Markdown upload after enabling hashed_storage + UploadService.new(project2, uploaded_file, FileUploader).execute + end - expect(untracked_files_for_uploads.count).to eq(5) - end + it 'adds unhashed files to the untracked_files_for_uploads table' do + described_class.new.perform - it 'adds files with paths relative to CarrierWave.root' do - described_class.new.perform - untracked_files_for_uploads.all.each do |file| - expect(file.path.start_with?('uploads/')).to be_truthy + expect(untracked_files_for_uploads.count).to eq(5) end - end - it 'does not add hashed files to the untracked_files_for_uploads table' do - described_class.new.perform - - hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path - expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey - end + it 'adds files with paths relative to CarrierWave.root' do + described_class.new.perform + untracked_files_for_uploads.all.each do |file| + expect(file.path.start_with?('uploads/')).to be_truthy + end + end - it 'correctly schedules the follow-up background migration jobs' do - described_class.new.perform + it 'does not add hashed files to the untracked_files_for_uploads table' do + described_class.new.perform - expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) - expect(BackgroundMigrationWorker.jobs.size).to eq(1) - end + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end - # E.g. from a previous failed run of this background migration - context 'when there is existing data in untracked_files_for_uploads' do - before do + it 'correctly schedules the follow-up background migration jobs' do described_class.new.perform + + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) end - it 'does not error or produce duplicates of existing data' do - expect do + # E.g. from a previous failed run of this background migration + context 'when there is existing data in untracked_files_for_uploads' do + before do described_class.new.perform - end.not_to change { untracked_files_for_uploads.count }.from(5) + end + + it 'does not error or produce duplicates of existing data' do + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(5) + end end + + # E.g. The installation is in use at the time of migration, and someone has + # just uploaded a file + context 'when there are files in /uploads/tmp' do + let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } + + before do + FileUtils.touch(tmp_file) + end + + after do + FileUtils.rm(tmp_file) + end + + it 'does not add files from /uploads/tmp' do + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(5) + end + end + end + end + + context 'test bulk insert without ON CONFLICT DO NOTHING or IGNORE' do + before do + # If this is CI, we use Postgres 9.2 so this stub has no effect. + # + # If this is being run on Postgres 9.5+ or MySQL, then this stub allows us + # to test the bulk insert functionality without ON CONFLICT DO NOTHING or + # IGNORE. + allow_any_instance_of(described_class).to receive(:postgresql_pre_9_5?).and_return(true) end - # E.g. The installation is in use at the time of migration, and someone has - # just uploaded a file - context 'when there are files in /uploads/tmp' do - let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } + context 'when files were uploaded before and after hashed storage was enabled' do + let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:user) { create(:user, :with_avatar) } + let!(:project1) { create(:project, :with_avatar) } + let(:project2) { create(:project) } # instantiate after enabling hashed_storage before do - FileUtils.touch(tmp_file) - end + # Markdown upload before enabling hashed_storage + UploadService.new(project1, uploaded_file, FileUploader).execute - after do - FileUtils.rm(tmp_file) + stub_application_setting(hashed_storage_enabled: true) + + # Markdown upload after enabling hashed_storage + UploadService.new(project2, uploaded_file, FileUploader).execute end - it 'does not add files from /uploads/tmp' do + it 'adds unhashed files to the untracked_files_for_uploads table' do described_class.new.perform expect(untracked_files_for_uploads.count).to eq(5) end + + it 'adds files with paths relative to CarrierWave.root' do + described_class.new.perform + untracked_files_for_uploads.all.each do |file| + expect(file.path.start_with?('uploads/')).to be_truthy + end + end + + it 'does not add hashed files to the untracked_files_for_uploads table' do + described_class.new.perform + + hashed_file_path = project2.uploads.where(uploader: 'FileUploader').first.path + expect(untracked_files_for_uploads.where("path like '%#{hashed_file_path}%'").exists?).to be_falsey + end + + it 'correctly schedules the follow-up background migration jobs' do + described_class.new.perform + + expect(described_class::FOLLOW_UP_MIGRATION).to be_scheduled_migration(1, 5) + expect(BackgroundMigrationWorker.jobs.size).to eq(1) + end + + # E.g. from a previous failed run of this background migration + context 'when there is existing data in untracked_files_for_uploads' do + before do + described_class.new.perform + end + + it 'does not error or produce duplicates of existing data' do + expect do + described_class.new.perform + end.not_to change { untracked_files_for_uploads.count }.from(5) + end + end + + # E.g. The installation is in use at the time of migration, and someone has + # just uploaded a file + context 'when there are files in /uploads/tmp' do + let(:tmp_file) { Rails.root.join(described_class::ABSOLUTE_UPLOAD_DIR, 'tmp', 'some_file.jpg') } + + before do + FileUtils.touch(tmp_file) + end + + after do + FileUtils.rm(tmp_file) + end + + it 'does not add files from /uploads/tmp' do + described_class.new.perform + + expect(untracked_files_for_uploads.count).to eq(5) + end + end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 11824bebb91..01bfe26744f 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -62,8 +62,8 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) - expect(project2.reload.uploads.first.attributes).to include(@project2_avatar_attributes) - expect(project2.uploads.last.attributes).to include(@project2_markdown_attributes) + expect(project2.reload.uploads.where(uploader: 'AvatarUploader').first.attributes).to include(@project2_avatar_attributes) + expect(project2.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project2_markdown_attributes) end it 'ignores already-tracked uploads' do @@ -71,8 +71,8 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) - expect(project1.reload.uploads.first.attributes).to include(@project1_avatar_attributes) - expect(project1.uploads.last.attributes).to include(@project1_markdown_attributes) + expect(project1.reload.uploads.where(uploader: 'AvatarUploader').first.attributes).to include(@project1_avatar_attributes) + expect(project1.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project1_markdown_attributes) end it 'the temporary table untracked_files_for_uploads no longer exists' do -- cgit v1.2.1 From aefbdcdcf25cbcd8d80dcaa3216264d91bdcac91 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 21 Nov 2017 16:05:33 -0800 Subject: Fix Rubocop offense --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index f28892174bb..b8872477e63 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -138,6 +138,7 @@ module Gitlab def file_uploader_model_id matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN) raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + full_path = matchd[1] project = Project.find_by_full_path(full_path) project.id.to_s -- cgit v1.2.1 From 8def25d9f163cc22f71716d8a22fc5adfe0a762e Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 21 Nov 2017 16:05:57 -0800 Subject: Fix datetime inserts on MySQL --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 022b2f41393..f5a11658c0b 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -124,10 +124,8 @@ module Gitlab end def table_columns_and_values_for_insert(file_paths) - timestamp = Time.now.utc.iso8601 - values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?, ?, ?)', file_path, timestamp, timestamp]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.send(:sanitize_sql_array, ['(?, NOW(), NOW())', file_path]) # rubocop:disable GitlabSecurity/PublicSend end.join(', ') "#{UntrackedFile.table_name} (path, created_at, updated_at) VALUES #{values}" -- cgit v1.2.1 From a9155a94fe29aa67230f2e5ef3d6393345677ce0 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 22 Nov 2017 10:23:24 -0800 Subject: Refactor --- .../background_migration/prepare_untracked_uploads.rb | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index f5a11658c0b..8333a6218de 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -44,16 +44,10 @@ module Gitlab t.string :path, limit: 600, null: false t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false + t.index :path, unique: true + t.index :tracked end end - - unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :path) - UntrackedFile.connection.add_index :untracked_files_for_uploads, :path, unique: true - end - - unless UntrackedFile.connection.index_exists?(:untracked_files_for_uploads, :tracked) - UntrackedFile.connection.add_index :untracked_files_for_uploads, :tracked - end end def clear_untracked_file_paths @@ -140,8 +134,7 @@ module Gitlab end def postgresql_pre_9_5? - @postgresql_pre_9_5 ||= postgresql? && - ActiveRecord::Base.connection.select_value('SHOW server_version_num').to_i < 90500 + @postgresql_pre_9_5 ||= postgresql? && Gitlab::Database.version.to_f < 9.5 end def schedule_populate_untracked_uploads_jobs -- cgit v1.2.1 From 67b58ffdc357bb94ec62d696b7b9a4aecf751c75 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 22 Nov 2017 10:44:33 -0800 Subject: Get rid of tracked field It makes a debugging slightly easier, but is not necessary, and is a waste of resources. --- .../populate_untracked_uploads.rb | 14 +++----------- .../prepare_untracked_uploads.rb | 2 -- .../populate_untracked_uploads_spec.rb | 20 ++++---------------- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index b8872477e63..03e7b7b71cb 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -50,14 +50,10 @@ module Gitlab } ].freeze - scope :untracked, -> { where(tracked: false) } - def ensure_tracked! - return if persisted? && tracked? - add_to_uploads unless in_uploads? - mark_as_tracked + delete end def in_uploads? @@ -79,10 +75,6 @@ module Gitlab ) end - def mark_as_tracked - update!(tracked: true) - end - def upload_path # UntrackedFile#path is absolute, but Upload#path depends on uploader if uploader == 'FileUploader' @@ -197,7 +189,7 @@ module Gitlab def perform(start_id, end_id) return unless migrate? - files = UntrackedFile.untracked.where(id: start_id..end_id) + files = UntrackedFile.where(id: start_id..end_id) files.each do |untracked_file| begin untracked_file.ensure_tracked! @@ -220,7 +212,7 @@ module Gitlab end def drop_temp_table_if_finished - UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.untracked.empty? + UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty? end end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 8333a6218de..c076c13815d 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -42,10 +42,8 @@ module Gitlab unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| t.string :path, limit: 600, null: false - t.boolean :tracked, default: false, null: false t.timestamps_with_timezone null: false t.index :path, unique: true - t.index :tracked end end end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 317890bd854..04719d50f5c 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -54,12 +54,12 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(appearance.uploads.count).to eq(2) end - it 'sets all added or confirmed tracked files to tracked' do + it 'deletes rows after processing them' do expect(subject).to receive(:drop_temp_table_if_finished) # Don't drop the table so we can look at it expect do subject.perform(1, 1000) - end.to change { untracked_files_for_uploads.where(tracked: true).count }.from(0).to(8) + end.to change { untracked_files_for_uploads.count }.from(8).to(0) end it 'does not create duplicate uploads of already tracked files' do @@ -85,7 +85,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project2.uploads.count).to eq(0) # Only 4 have been either confirmed or added to uploads - expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) + expect(untracked_files_for_uploads.count).to eq(4) end it 'uses the start and end batch ids [only 2nd half]' do @@ -103,7 +103,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(project2.uploads.count).to eq(2) # Only 4 have been either confirmed or added to uploads - expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4) + expect(untracked_files_for_uploads.count).to eq(4) end it 'does not drop the temporary tracking table after processing the batch, if there are still untracked rows' do @@ -254,18 +254,6 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end end - describe '#mark_as_tracked' do - it 'saves the record with tracked set to true' do - untracked_file = create_untracked_file("/-/system/appearance/logo/1/some_logo.jpg") - - expect do - untracked_file.mark_as_tracked - end.to change { untracked_file.tracked }.from(false).to(true) - - expect(untracked_file.persisted?).to be_truthy - end - end - describe '#upload_path' do def assert_upload_path(file_path, expected_upload_path) untracked_file = create_untracked_file(file_path) -- cgit v1.2.1 From 7549d17f721b3be84f83c1dfa491d6a2ebf4ec28 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 22 Nov 2017 12:19:25 -0800 Subject: Refactor --- .../populate_untracked_uploads.rb | 23 ++++++++++------------ .../populate_untracked_uploads_spec.rb | 6 +++--- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 03e7b7b71cb..41dc5a3ed7e 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -51,28 +51,25 @@ module Gitlab ].freeze def ensure_tracked! - add_to_uploads unless in_uploads? + add_to_uploads_if_needed delete end - def in_uploads? + def add_to_uploads_if_needed # Even though we are checking relative paths, path is enough to # uniquely identify uploads. There is no ambiguity between # FileUploader paths and other Uploader paths because we use the /-/ # separator kind of like an escape character. Project full_path will # never conflict with an upload path starting with "uploads/-/". - Upload.exists?(path: upload_path) - end - - def add_to_uploads - Upload.create!( - path: upload_path, - uploader: uploader, - model_type: model_type, - model_id: model_id, - size: file_size - ) + Upload. + where(path: upload_path). + first_or_create!( + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + ) end def upload_path diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 04719d50f5c..72243ff98a4 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -166,7 +166,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do end end - describe '#add_to_uploads' do + describe '#add_to_uploads_if_needed' do shared_examples_for 'add_to_uploads_non_markdown_files' do let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') } let!(:untracked_file) { described_class.create!(path: expected_upload_attrs['path']) } @@ -177,7 +177,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'creates an Upload record' do expect do - untracked_file.add_to_uploads + untracked_file.add_to_uploads_if_needed end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(expected_upload_attrs) @@ -246,7 +246,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'creates an Upload record' do expect do - untracked_file.add_to_uploads + untracked_file.add_to_uploads_if_needed end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(@expected_upload_attrs) -- cgit v1.2.1 From 908aacddda571bc82c722798f399fd5a68ebe1da Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 23 Nov 2017 23:12:24 -0800 Subject: Filter existing uploads with one query --- .../populate_untracked_uploads.rb | 80 ++++++++++++---------- .../populate_untracked_uploads_spec.rb | 80 +++++++--------------- 2 files changed, 68 insertions(+), 92 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 41dc5a3ed7e..8d6da0a7a67 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -50,37 +50,25 @@ module Gitlab } ].freeze - def ensure_tracked! - add_to_uploads_if_needed - - delete - end - - def add_to_uploads_if_needed - # Even though we are checking relative paths, path is enough to - # uniquely identify uploads. There is no ambiguity between - # FileUploader paths and other Uploader paths because we use the /-/ - # separator kind of like an escape character. Project full_path will - # never conflict with an upload path starting with "uploads/-/". - Upload. - where(path: upload_path). - first_or_create!( - uploader: uploader, - model_type: model_type, - model_id: model_id, - size: file_size - ) + def to_h + { + path: upload_path, + uploader: uploader, + model_type: model_type, + model_id: model_id, + size: file_size + } end def upload_path # UntrackedFile#path is absolute, but Upload#path depends on uploader - if uploader == 'FileUploader' - # Path relative to project directory in uploads - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) - matchd[0].sub(%r{\A/}, '') # remove leading slash - else - path - end + @upload_path ||= if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) + matchd[0].sub(%r{\A/}, '') # remove leading slash + else + path + end end def uploader @@ -187,17 +175,8 @@ module Gitlab return unless migrate? files = UntrackedFile.where(id: start_id..end_id) - files.each do |untracked_file| - begin - untracked_file.ensure_tracked! - rescue StandardError => e - Rails.logger.warn "Failed to add untracked file to uploads: #{e.message}" - - # The untracked rows will remain in the DB. We will be able to see - # which ones failed to become tracked, and then we can decide what - # to do. - end - end + insert_uploads_if_needed(files) + files.delete_all drop_temp_table_if_finished end @@ -208,6 +187,31 @@ module Gitlab UntrackedFile.table_exists? && Upload.table_exists? end + def insert_uploads_if_needed(files) + filtered_files = filter_existing_uploads(files) + filtered_files = filter_deleted_models(filtered_files) + insert(filtered_files) + end + + def filter_existing_uploads(files) + paths = files.map(&:upload_path) + existing_paths = Upload.where(path: paths).pluck(:path).to_set + + files.reject do |file| + existing_paths.include?(file.upload_path) + end + end + + def filter_deleted_models(files) + files # TODO + end + + def insert(files) + files.each do |file| + Upload.create!(file.to_h) + end + end + def drop_temp_table_if_finished UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty? end diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 72243ff98a4..623725bffca 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -126,50 +126,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end.not_to change { uploads.count }.from(0) end end -end - -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do - include TrackUntrackedUploadsHelpers - - let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } - - before(:all) do - ensure_temporary_tracking_table_exists - end - - after(:all) do - drop_temp_table_if_exists - end - describe '#ensure_tracked!' do - let!(:user1) { create(:user, :with_avatar) } - let!(:untracked_file) { described_class.create!(path: user1.uploads.first.path) } - - context 'when the file is already in the uploads table' do - it 'does not add an upload' do - expect do - untracked_file.ensure_tracked! - end.not_to change { upload_class.count }.from(1) - end - end - - context 'when the file is not already in the uploads table' do - before do - user1.uploads.delete_all - end - - it 'adds an upload' do - expect do - untracked_file.ensure_tracked! - end.to change { upload_class.count }.from(0).to(1) - end - end - end - - describe '#add_to_uploads_if_needed' do - shared_examples_for 'add_to_uploads_non_markdown_files' do + describe 'upload outcomes for each path pattern' do + shared_examples_for 'non_markdown_file' do let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') } - let!(:untracked_file) { described_class.create!(path: expected_upload_attrs['path']) } + let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) } before do model.uploads.delete_all @@ -177,7 +138,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'creates an Upload record' do expect do - untracked_file.add_to_uploads_if_needed + subject.perform(1, 1000) end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(expected_upload_attrs) @@ -187,13 +148,13 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance logo file path' do let(:model) { create(:appearance, logo: uploaded_file) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for an appearance header_logo file path' do let(:model) { create(:appearance, header_logo: uploaded_file) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a pre-Markdown Note attachment file path' do @@ -203,39 +164,36 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do let(:model) { create(:note, :with_attachment) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a user avatar file path' do let(:model) { create(:user, :with_avatar) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a group avatar file path' do let(:model) { create(:group, :with_avatar) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a project avatar file path' do let(:model) { create(:project, :with_avatar) } - it_behaves_like 'add_to_uploads_non_markdown_files' + it_behaves_like 'non_markdown_file' end context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do let(:model) { create(:project) } - # UntrackedFile.path is different than Upload.path - let(:untracked_file) { create_untracked_file("/#{model.full_path}/#{model.uploads.first.path}") } - before do # Upload the file UploadService.new(model, uploaded_file, FileUploader).execute # Create the untracked_files_for_uploads record - untracked_file + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}") # Save the expected upload attributes @expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') @@ -246,13 +204,27 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'creates an Upload record' do expect do - untracked_file.add_to_uploads_if_needed + subject.perform(1, 1000) end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(@expected_upload_attrs) end end end +end + +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do + include TrackUntrackedUploadsHelpers + + let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload } + + before(:all) do + ensure_temporary_tracking_table_exists + end + + after(:all) do + drop_temp_table_if_exists + end describe '#upload_path' do def assert_upload_path(file_path, expected_upload_path) -- cgit v1.2.1 From a9c868d111c0231c4358d0ee017d1a9e9a28d3dd Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 23 Nov 2017 23:49:16 -0800 Subject: Bulk insert uploads --- .../populate_untracked_uploads.rb | 63 +++++----------------- 1 file changed, 14 insertions(+), 49 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 8d6da0a7a67..9fb5b3f77a2 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -56,7 +56,8 @@ module Gitlab uploader: uploader, model_type: model_type, model_id: model_id, - size: file_size + size: file_size, + checksum: checksum } end @@ -90,10 +91,13 @@ module Gitlab end def file_size - absolute_path = File.join(CarrierWave.root, path) File.size(absolute_path) end + def checksum + Digest::SHA256.file(absolute_path).hexdigest + end + # Not including a leading slash def path_relative_to_upload_dir base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} @@ -120,55 +124,14 @@ module Gitlab project = Project.find_by_full_path(full_path) project.id.to_s end - end - - # Copy-pasted class for less fragile migration - class Upload < ActiveRecord::Base - self.table_name = 'uploads' # This is the only line different from copy-paste - - # Upper limit for foreground checksum processing - CHECKSUM_THRESHOLD = 100.megabytes - - belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations - - before_save :calculate_checksum, if: :foreground_checksum? - after_commit :schedule_checksum, unless: :foreground_checksum? def absolute_path - return path unless relative_path? - - uploader_class.absolute_path(self) - end - - def calculate_checksum - return unless exist? - - self.checksum = Digest::SHA256.file(absolute_path).hexdigest - rescue StandardError - schedule_checksum - end - - def exist? - File.exist?(absolute_path) - end - - private - - def foreground_checksum? - size <= CHECKSUM_THRESHOLD - end - - def schedule_checksum - UploadChecksumWorker.perform_async(id) - end - - def relative_path? - !path.start_with?('/') + File.join(CarrierWave.root, path) end + end - def uploader_class - Object.const_get(uploader) - end + class Upload < ActiveRecord::Base + self.table_name = 'uploads' end def perform(start_id, end_id) @@ -207,9 +170,11 @@ module Gitlab end def insert(files) - files.each do |file| - Upload.create!(file.to_h) + rows = files.map do |file| + file.to_h.merge(created_at: 'NOW()') end + + Gitlab::Database.bulk_insert('uploads', rows) end def drop_temp_table_if_finished -- cgit v1.2.1 From 61a73cadb7f21de9f863fc1a16f13880861ac9f4 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Thu, 23 Nov 2017 23:51:29 -0800 Subject: Get rid of timestamps on untracked files table `updated_at` is now unnecessary and `created_at` is less useful due to removing the tracked field. --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index c076c13815d..358b76d39fb 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -42,7 +42,6 @@ module Gitlab unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| t.string :path, limit: 600, null: false - t.timestamps_with_timezone null: false t.index :path, unique: true end end @@ -117,10 +116,10 @@ module Gitlab def table_columns_and_values_for_insert(file_paths) values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?, NOW(), NOW())', file_path]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend end.join(', ') - "#{UntrackedFile.table_name} (path, created_at, updated_at) VALUES #{values}" + "#{UntrackedFile.table_name} (path) VALUES #{values}" end def postgresql? -- cgit v1.2.1 From 473ddfb453d820f1a32fb48477e17ba45bdbd2f0 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 24 Nov 2017 00:49:04 -0800 Subject: =?UTF-8?q?Don=E2=80=99t=20recreate=20deleted=20uploads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../populate_untracked_uploads.rb | 40 +++++++++++++++++++--- .../populate_untracked_uploads_spec.rb | 14 ++++---- spec/migrations/track_untracked_uploads_spec.rb | 9 +++++ 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 9fb5b3f77a2..50ec2260c60 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -81,13 +81,13 @@ module Gitlab end def model_id + return @model_id if defined?(@model_id) + matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern]) # If something is captured (matchd[1] is not nil), it is a model_id - return matchd[1] if matchd[1] - # Only the FileUploader pattern will not match an ID - file_uploader_model_id + @model_id = matchd[1] ? matchd[1].to_i : file_uploader_model_id end def file_size @@ -122,7 +122,9 @@ module Gitlab full_path = matchd[1] project = Project.find_by_full_path(full_path) - project.id.to_s + return nil unless project + + project.id end def absolute_path @@ -165,8 +167,36 @@ module Gitlab end end + # There are files on disk that are not in the uploads table because their + # model was deleted, and we don't delete the files on disk. def filter_deleted_models(files) - files # TODO + ids = deleted_model_ids(files) + + files.reject do |file| + ids[file.model_type].include?(file.model_id) + end + end + + def deleted_model_ids(files) + ids = { + 'Appearance' => [], + 'Namespace' => [], + 'Note' => [], + 'Project' => [], + 'User' => [] + } + + # group model IDs by model type + files.each do |file| + ids[file.model_type] << file.model_id + end + + ids.each do |model_type, model_ids| + found_ids = Object.const_get(model_type).where(id: model_ids.uniq).pluck(:id) + ids[model_type] = ids[model_type] - found_ids # replace with deleted ids + end + + ids end def insert(files) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 623725bffca..85da8ac5b1e 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -392,37 +392,37 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do context 'for an appearance logo file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', '1') + assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', 1) end end context 'for an appearance header_logo file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', '1') + assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', 1) end end context 'for a pre-Markdown Note attachment file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', '1234') + assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', 1234) end end context 'for a user avatar file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/user/avatar/1234/avatar.jpg', '1234') + assert_model_id('/-/system/user/avatar/1234/avatar.jpg', 1234) end end context 'for a group avatar file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/group/avatar/1234/avatar.jpg', '1234') + assert_model_id('/-/system/group/avatar/1234/avatar.jpg', 1234) end end context 'for a project avatar file path' do it 'returns the ID as a string' do - assert_model_id('/-/system/project/avatar/1234/avatar.jpg', '1234') + assert_model_id('/-/system/project/avatar/1234/avatar.jpg', 1234) end end @@ -430,7 +430,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do it 'returns the ID as a string' do project = create(:project) - assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id.to_s) + assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id) end end end diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 01bfe26744f..9fa586ff177 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -75,6 +75,15 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(project1.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project1_markdown_attributes) end + it 'ignores uploads for deleted models' do + user2.destroy + project2.destroy + + expect do + migrate! + end.to change { uploads.count }.from(4).to(5) + end + it 'the temporary table untracked_files_for_uploads no longer exists' do migrate! -- cgit v1.2.1 From 6e36452e96139658ff8eae88209710438dd14eba Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 24 Nov 2017 00:52:16 -0800 Subject: Refactor --- .../background_migration/populate_untracked_uploads.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 50ec2260c60..f06b4ac1940 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -98,12 +98,6 @@ module Gitlab Digest::SHA256.file(absolute_path).hexdigest end - # Not including a leading slash - def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} - @path_relative_to_upload_dir ||= path.sub(base, '') - end - private def matching_pattern_map @@ -127,6 +121,12 @@ module Gitlab project.id end + # Not including a leading slash + def path_relative_to_upload_dir + base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} + @path_relative_to_upload_dir ||= path.sub(base, '') + end + def absolute_path File.join(CarrierWave.root, path) end -- cgit v1.2.1 From 3694fe0f3d43619174878805ba09f0943e1dbaad Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Fri, 24 Nov 2017 10:04:44 -0800 Subject: =?UTF-8?q?Don=E2=80=99t=20quote=20`NOW()`=20for=20created=5Fat=20?= =?UTF-8?q?column?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To fix for MySQL. --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 2 +- lib/gitlab/database.rb | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index f06b4ac1940..ea04484c6a1 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -204,7 +204,7 @@ module Gitlab file.to_h.merge(created_at: 'NOW()') end - Gitlab::Database.bulk_insert('uploads', rows) + Gitlab::Database.bulk_insert('uploads', rows, disable_quote: :created_at) end def drop_temp_table_if_finished diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index cd7b4c043da..16308b308b2 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -116,15 +116,21 @@ module Gitlab # values. # return_ids - When set to true the return value will be an Array of IDs of # the inserted rows, this only works on PostgreSQL. - def self.bulk_insert(table, rows, return_ids: false) + # disable_quote - A key or an Array of keys to exclude from quoting (You + # become responsible for protection from SQL injection for + # these keys!) + def self.bulk_insert(table, rows, return_ids: false, disable_quote: []) return if rows.empty? keys = rows.first.keys columns = keys.map { |key| connection.quote_column_name(key) } return_ids = false if mysql? + disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| - row.values_at(*keys).map { |value| connection.quote(value) } + row.keys.map do |k| + disable_quote.include?(k) ? row[k] : connection.quote(row[k]) + end end sql = <<-EOF -- cgit v1.2.1 From e5cf23dfcf22f5a82ec027212d0a178894ea2396 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 20:37:23 -0800 Subject: Ensure consistent column order --- lib/gitlab/database.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 16308b308b2..4c40e8a528c 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -128,7 +128,7 @@ module Gitlab disable_quote = Array(disable_quote).to_set tuples = rows.map do |row| - row.keys.map do |k| + keys.map do |k| disable_quote.include?(k) ? row[k] : connection.quote(row[k]) end end -- cgit v1.2.1 From dd4b35f864e43b94532c2229e0135eaa47ddc9fe Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 20:57:51 -0800 Subject: Fix test for MySQL --- .../background_migration/populate_untracked_uploads_spec.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 85da8ac5b1e..7c69755c4cf 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -71,8 +71,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end it 'uses the start and end batch ids [only 1st half]' do - start_id = untracked_files_for_uploads.all.to_a[0].id - end_id = untracked_files_for_uploads.all.to_a[3].id + ids = untracked_files_for_uploads.all.order(:id).pluck(:id) + start_id = ids[0] + end_id = ids[3] expect do subject.perform(start_id, end_id) @@ -89,8 +90,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end it 'uses the start and end batch ids [only 2nd half]' do - start_id = untracked_files_for_uploads.all.to_a[4].id - end_id = untracked_files_for_uploads.all.to_a[7].id + ids = untracked_files_for_uploads.all.order(:id).pluck(:id) + start_id = ids[4] + end_id = ids[7] expect do subject.perform(start_id, end_id) -- cgit v1.2.1 From 74b3870a958cadf35fd3c13a78334c96d46de939 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 22:57:21 -0800 Subject: Address Rubocop offenses --- .../populate_untracked_uploads.rb | 64 +++++++++++++++------- .../prepare_untracked_uploads.rb | 57 +++++++++++++------ 2 files changed, 82 insertions(+), 39 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index ea04484c6a1..802b661886b 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -1,12 +1,18 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration - class PopulateUntrackedUploads - class UntrackedFile < ActiveRecord::Base + # This class processes a batch of rows in `untracked_files_for_uploads` by + # adding each file to the `uploads` table if it does not exist. + class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength + # This class is responsible for producing the attributes necessary to + # track an uploaded file in the `uploads` table. + class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength self.table_name = 'untracked_files_for_uploads' # Ends with /:random_hex/:filename - FILE_UPLOADER_PATH_PATTERN = %r{/\h+/[^/]+\z} - FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN = %r{\A(.+)#{FILE_UPLOADER_PATH_PATTERN}} + FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} + FULL_PATH_CAPTURE = %r{\A(.+)#{FILE_UPLOADER_PATH}} # These regex patterns are tested against a relative path, relative to # the upload directory. @@ -44,7 +50,7 @@ module Gitlab model_type: 'Project' }, { - pattern: FILE_UPLOADER_PATH_PATTERN, + pattern: FILE_UPLOADER_PATH, uploader: 'FileUploader', model_type: 'Project' } @@ -63,13 +69,14 @@ module Gitlab def upload_path # UntrackedFile#path is absolute, but Upload#path depends on uploader - @upload_path ||= if uploader == 'FileUploader' - # Path relative to project directory in uploads - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH_PATTERN) - matchd[0].sub(%r{\A/}, '') # remove leading slash - else - path - end + @upload_path ||= + if uploader == 'FileUploader' + # Path relative to project directory in uploads + matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_PATH) + matchd[0].sub(%r{\A/}, '') # remove leading slash + else + path + end end def uploader @@ -83,7 +90,8 @@ module Gitlab def model_id return @model_id if defined?(@model_id) - matchd = path_relative_to_upload_dir.match(matching_pattern_map[:pattern]) + pattern = matching_pattern_map[:pattern] + matchd = path_relative_to_upload_dir.match(pattern) # If something is captured (matchd[1] is not nil), it is a model_id # Only the FileUploader pattern will not match an ID @@ -105,14 +113,20 @@ module Gitlab path_relative_to_upload_dir.match(path_pattern_map[:pattern]) end - raise "Unknown upload path pattern \"#{path}\"" unless @matching_pattern_map + unless @matching_pattern_map + raise "Unknown upload path pattern \"#{path}\"" + end @matching_pattern_map end def file_uploader_model_id - matchd = path_relative_to_upload_dir.match(FILE_UPLOADER_CAPTURE_FULL_PATH_PATTERN) - raise "Could not capture project full_path from a FileUploader path: \"#{path_relative_to_upload_dir}\"" unless matchd + matchd = path_relative_to_upload_dir.match(FULL_PATH_CAPTURE) + not_found_msg = <<~MSG + Could not capture project full_path from a FileUploader path: + "#{path_relative_to_upload_dir}" + MSG + raise not_found_msg unless matchd full_path = matchd[1] project = Project.find_by_full_path(full_path) @@ -123,7 +137,8 @@ module Gitlab # Not including a leading slash def path_relative_to_upload_dir - base = %r{\A#{Regexp.escape(Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR)}/} + upload_dir = Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR # rubocop:disable Metrics/LineLength + base = %r{\A#{Regexp.escape(upload_dir)}/} @path_relative_to_upload_dir ||= path.sub(base, '') end @@ -132,6 +147,7 @@ module Gitlab end end + # This class is used to query the `uploads` table. class Upload < ActiveRecord::Base self.table_name = 'uploads' end @@ -192,8 +208,10 @@ module Gitlab end ids.each do |model_type, model_ids| - found_ids = Object.const_get(model_type).where(id: model_ids.uniq).pluck(:id) - ids[model_type] = ids[model_type] - found_ids # replace with deleted ids + model_class = Object.const_get(model_type) + found_ids = model_class.where(id: model_ids.uniq).pluck(:id) + deleted_ids = ids[model_type] - found_ids + ids[model_type] = deleted_ids end ids @@ -204,11 +222,15 @@ module Gitlab file.to_h.merge(created_at: 'NOW()') end - Gitlab::Database.bulk_insert('uploads', rows, disable_quote: :created_at) + Gitlab::Database.bulk_insert('uploads', + rows, + disable_quote: :created_at) end def drop_temp_table_if_finished - UntrackedFile.connection.drop_table(:untracked_files_for_uploads) if UntrackedFile.all.empty? + if UntrackedFile.all.empty? + UntrackedFile.connection.drop_table(:untracked_files_for_uploads) + end end end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 358b76d39fb..d8cddd98cbb 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -1,10 +1,14 @@ +# frozen_string_literal: true + module Gitlab module BackgroundMigration - class PrepareUntrackedUploads + # This class finds all non-hashed uploaded file paths and saves them to a + # `untracked_files_for_uploads` table. + class PrepareUntrackedUploads # rubocop:disable Metrics/ClassLength # For bulk_queue_background_migration_jobs_by_range include Database::MigrationHelpers - FILE_PATH_BATCH_SIZE = 500 + FIND_BATCH_SIZE = 500 RELATIVE_UPLOAD_DIR = "uploads".freeze ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze @@ -12,6 +16,8 @@ module Gitlab EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze + # This class is used to iterate over batches of + # `untracked_files_for_uploads` rows. class UntrackedFile < ActiveRecord::Base include EachBatch @@ -39,8 +45,9 @@ module Gitlab private def ensure_temporary_tracking_table_exists - unless UntrackedFile.connection.table_exists?(:untracked_files_for_uploads) - UntrackedFile.connection.create_table :untracked_files_for_uploads do |t| + table_name = :untracked_files_for_uploads + unless UntrackedFile.connection.table_exists?(table_name) + UntrackedFile.connection.create_table table_name do |t| t.string :path, limit: 600, null: false t.index :path, unique: true end @@ -54,7 +61,7 @@ module Gitlab def store_untracked_file_paths return unless Dir.exist?(ABSOLUTE_UPLOAD_DIR) - each_file_batch(ABSOLUTE_UPLOAD_DIR, FILE_PATH_BATCH_SIZE) do |file_paths| + each_file_batch(ABSOLUTE_UPLOAD_DIR, FIND_BATCH_SIZE) do |file_paths| insert_file_paths(file_paths) end end @@ -85,12 +92,17 @@ module Gitlab end def build_find_command(search_dir) - cmd = %W[find #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -print0] + cmd = %W[find #{search_dir} + -type f + ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) + ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) + -print0] ionice = which_ionice cmd = %W[#{ionice} -c Idle] + cmd if ionice - Rails.logger.info "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + log_msg = "PrepareUntrackedUploads find command: \"#{cmd.join(' ')}\"" + Rails.logger.info log_msg cmd end @@ -98,25 +110,32 @@ module Gitlab def which_ionice Gitlab::Utils.which('ionice') rescue StandardError - # In this case, returning false is relatively safe, even though it isn't very nice + # In this case, returning false is relatively safe, + # even though it isn't very nice false end def insert_file_paths(file_paths) - sql = if postgresql_pre_9_5? - "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" - elsif postgresql? - "INSERT INTO #{table_columns_and_values_for_insert(file_paths)} ON CONFLICT DO NOTHING;" - else # MySQL - "INSERT IGNORE INTO #{table_columns_and_values_for_insert(file_paths)};" - end + sql = insert_sql(file_paths) ActiveRecord::Base.connection.execute(sql) end + def insert_sql(file_paths) + if postgresql_pre_9_5? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)};" + elsif postgresql? + "INSERT INTO #{table_columns_and_values_for_insert(file_paths)}"\ + " ON CONFLICT DO NOTHING;" + else # MySQL + "INSERT IGNORE INTO"\ + " #{table_columns_and_values_for_insert(file_paths)};" + end + end + def table_columns_and_values_for_insert(file_paths) values = file_paths.map do |file_path| - ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend + ActiveRecord::Base.send(:sanitize_sql_array, ['(?)', file_path]) # rubocop:disable GitlabSecurity/PublicSend, Metrics/LineLength end.join(', ') "#{UntrackedFile.table_name} (path) VALUES #{values}" @@ -131,11 +150,13 @@ module Gitlab end def postgresql_pre_9_5? - @postgresql_pre_9_5 ||= postgresql? && Gitlab::Database.version.to_f < 9.5 + @postgresql_pre_9_5 ||= postgresql? && + Gitlab::Database.version.to_f < 9.5 end def schedule_populate_untracked_uploads_jobs - bulk_queue_background_migration_jobs_by_range(UntrackedFile, FOLLOW_UP_MIGRATION) + bulk_queue_background_migration_jobs_by_range( + UntrackedFile, FOLLOW_UP_MIGRATION) end end end -- cgit v1.2.1 From 71d27044189bf114904e798017e541697acad8e9 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Sun, 26 Nov 2017 23:10:51 -0800 Subject: Add tests for disable_quote option --- spec/lib/gitlab/database_spec.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index fcddfad3f9f..8872bf7fc87 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -199,6 +199,22 @@ describe Gitlab::Database do described_class.bulk_insert('test', rows) end + it 'does not quote values of a column in the disable_quote option' do + [1, 2, 4, 5].each do |i| + expect(connection).to receive(:quote).with(i) + end + + described_class.bulk_insert('test', rows, disable_quote: :c) + end + + it 'does not quote values of columns in the disable_quote option' do + [2, 5].each do |i| + expect(connection).to receive(:quote).with(i) + end + + described_class.bulk_insert('test', rows, disable_quote: [:a, :c]) + end + it 'handles non-UTF-8 data' do expect { described_class.bulk_insert('test', [{ a: "\255" }]) }.not_to raise_error end -- cgit v1.2.1 From ca817816801c26847b66d105d07fd4473b015b31 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 27 Nov 2017 09:33:13 -0800 Subject: Handle race condition --- lib/gitlab/background_migration/populate_untracked_uploads.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 802b661886b..ebb483c3cff 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -229,7 +229,8 @@ module Gitlab def drop_temp_table_if_finished if UntrackedFile.all.empty? - UntrackedFile.connection.drop_table(:untracked_files_for_uploads) + UntrackedFile.connection.drop_table(:untracked_files_for_uploads, + if_exists: true) end end end -- cgit v1.2.1 From fcbd2fe62568c55c86de4eeb5969bfe4d82af6ce Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 4 Dec 2017 12:57:44 -0800 Subject: Follow symlinks In particular, the Omnibus uploads directory is generally a symlink. --- lib/gitlab/background_migration/prepare_untracked_uploads.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index d8cddd98cbb..476c46341ae 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -92,7 +92,7 @@ module Gitlab end def build_find_command(search_dir) - cmd = %W[find #{search_dir} + cmd = %W[find -L #{search_dir} -type f ! ( -path #{EXCLUDED_HASHED_UPLOADS_PATH} -prune ) ! ( -path #{EXCLUDED_TMP_UPLOADS_PATH} -prune ) -- cgit v1.2.1 From 602f6bc89c9464fabab827b833133439af26a3c4 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Mon, 4 Dec 2017 12:58:02 -0800 Subject: =?UTF-8?q?Make=20sure=20empty=20uploads=20doesn=E2=80=99t=20break?= =?UTF-8?q?=20anything?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- spec/migrations/track_untracked_uploads_spec.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index 9fa586ff177..c9bbc09cc49 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -5,6 +5,7 @@ describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } + let(:uploads) { table(:uploads) } matcher :be_scheduled_migration do match do |migration| @@ -33,7 +34,6 @@ describe TrackUntrackedUploads, :migration, :sidekiq do let!(:user2) { create(:user, :with_avatar) } let!(:project1) { create(:project, :with_avatar) } let!(:project2) { create(:project, :with_avatar) } - let(:uploads) { table(:uploads) } before do UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload @@ -90,4 +90,12 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(table_exists?(:untracked_files_for_uploads)).to be_falsey end end + + context 'without any uploads ever' do + it 'does not add any upload records' do + expect do + migrate! + end.not_to change { uploads.count }.from(0) + end + end end -- cgit v1.2.1 From 77be7efcf58e0c6cf7a16c249161c0c791c545e1 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 5 Dec 2017 10:43:08 -0800 Subject: Guarantee all IDs are included --- .../background_migration/populate_untracked_uploads_spec.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 7c69755c4cf..35ea8059510 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -46,7 +46,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'adds untracked files to the uploads table' do expect do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) end.to change { uploads.count }.from(4).to(8) expect(user2.uploads.count).to eq(1) @@ -58,12 +58,12 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(subject).to receive(:drop_temp_table_if_finished) # Don't drop the table so we can look at it expect do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) end.to change { untracked_files_for_uploads.count }.from(8).to(0) end it 'does not create duplicate uploads of already tracked files' do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) expect(user1.uploads.count).to eq(1) expect(project1.uploads.count).to eq(2) @@ -140,7 +140,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'creates an Upload record' do expect do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(expected_upload_attrs) @@ -206,7 +206,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'creates an Upload record' do expect do - subject.perform(1, 1000) + subject.perform(1, untracked_files_for_uploads.last.id) end.to change { model.reload.uploads.count }.from(0).to(1) expect(model.uploads.first.attributes).to include(@expected_upload_attrs) -- cgit v1.2.1 From 869d08b581495161352a661ac29b20b3925deaf0 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 5 Dec 2017 12:26:20 -0800 Subject: Process normal paths in batch containing bad paths --- .../populate_untracked_uploads.rb | 29 +++++++++++++++++++--- .../populate_untracked_uploads_spec.rb | 20 +++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index ebb483c3cff..81e95e5832d 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -57,7 +57,7 @@ module Gitlab ].freeze def to_h - { + @upload_hash ||= { path: upload_path, uploader: uploader, model_type: model_type, @@ -156,8 +156,8 @@ module Gitlab return unless migrate? files = UntrackedFile.where(id: start_id..end_id) - insert_uploads_if_needed(files) - files.delete_all + processed_files = insert_uploads_if_needed(files) + processed_files.delete_all drop_temp_table_if_finished end @@ -169,9 +169,30 @@ module Gitlab end def insert_uploads_if_needed(files) - filtered_files = filter_existing_uploads(files) + filtered_files, error_files = filter_error_files(files) + filtered_files = filter_existing_uploads(filtered_files) filtered_files = filter_deleted_models(filtered_files) insert(filtered_files) + + processed_files = files.where.not(id: error_files.map(&:id)) + processed_files + end + + def filter_error_files(files) + files.partition do |file| + begin + file.to_h + true + rescue => e + msg = <<~MSG + Error parsing path "#{file.path}": + #{e.message} + #{e.backtrace.join("\n ")} + MSG + Rails.logger.error(msg) + false + end + end end def filter_existing_uploads(files) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 35ea8059510..e1a5a17a60c 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -119,6 +119,26 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid expect(table_exists?(:untracked_files_for_uploads)).to be_falsey end + + it 'does not block a whole batch because of one bad path' do + untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a") + expect(untracked_files_for_uploads.count).to eq(9) + expect(uploads.count).to eq(4) + + subject.perform(1, untracked_files_for_uploads.last.id) + + expect(untracked_files_for_uploads.count).to eq(1) + expect(uploads.count).to eq(8) + end + + it 'an unparseable path is shown in error output' do + bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a" + untracked_files_for_uploads.create!(path: bad_path) + + expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/) + + subject.perform(1, untracked_files_for_uploads.last.id) + end end context 'with no untracked files' do -- cgit v1.2.1 From 03cba8c0f18f18a14453b17c9f4fa300547f0ab5 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Tue, 5 Dec 2017 23:08:45 -0800 Subject: Fix specs after rebase Later migrations added fields to the EE DB which were used by factories which were used in these specs. And in CE on MySQL, a single appearance row is enforced. The migration and migration specs should not depend on the codebase staying the same. --- .../populate_untracked_uploads_spec.rb | 19 +++--- .../prepare_untracked_uploads_spec.rb | 10 +-- spec/migrations/track_untracked_uploads_spec.rb | 74 ---------------------- spec/support/track_untracked_uploads_helpers.rb | 6 ++ 4 files changed, 20 insertions(+), 89 deletions(-) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index e1a5a17a60c..9c1261881df 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -1,13 +1,12 @@ require 'spec_helper' -require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_uploads') -describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq do include TrackUntrackedUploadsHelpers subject { described_class.new } - let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } - let!(:uploads) { table(:uploads) } + let!(:untracked_files_for_uploads) { described_class::UntrackedFile } + let!(:uploads) { described_class::Upload } before do ensure_temporary_tracking_table_exists @@ -18,7 +17,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end context 'with untracked files and tracked files in untracked_files_for_uploads' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) } let!(:user1) { create(:user, :with_avatar) } let!(:user2) { create(:user, :with_avatar) } let!(:project1) { create(:project, :with_avatar) } @@ -111,13 +110,13 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid it 'does not drop the temporary tracking table after processing the batch, if there are still untracked rows' do subject.perform(1, untracked_files_for_uploads.last.id - 1) - expect(table_exists?(:untracked_files_for_uploads)).to be_truthy + expect(ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads)).to be_truthy end it 'drops the temporary tracking table after processing the batch, if there are no untracked rows left' do subject.perform(1, untracked_files_for_uploads.last.id) - expect(table_exists?(:untracked_files_for_uploads)).to be_falsey + expect(ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads)).to be_falsey end it 'does not block a whole batch because of one bad path' do @@ -168,13 +167,13 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sid end context 'for an appearance logo file path' do - let(:model) { create(:appearance, logo: uploaded_file) } + let(:model) { create_or_update_appearance(logo: uploaded_file) } it_behaves_like 'non_markdown_file' end context 'for an appearance header_logo file path' do - let(:model) { create(:appearance, header_logo: uploaded_file) } + let(:model) { create_or_update_appearance(header_logo: uploaded_file) } it_behaves_like 'non_markdown_file' end @@ -459,7 +458,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do describe '#file_size' do context 'for an appearance logo file path' do - let(:appearance) { create(:appearance, logo: uploaded_file) } + let(:appearance) { create_or_update_appearance(logo: uploaded_file) } let(:untracked_file) { described_class.create!(path: appearance.uploads.first.path) } it 'returns the file size' do diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index f1eb7173717..dfe37e43751 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -1,9 +1,9 @@ require 'spec_helper' -describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do +describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do include TrackUntrackedUploadsHelpers - let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } + let!(:untracked_files_for_uploads) { described_class::UntrackedFile } matcher :be_scheduled_migration do |*expected| match do |migration| @@ -35,7 +35,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side it 'ensures the untracked_files_for_uploads table exists' do expect do described_class.new.perform - end.to change { table_exists?(:untracked_files_for_uploads) }.from(false).to(true) + end.to change { ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) }.from(false).to(true) end it 'has a path field long enough for really long paths' do @@ -63,7 +63,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end context 'when files were uploaded before and after hashed storage was enabled' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) } let!(:user) { create(:user, :with_avatar) } let!(:project1) { create(:project, :with_avatar) } let(:project2) { create(:project) } # instantiate after enabling hashed_storage @@ -151,7 +151,7 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :migration, :side end context 'when files were uploaded before and after hashed storage was enabled' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } + let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) } let!(:user) { create(:user, :with_avatar) } let!(:project1) { create(:project, :with_avatar) } let(:project2) { create(:project) } # instantiate after enabling hashed_storage diff --git a/spec/migrations/track_untracked_uploads_spec.rb b/spec/migrations/track_untracked_uploads_spec.rb index c9bbc09cc49..7fe7a140e2f 100644 --- a/spec/migrations/track_untracked_uploads_spec.rb +++ b/spec/migrations/track_untracked_uploads_spec.rb @@ -4,9 +4,6 @@ require Rails.root.join('db', 'post_migrate', '20171103140253_track_untracked_up describe TrackUntrackedUploads, :migration, :sidekiq do include TrackUntrackedUploadsHelpers - let(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) } - let(:uploads) { table(:uploads) } - matcher :be_scheduled_migration do match do |migration| BackgroundMigrationWorker.jobs.any? do |job| @@ -27,75 +24,4 @@ describe TrackUntrackedUploads, :migration, :sidekiq do expect(BackgroundMigrationWorker.jobs.size).to eq(1) end end - - context 'with tracked and untracked uploads' do - let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) } - let!(:user1) { create(:user, :with_avatar) } - let!(:user2) { create(:user, :with_avatar) } - let!(:project1) { create(:project, :with_avatar) } - let!(:project2) { create(:project, :with_avatar) } - - before do - UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload - UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload - - # Save expected Upload attributes - @appearance_logo_attributes = appearance.uploads.where("path like '%/logo/%'").first.attributes.slice('path', 'uploader', 'size', 'checksum') - @appearance_header_logo_attributes = appearance.uploads.where("path like '%/header_logo/%'").first.attributes.slice('path', 'uploader', 'size', 'checksum') - @user1_avatar_attributes = user1.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') - @user2_avatar_attributes = user2.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') - @project1_avatar_attributes = project1.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') - @project2_avatar_attributes = project2.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') - @project1_markdown_attributes = project1.uploads.last.attributes.slice('path', 'uploader', 'size', 'checksum') - @project2_markdown_attributes = project2.uploads.last.attributes.slice('path', 'uploader', 'size', 'checksum') - - # Untrack 4 files - user2.uploads.delete_all - project2.uploads.delete_all # 2 files: avatar and a Markdown upload - appearance.uploads.where("path like '%header_logo%'").delete_all - end - - it 'tracks untracked uploads' do - expect do - migrate! - end.to change { uploads.count }.from(4).to(8) - - expect(appearance.reload.uploads.where("path like '%/header_logo/%'").first.attributes).to include(@appearance_header_logo_attributes) - expect(user2.reload.uploads.first.attributes).to include(@user2_avatar_attributes) - expect(project2.reload.uploads.where(uploader: 'AvatarUploader').first.attributes).to include(@project2_avatar_attributes) - expect(project2.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project2_markdown_attributes) - end - - it 'ignores already-tracked uploads' do - migrate! - - expect(appearance.reload.uploads.where("path like '%/logo/%'").first.attributes).to include(@appearance_logo_attributes) - expect(user1.reload.uploads.first.attributes).to include(@user1_avatar_attributes) - expect(project1.reload.uploads.where(uploader: 'AvatarUploader').first.attributes).to include(@project1_avatar_attributes) - expect(project1.uploads.where(uploader: 'FileUploader').first.attributes).to include(@project1_markdown_attributes) - end - - it 'ignores uploads for deleted models' do - user2.destroy - project2.destroy - - expect do - migrate! - end.to change { uploads.count }.from(4).to(5) - end - - it 'the temporary table untracked_files_for_uploads no longer exists' do - migrate! - - expect(table_exists?(:untracked_files_for_uploads)).to be_falsey - end - end - - context 'without any uploads ever' do - it 'does not add any upload records' do - expect do - migrate! - end.not_to change { uploads.count }.from(0) - end - end end diff --git a/spec/support/track_untracked_uploads_helpers.rb b/spec/support/track_untracked_uploads_helpers.rb index 4d4745fd7f4..d05eda08201 100644 --- a/spec/support/track_untracked_uploads_helpers.rb +++ b/spec/support/track_untracked_uploads_helpers.rb @@ -11,4 +11,10 @@ module TrackUntrackedUploadsHelpers def drop_temp_table_if_exists ActiveRecord::Base.connection.drop_table(:untracked_files_for_uploads) if ActiveRecord::Base.connection.table_exists?(:untracked_files_for_uploads) end + + def create_or_update_appearance(attrs) + a = Appearance.first_or_initialize(title: 'foo', description: 'bar') + a.update!(attrs) + a + end end -- cgit v1.2.1 From 24c348f0d1270fe27268aa23e034473651b0cdf9 Mon Sep 17 00:00:00 2001 From: Michael Kozono Date: Wed, 6 Dec 2017 12:20:06 -0800 Subject: Fix specs for MySQL --- .../lib/gitlab/background_migration/populate_untracked_uploads_spec.rb | 3 +++ spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb | 2 ++ 2 files changed, 5 insertions(+) diff --git a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb index 9c1261881df..b80df6956b0 100644 --- a/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb @@ -9,7 +9,10 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq do let!(:uploads) { described_class::Upload } before do + DatabaseCleaner.clean + drop_temp_table_if_exists ensure_temporary_tracking_table_exists + uploads.delete_all end after(:all) do diff --git a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb index dfe37e43751..cd3f1a45270 100644 --- a/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb +++ b/spec/lib/gitlab/background_migration/prepare_untracked_uploads_spec.rb @@ -18,6 +18,8 @@ describe Gitlab::BackgroundMigration::PrepareUntrackedUploads, :sidekiq do end before do + DatabaseCleaner.clean + drop_temp_table_if_exists end -- cgit v1.2.1