diff options
author | Gabriel Mazetto <brodock@gmail.com> | 2017-08-04 07:30:42 +0200 |
---|---|---|
committer | Gabriel Mazetto <brodock@gmail.com> | 2017-08-22 06:33:20 +0200 |
commit | 9e6fa996eab978506af1084b79a9c3f91f6d575b (patch) | |
tree | 39cdc0aafa13c16551f4f7f244a19e94d6b8582f | |
parent | 53403399577bdca0e8f0886fa62ce0e75c14a8e0 (diff) | |
download | gitlab-ce-9e6fa996eab978506af1084b79a9c3f91f6d575b.tar.gz |
New storage is now "Hashed" instead of "UUID"
-rw-r--r-- | app/models/concerns/storage/hashed_project.rb (renamed from app/models/concerns/storage/uuid_project.rb) | 24 | ||||
-rw-r--r-- | app/models/concerns/storage/legacy_project.rb | 12 | ||||
-rw-r--r-- | app/models/project.rb | 8 | ||||
-rw-r--r-- | db/migrate/20170802013652_add_storage_fields_to_project.rb | 24 | ||||
-rw-r--r-- | db/schema.rb | 5 | ||||
-rw-r--r-- | lib/gitlab/import_export/import_export.yml | 1 | ||||
-rw-r--r-- | spec/factories/projects.rb | 4 | ||||
-rw-r--r-- | spec/models/project_spec.rb | 213 |
8 files changed, 203 insertions, 88 deletions
diff --git a/app/models/concerns/storage/uuid_project.rb b/app/models/concerns/storage/hashed_project.rb index 8a73287e518..292a73903b5 100644 --- a/app/models/concerns/storage/uuid_project.rb +++ b/app/models/concerns/storage/hashed_project.rb @@ -1,17 +1,23 @@ module Storage - module UUIDProject + module HashedProject extend ActiveSupport::Concern - def uuid_dir - %Q(#{uuid[0..1]}/#{uuid[2..3]}) + # Base directory + # + # @return [String] directory where repository is stored + def base_dir + %Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash end + # Disk path is used to build repository and project's wiki path on disk + # + # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions def disk_path - %Q(#{uuid_dir}/#{uuid}) + %Q(#{base_dir}/#{disk_hash}) end def ensure_storage_path_exist - gitlab_shell.add_namespace(repository_storage_path, uuid_dir) + gitlab_shell.add_namespace(repository_storage_path, base_dir) end def rename_repo @@ -54,5 +60,13 @@ module Storage Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path) Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path) end + + private + + # Generates the hash for the project path and name on disk + # If you need to refer to the repository on disk, use the `#disk_path` + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(self.id.to_s) if self.id + end end end diff --git a/app/models/concerns/storage/legacy_project.rb b/app/models/concerns/storage/legacy_project.rb index c38042495f4..839bbcc76ea 100644 --- a/app/models/concerns/storage/legacy_project.rb +++ b/app/models/concerns/storage/legacy_project.rb @@ -2,12 +2,22 @@ module Storage module LegacyProject extend ActiveSupport::Concern + # Base directory + # + # @return [String] directory where repository is stored + def base_dir + namespace.full_path + end + + # Disk path is used to build repository and project's wiki path on disk + # + # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions def disk_path full_path end def ensure_storage_path_exist - gitlab_shell.add_namespace(repository_storage_path, namespace.full_path) + gitlab_shell.add_namespace(repository_storage_path, base_dir) end def rename_repo diff --git a/app/models/project.rb b/app/models/project.rb index f0ea83dda0c..0e000df9790 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1424,10 +1424,12 @@ class Project < ActiveRecord::Base private def load_storage - if self.storage_version > 1 - self.class.include Storage::UUIDProject + return unless has_attribute?(:storage_version) + + if self.storage_version && self.storage_version >= 1 + self.extend Storage::HashedProject else - self.class.include Storage::LegacyProject + self.extend Storage::LegacyProject end end diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb index 269103cd472..a0815da0fcd 100644 --- a/db/migrate/20170802013652_add_storage_fields_to_project.rb +++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb @@ -4,36 +4,16 @@ class AddStorageFieldsToProject < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers - # Set this constant to true if this migration requires downtime. DOWNTIME = false - - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index", "remove_concurrent_index" or - # "add_column_with_default" you must disable the use of transactions - # as these methods can not run in an existing transaction. - # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure - # that either of them is the _only_ method called in the migration, - # any other changes should go in a separate migration. - # This ensures that upon failure _only_ the index creation or removing fails - # and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: disable_ddl_transaction! def up # rubocop:disable Migration/AddColumnWithDefaultToLargeTable - add_column :projects, :uuid, :uuid - add_column_with_default :projects, :storage_version, :integer, default: 0, limit: 1 - add_concurrent_index :projects, :uuid + add_column :projects, :storage_version, :integer, limit: 2 + add_concurrent_index :projects, :storage_version end def down - remove_column :projects, :uuid remove_column :projects, :storage_version end end diff --git a/db/schema.rb b/db/schema.rb index dcd9532e4be..5a85a00bb12 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1208,8 +1208,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do t.datetime "last_repository_updated_at" t.string "ci_config_path" t.text "delete_error" - t.uuid "uuid" - t.integer "storage_version", limit: 2, default: 0, null: false + t.integer "storage_version", limit: 2 end add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree @@ -1226,7 +1225,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree - add_index "projects", ["uuid"], name: "index_projects_on_uuid", using: :btree + add_index "projects", ["storage_version"], name: "index_projects_on_storage_version", using: :btree add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree create_table "protected_branch_merge_access_levels", force: :cascade do |t| diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 9d9ebcb389a..894950e341f 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -98,6 +98,7 @@ excluded_attributes: - :last_activity_at - :last_repository_updated_at - :last_repository_check_at + - :storage_version snippets: - :expired_at merge_request_diff: diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index 4a2034b31b3..c6f1da82f3c 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -81,6 +81,10 @@ FactoryGirl.define do archived true end + trait :hashed do + storage_version 1 + end + trait :access_requestable do request_access_enabled true end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 5e60511f3a8..e6fde833c6b 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1251,60 +1251,6 @@ describe Project do end end - describe '#rename_repo' do - let(:project) { create(:project, :repository) } - let(:gitlab_shell) { Gitlab::Shell.new } - - before do - # Project#gitlab_shell returns a new instance of Gitlab::Shell on every - # call. This makes testing a bit easier. - allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) - allow(project).to receive(:previous_changes).and_return('path' => ['foo']) - end - - it 'renames a repository' do - stub_container_registry_config(enabled: false) - - expect(gitlab_shell).to receive(:mv_repository) - .ordered - .with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}") - .and_return(true) - - expect(gitlab_shell).to receive(:mv_repository) - .ordered - .with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki") - .and_return(true) - - expect_any_instance_of(SystemHooksService) - .to receive(:execute_hooks_for) - .with(project, :rename) - - expect_any_instance_of(Gitlab::UploadsTransfer) - .to receive(:rename_project) - .with('foo', project.path, project.namespace.full_path) - - expect(project).to receive(:expire_caches_before_rename) - - expect(project).to receive(:expires_full_path_cache) - - project.rename_repo - end - - context 'container registry with images' do - let(:container_repository) { create(:container_repository) } - - before do - stub_container_registry_config(enabled: true) - stub_container_registry_tags(repository: :any, tags: ['tag']) - project.container_repositories << container_repository - end - - subject { project.rename_repo } - - it { expect {subject}.to raise_error(StandardError) } - end - end - describe '#expire_caches_before_rename' do let(:project) { create(:project, :repository) } let(:repo) { double(:repo, exists?: true) } @@ -2367,4 +2313,163 @@ describe Project do expect(project.forks_count).to eq(1) end end + + context 'legacy storage' do + let(:project) { create(:project, :repository) } + let(:gitlab_shell) { Gitlab::Shell.new } + + describe '#base_dir' do + it 'returns base_dir based on namespace only' do + expect(project.base_dir).to eq(project.namespace.full_path) + end + end + + describe '#disk_path' do + it 'returns disk_path based on namespace and project path' do + expect(project.disk_path).to eq("#{project.namespace.full_path}/#{project.path}") + end + end + + describe '#ensure_storage_path_exist' do + before do + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + end + + it 'delegates to gitlab_shell to ensure namespace is created' do + expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, project.base_dir) + + project.ensure_storage_path_exist + end + end + + describe '#rename_repo' do + before do + # Project#gitlab_shell returns a new instance of Gitlab::Shell on every + # call. This makes testing a bit easier. + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + allow(project).to receive(:previous_changes).and_return('path' => ['foo']) + end + + it 'renames a repository' do + stub_container_registry_config(enabled: false) + + expect(gitlab_shell).to receive(:mv_repository) + .ordered + .with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}") + .and_return(true) + + expect(gitlab_shell).to receive(:mv_repository) + .ordered + .with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki") + .and_return(true) + + expect_any_instance_of(SystemHooksService) + .to receive(:execute_hooks_for) + .with(project, :rename) + + expect_any_instance_of(Gitlab::UploadsTransfer) + .to receive(:rename_project) + .with('foo', project.path, project.namespace.full_path) + + expect(project).to receive(:expire_caches_before_rename) + + expect(project).to receive(:expires_full_path_cache) + + project.rename_repo + end + + context 'container registry with images' do + let(:container_repository) { create(:container_repository) } + + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags(repository: :any, tags: ['tag']) + project.container_repositories << container_repository + end + + subject { project.rename_repo } + + it { expect{subject}.to raise_error(StandardError) } + end + end + end + + context 'hashed storage' do + let(:project) { create(:project, :repository, :hashed) } + let(:gitlab_shell) { Gitlab::Shell.new } + let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' } + + before do + allow(Digest::SHA2).to receive(:hexdigest) { hash } + end + + describe '#base_dir' do + it 'returns base_dir based on hash of project id' do + expect(project.base_dir).to eq('6b/86') + end + end + + describe '#disk_path' do + it 'returns disk_path based on has of project id' do + hashed_path = '6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' + + expect(project.disk_path).to eq(hashed_path) + end + end + + describe '#ensure_storage_path_exist' do + before do + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + end + + it 'delegates to gitlab_shell to ensure namespace is created' do + expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '6b/86') + + project.ensure_storage_path_exist + end + end + + describe '#rename_repo' do + before do + # Project#gitlab_shell returns a new instance of Gitlab::Shell on every + # call. This makes testing a bit easier. + allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) + allow(project).to receive(:previous_changes).and_return('path' => ['foo']) + end + + it 'renames a repository' do + stub_container_registry_config(enabled: false) + + expect(gitlab_shell).not_to receive(:mv_repository) + + expect_any_instance_of(SystemHooksService) + .to receive(:execute_hooks_for) + .with(project, :rename) + + expect_any_instance_of(Gitlab::UploadsTransfer) + .to receive(:rename_project) + .with('foo', project.path, project.namespace.full_path) + + expect(project).to receive(:expire_caches_before_rename) + + expect(project).to receive(:expires_full_path_cache) + + project.rename_repo + end + + context 'container registry with images' do + let(:container_repository) { create(:container_repository) } + + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags(repository: :any, tags: ['tag']) + project.container_repositories << container_repository + end + + subject { project.rename_repo } + + it { expect{subject}.to raise_error(StandardError) } + end + end + end end |