New storage is now "Hashed" instead of "UUID"

8 files changed, 203 insertions, 88 deletions

diff --git a/app/models/concerns/storage/uuid_project.rb b/app/models/concerns/storage/hashed_project.rb
index 8a73287e518..292a73903b5 100644
--- a/app/models/concerns/storage/uuid_project.rb
+++ b/app/models/concerns/storage/hashed_project.rb
@@ -1,17 +1,23 @@
 module Storage
-  module UUIDProject
+  module HashedProject
     extend ActiveSupport::Concern
 
-    def uuid_dir
-      %Q(#{uuid[0..1]}/#{uuid[2..3]})
+    # Base directory
+    #
+    # @return [String] directory where repository is stored
+    def base_dir
+      %Q(#{disk_hash[0..1]}/#{disk_hash[2..3]}) if disk_hash
     end
 
+    # Disk path is used to build repository and project's wiki path on disk
+    #
+    # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
     def disk_path
-      %Q(#{uuid_dir}/#{uuid})
+      %Q(#{base_dir}/#{disk_hash})
     end
 
     def ensure_storage_path_exist
-      gitlab_shell.add_namespace(repository_storage_path, uuid_dir)
+      gitlab_shell.add_namespace(repository_storage_path, base_dir)
     end
 
     def rename_repo
@@ -54,5 +60,13 @@ module Storage
       Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
       Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
     end
+
+    private
+
+    # Generates the hash for the project path and name on disk
+    # If you need to refer to the repository on disk, use the `#disk_path`
+    def disk_hash
+      @disk_hash ||= Digest::SHA2.hexdigest(self.id.to_s) if self.id
+    end
   end
 end
diff --git a/app/models/concerns/storage/legacy_project.rb b/app/models/concerns/storage/legacy_project.rb
index c38042495f4..839bbcc76ea 100644
--- a/app/models/concerns/storage/legacy_project.rb
+++ b/app/models/concerns/storage/legacy_project.rb
@@ -2,12 +2,22 @@ module Storage
   module LegacyProject
     extend ActiveSupport::Concern
 
+    # Base directory
+    #
+    # @return [String] directory where repository is stored
+    def base_dir
+      namespace.full_path
+    end
+
+    # Disk path is used to build repository and project's wiki path on disk
+    #
+    # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
     def disk_path
       full_path
     end
 
     def ensure_storage_path_exist
-      gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
+      gitlab_shell.add_namespace(repository_storage_path, base_dir)
     end
 
     def rename_repo
diff --git a/app/models/project.rb b/app/models/project.rb
index f0ea83dda0c..0e000df9790 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1424,10 +1424,12 @@ class Project < ActiveRecord::Base
   private
 
   def load_storage
-    if self.storage_version > 1
-      self.class.include Storage::UUIDProject
+    return unless has_attribute?(:storage_version)
+
+    if self.storage_version && self.storage_version >= 1
+      self.extend Storage::HashedProject
     else
-      self.class.include Storage::LegacyProject
+      self.extend Storage::LegacyProject
     end
   end
 
diff --git a/db/migrate/20170802013652_add_storage_fields_to_project.rb b/db/migrate/20170802013652_add_storage_fields_to_project.rb
index 269103cd472..a0815da0fcd 100644
--- a/db/migrate/20170802013652_add_storage_fields_to_project.rb
+++ b/db/migrate/20170802013652_add_storage_fields_to_project.rb
@@ -4,36 +4,16 @@
 class AddStorageFieldsToProject < ActiveRecord::Migration
   include Gitlab::Database::MigrationHelpers
 
-  # Set this constant to true if this migration requires downtime.
   DOWNTIME = false
-
-  # When a migration requires downtime you **must** uncomment the following
-  # constant and define a short and easy to understand explanation as to why the
-  # migration requires downtime.
-  # DOWNTIME_REASON = ''
-
-  # When using the methods "add_concurrent_index", "remove_concurrent_index" or
-  # "add_column_with_default" you must disable the use of transactions
-  # as these methods can not run in an existing transaction.
-  # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure
-  # that either of them is the _only_ method called in the migration,
-  # any other changes should go in a separate migration.
-  # This ensures that upon failure _only_ the index creation or removing fails
-  # and can be retried or reverted easily.
-  #
-  # To disable transactions uncomment the following line and remove these
-  # comments:
   disable_ddl_transaction!
 
   def up
     # rubocop:disable Migration/AddColumnWithDefaultToLargeTable
-    add_column :projects, :uuid, :uuid
-    add_column_with_default :projects, :storage_version, :integer, default: 0, limit: 1
-    add_concurrent_index :projects, :uuid
+    add_column :projects, :storage_version, :integer, limit: 2
+    add_concurrent_index :projects, :storage_version
   end
 
   def down
-    remove_column :projects, :uuid
     remove_column :projects, :storage_version
   end
 end
diff --git a/db/schema.rb b/db/schema.rb
index dcd9532e4be..5a85a00bb12 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1208,8 +1208,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do
     t.datetime "last_repository_updated_at"
     t.string "ci_config_path"
     t.text "delete_error"
-    t.uuid "uuid"
-    t.integer "storage_version", limit: 2, default: 0, null: false
+    t.integer "storage_version", limit: 2
   end
 
   add_index "projects", ["ci_id"], name: "index_projects_on_ci_id", using: :btree
@@ -1226,7 +1225,7 @@ ActiveRecord::Schema.define(version: 20170820100558) do
   add_index "projects", ["pending_delete"], name: "index_projects_on_pending_delete", using: :btree
   add_index "projects", ["runners_token"], name: "index_projects_on_runners_token", using: :btree
   add_index "projects", ["star_count"], name: "index_projects_on_star_count", using: :btree
-  add_index "projects", ["uuid"], name: "index_projects_on_uuid", using: :btree
+  add_index "projects", ["storage_version"], name: "index_projects_on_storage_version", using: :btree
   add_index "projects", ["visibility_level"], name: "index_projects_on_visibility_level", using: :btree
 
   create_table "protected_branch_merge_access_levels", force: :cascade do |t|
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml
index 9d9ebcb389a..894950e341f 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/import_export.yml
@@ -98,6 +98,7 @@ excluded_attributes:
     - :last_activity_at
     - :last_repository_updated_at
     - :last_repository_check_at
+    - :storage_version
   snippets:
     - :expired_at
   merge_request_diff:
diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb
index 4a2034b31b3..c6f1da82f3c 100644
--- a/spec/factories/projects.rb
+++ b/spec/factories/projects.rb
@@ -81,6 +81,10 @@ FactoryGirl.define do
       archived true
     end
 
+    trait :hashed do
+      storage_version 1
+    end
+
     trait :access_requestable do
       request_access_enabled true
     end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 5e60511f3a8..e6fde833c6b 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -1251,60 +1251,6 @@ describe Project do
     end
   end
 
-  describe '#rename_repo' do
-    let(:project) { create(:project, :repository) }
-    let(:gitlab_shell) { Gitlab::Shell.new }
-
-    before do
-      # Project#gitlab_shell returns a new instance of Gitlab::Shell on every
-      # call. This makes testing a bit easier.
-      allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
-      allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
-    end
-
-    it 'renames a repository' do
-      stub_container_registry_config(enabled: false)
-
-      expect(gitlab_shell).to receive(:mv_repository)
-        .ordered
-        .with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}")
-        .and_return(true)
-
-      expect(gitlab_shell).to receive(:mv_repository)
-        .ordered
-        .with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
-        .and_return(true)
-
-      expect_any_instance_of(SystemHooksService)
-        .to receive(:execute_hooks_for)
-        .with(project, :rename)
-
-      expect_any_instance_of(Gitlab::UploadsTransfer)
-        .to receive(:rename_project)
-        .with('foo', project.path, project.namespace.full_path)
-
-      expect(project).to receive(:expire_caches_before_rename)
-
-      expect(project).to receive(:expires_full_path_cache)
-
-      project.rename_repo
-    end
-
-    context 'container registry with images' do
-      let(:container_repository) { create(:container_repository) }
-
-      before do
-        stub_container_registry_config(enabled: true)
-        stub_container_registry_tags(repository: :any, tags: ['tag'])
-        project.container_repositories << container_repository
-      end
-
-      subject { project.rename_repo }
-
-      it { expect {subject}.to raise_error(StandardError) }
-    end
-  end
-
   describe '#expire_caches_before_rename' do
     let(:project) { create(:project, :repository) }
     let(:repo)    { double(:repo, exists?: true) }
@@ -2367,4 +2313,163 @@ describe Project do
       expect(project.forks_count).to eq(1)
     end
   end
+
+  context 'legacy storage' do
+    let(:project) { create(:project, :repository) }
+    let(:gitlab_shell) { Gitlab::Shell.new }
+
+    describe '#base_dir' do
+      it 'returns base_dir based on namespace only' do
+        expect(project.base_dir).to eq(project.namespace.full_path)
+      end
+    end
+
+    describe '#disk_path' do
+      it 'returns disk_path based on namespace and project path' do
+        expect(project.disk_path).to eq("#{project.namespace.full_path}/#{project.path}")
+      end
+    end
+
+    describe '#ensure_storage_path_exist' do
+      before do
+        allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
+      end
+
+      it 'delegates to gitlab_shell to ensure namespace is created' do
+        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, project.base_dir)
+
+        project.ensure_storage_path_exist
+      end
+    end
+
+    describe '#rename_repo' do
+      before do
+        # Project#gitlab_shell returns a new instance of Gitlab::Shell on every
+        # call. This makes testing a bit easier.
+        allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
+        allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
+      end
+
+      it 'renames a repository' do
+        stub_container_registry_config(enabled: false)
+
+        expect(gitlab_shell).to receive(:mv_repository)
+          .ordered
+          .with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}")
+          .and_return(true)
+
+        expect(gitlab_shell).to receive(:mv_repository)
+          .ordered
+          .with(project.repository_storage_path, "#{project.namespace.full_path}/foo.wiki", "#{project.full_path}.wiki")
+          .and_return(true)
+
+        expect_any_instance_of(SystemHooksService)
+          .to receive(:execute_hooks_for)
+            .with(project, :rename)
+
+        expect_any_instance_of(Gitlab::UploadsTransfer)
+          .to receive(:rename_project)
+            .with('foo', project.path, project.namespace.full_path)
+
+        expect(project).to receive(:expire_caches_before_rename)
+
+        expect(project).to receive(:expires_full_path_cache)
+
+        project.rename_repo
+      end
+
+      context 'container registry with images' do
+        let(:container_repository) { create(:container_repository) }
+
+        before do
+          stub_container_registry_config(enabled: true)
+          stub_container_registry_tags(repository: :any, tags: ['tag'])
+          project.container_repositories << container_repository
+        end
+
+        subject { project.rename_repo }
+
+        it { expect{subject}.to raise_error(StandardError) }
+      end
+    end
+  end
+
+  context 'hashed storage' do
+    let(:project) { create(:project, :repository, :hashed) }
+    let(:gitlab_shell) { Gitlab::Shell.new }
+    let(:hash) { '6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b' }
+
+    before do
+      allow(Digest::SHA2).to receive(:hexdigest) { hash }
+    end
+
+    describe '#base_dir' do
+      it 'returns base_dir based on hash of project id' do
+        expect(project.base_dir).to eq('6b/86')
+      end
+    end
+
+    describe '#disk_path' do
+      it 'returns disk_path based on has of project id' do
+        hashed_path = '6b/86/6b86b273ff34fce19d6b804eff5a3f5747ada4eaa22f1d49c01e52ddb7875b4b'
+
+        expect(project.disk_path).to eq(hashed_path)
+      end
+    end
+
+    describe '#ensure_storage_path_exist' do
+      before do
+        allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
+      end
+
+      it 'delegates to gitlab_shell to ensure namespace is created' do
+        expect(gitlab_shell).to receive(:add_namespace).with(project.repository_storage_path, '6b/86')
+
+        project.ensure_storage_path_exist
+      end
+    end
+
+    describe '#rename_repo' do
+      before do
+        # Project#gitlab_shell returns a new instance of Gitlab::Shell on every
+        # call. This makes testing a bit easier.
+        allow(project).to receive(:gitlab_shell).and_return(gitlab_shell)
+        allow(project).to receive(:previous_changes).and_return('path' => ['foo'])
+      end
+
+      it 'renames a repository' do
+        stub_container_registry_config(enabled: false)
+
+        expect(gitlab_shell).not_to receive(:mv_repository)
+
+        expect_any_instance_of(SystemHooksService)
+          .to receive(:execute_hooks_for)
+            .with(project, :rename)
+
+        expect_any_instance_of(Gitlab::UploadsTransfer)
+          .to receive(:rename_project)
+            .with('foo', project.path, project.namespace.full_path)
+
+        expect(project).to receive(:expire_caches_before_rename)
+
+        expect(project).to receive(:expires_full_path_cache)
+
+        project.rename_repo
+      end
+
+      context 'container registry with images' do
+        let(:container_repository) { create(:container_repository) }
+
+        before do
+          stub_container_registry_config(enabled: true)
+          stub_container_registry_tags(repository: :any, tags: ['tag'])
+          project.container_repositories << container_repository
+        end
+
+        subject { project.rename_repo }
+
+        it { expect{subject}.to raise_error(StandardError) }
+      end
+    end
+  end
 end