Merge branch 'master' into backstage/gb/after-save-asynchronous-job-hooks

* master: (115 commits)
  Use event-based waiting in Gitlab::JobWaiter
  Make sure repository's removal work for legacy and hashed storages
  Use `@hashed` prefix for hashed paths on disk, to avoid collision with existing ones
  Refactor project and storage types
  Prevent using gitlab import task when hashed storage is enabled
  Some codestyle changes and fixes for GitLab pages
  Removed some useless code, codestyle changes and removed an index
  Fix repository reloading in some specs
  Changelog
  Moving away from the "extend" based factory to a more traditional one.
  Enable automatic hashed storage for new projects by application settings
  New storage is now "Hashed" instead of "UUID"
  Add UUID Storage to Project
  Move create_repository back to project model as we can use disk_path and share it
  Codestyle: move hooks to the same place and move dependent methods to private
  Use non-i18n values for setting new group-level issue/MR button text
  indexes external issue tracker
  copyedit
  indexes user/search/ from /user/index
  Correctly encode string params for Gitaly's TreeEntries RPC
  ...

12 files changed, 278 insertions, 109 deletions

diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb
index 085eeeae157..e7e02587759 100644
--- a/app/models/ci/pipeline_schedule.rb
+++ b/app/models/ci/pipeline_schedule.rb
@@ -9,7 +9,7 @@ module Ci
     belongs_to :owner, class_name: 'User'
     has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline'
     has_many :pipelines
-    has_many :variables, class_name: 'Ci::PipelineScheduleVariable'
+    has_many :variables, class_name: 'Ci::PipelineScheduleVariable', validate: false
 
     validates :cron, unless: :importing?, cron: true, presence: { unless: :importing? }
     validates :cron_timezone, cron_timezone: true, presence: { unless: :importing? }
diff --git a/app/models/ci/pipeline_schedule_variable.rb b/app/models/ci/pipeline_schedule_variable.rb
index 1ff177616e8..ee5b8733fac 100644
--- a/app/models/ci/pipeline_schedule_variable.rb
+++ b/app/models/ci/pipeline_schedule_variable.rb
@@ -4,5 +4,7 @@ module Ci
     include HasVariable
 
     belongs_to :pipeline_schedule
+
+    validates :key, uniqueness: { scope: :pipeline_schedule_id }
   end
 end
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index 59570924c8d..4ee972fa68d 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -1,11 +1,66 @@
 module Ci
   class Stage < ActiveRecord::Base
     extend Ci::Model
+    include Importable
+    include HasStatus
+    include Gitlab::OptimisticLocking
+
+    enum status: HasStatus::STATUSES_ENUM
 
     belongs_to :project
     belongs_to :pipeline
 
-    has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
-    has_many :builds, foreign_key: :commit_id
+    has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id
+    has_many :builds, foreign_key: :stage_id
+
+    validates :project, presence: true, unless: :importing?
+    validates :pipeline, presence: true, unless: :importing?
+    validates :name, presence: true, unless: :importing?
+
+    state_machine :status, initial: :created do
+      event :enqueue do
+        transition created: :pending
+        transition [:success, :failed, :canceled, :skipped] => :running
+      end
+
+      event :run do
+        transition any - [:running] => :running
+      end
+
+      event :skip do
+        transition any - [:skipped] => :skipped
+      end
+
+      event :drop do
+        transition any - [:failed] => :failed
+      end
+
+      event :succeed do
+        transition any - [:success] => :success
+      end
+
+      event :cancel do
+        transition any - [:canceled] => :canceled
+      end
+
+      event :block do
+        transition any - [:manual] => :manual
+      end
+    end
+
+    def update_status
+      retry_optimistic_lock(self) do
+        case statuses.latest.status
+        when 'pending' then enqueue
+        when 'running' then run
+        when 'success' then succeed
+        when 'failed' then drop
+        when 'canceled' then cancel
+        when 'manual' then block
+        when 'skipped' then skip
+        else skip
+        end
+      end
+    end
   end
 end
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 07cec63b939..842c6e5cb50 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -39,14 +39,14 @@ class CommitStatus < ActiveRecord::Base
   scope :after_stage, -> (index) { where('stage_idx > ?', index) }
 
   state_machine :status do
-    event :enqueue do
-      transition [:created, :skipped, :manual] => :pending
-    end
-
     event :process do
       transition [:skipped, :manual] => :created
     end
 
+    event :enqueue do
+      transition [:created, :skipped, :manual] => :pending
+    end
+
     event :run do
       transition pending: :running
     end
@@ -91,6 +91,7 @@ class CommitStatus < ActiveRecord::Base
           end
         end
 
+        StageUpdateWorker.perform_async(commit_status.stage_id)
         ExpireJobCacheWorker.perform_async(commit_status.id)
       end
     end
diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb
index 32af5566135..3803e18a96e 100644
--- a/app/models/concerns/has_status.rb
+++ b/app/models/concerns/has_status.rb
@@ -8,6 +8,8 @@ module HasStatus
   ACTIVE_STATUSES = %w[pending running].freeze
   COMPLETED_STATUSES = %w[success failed canceled skipped].freeze
   ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze
+  STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3,
+                    failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
 
   class_methods do
     def status_sql
diff --git a/app/models/concerns/storage/legacy_project.rb b/app/models/concerns/storage/legacy_project.rb
deleted file mode 100644
index 815db712285..00000000000
--- a/app/models/concerns/storage/legacy_project.rb
+++ /dev/null
@@ -1,76 +0,0 @@
-module Storage
-  module LegacyProject
-    extend ActiveSupport::Concern
-
-    def disk_path
-      full_path
-    end
-
-    def ensure_storage_path_exist
-      gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
-    end
-
-    def rename_repo
-      path_was = previous_changes['path'].first
-      old_path_with_namespace = File.join(namespace.full_path, path_was)
-      new_path_with_namespace = File.join(namespace.full_path, path)
-
-      Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
-
-      if has_container_registry_tags?
-        Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
-
-        # we currently doesn't support renaming repository if it contains images in container registry
-        raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
-      end
-
-      expire_caches_before_rename(old_path_with_namespace)
-
-      if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
-        # If repository moved successfully we need to send update instructions to users.
-        # However we cannot allow rollback since we moved repository
-        # So we basically we mute exceptions in next actions
-        begin
-          gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
-          send_move_instructions(old_path_with_namespace)
-          expires_full_path_cache
-
-          @old_path_with_namespace = old_path_with_namespace
-
-          SystemHooksService.new.execute_hooks_for(self, :rename)
-
-          @repository = nil
-        rescue => e
-          Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
-          # Returning false does not rollback after_* transaction but gives
-          # us information about failing some of tasks
-          false
-        end
-      else
-        Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
-
-        # if we cannot move namespace directory we should rollback
-        # db changes in order to prevent out of sync between db and fs
-        raise StandardError.new('repository cannot be renamed')
-      end
-
-      Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
-
-      Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
-      Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
-    end
-
-    def create_repository(force: false)
-      # Forked import is handled asynchronously
-      return if forked? && !force
-
-      if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
-        repository.after_create
-        true
-      else
-        errors.add(:base, 'Failed to create repository via gitlab-shell')
-        false
-      end
-    end
-  end
-end
diff --git a/app/models/event.rb b/app/models/event.rb
index f2a560a6b56..15ee170ca75 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -83,6 +83,10 @@ class Event < ActiveRecord::Base
   self.inheritance_column = 'action'
 
   class << self
+    def model_name
+      ActiveModel::Name.new(self, nil, 'event')
+    end
+
     def find_sti_class(action)
       if action.to_i == PUSHED
         PushEvent
@@ -438,6 +442,12 @@ class Event < ActiveRecord::Base
     EventForMigration.create!(new_attributes)
   end
 
+  def to_partial_path
+    # We are intentionally using `Event` rather than `self.class` so that
+    # subclasses also use the `Event` implementation.
+    Event._to_partial_path
+  end
+
   private
 
   def recent_update?
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 1c948c8957e..043da9967a1 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -9,11 +9,8 @@ class Issue < ActiveRecord::Base
   include Spammable
   include FasterCacheKeys
   include RelativePositioning
-  include IgnorableColumn
   include CreatedAtFilterable
 
-  ignore_column :position
-
   DueDateStruct = Struct.new(:title, :name).freeze
   NoDueDate     = DueDateStruct.new('No Due Date', '0').freeze
   AnyDueDate    = DueDateStruct.new('Any Due Date', '').freeze
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index ac08dc0ee1f..f028d2395c1 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -7,7 +7,6 @@ class MergeRequest < ActiveRecord::Base
   include IgnorableColumn
   include CreatedAtFilterable
 
-  ignore_column :position
   ignore_column :locked_at
 
   belongs_to :target_project, class_name: "Project"
diff --git a/app/models/project.rb b/app/models/project.rb
index be248bc99e1..37f4dd08355 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -17,7 +17,6 @@ class Project < ActiveRecord::Base
   include ProjectFeaturesCompatibility
   include SelectForProjectAuthorization
   include Routable
-  include Storage::LegacyProject
 
   extend Gitlab::ConfigHelper
 
@@ -25,6 +24,7 @@ class Project < ActiveRecord::Base
 
   NUMBER_OF_PERMITTED_BOARDS = 1
   UNKNOWN_IMPORT_URL = 'http://unknown.git'.freeze
+  LATEST_STORAGE_VERSION = 1
 
   cache_markdown_field :description, pipeline: :description
 
@@ -32,6 +32,8 @@ class Project < ActiveRecord::Base
            :merge_requests_enabled?, :issues_enabled?, to: :project_feature,
                                                        allow_nil: true
 
+  delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage
+
   default_value_for :archived, false
   default_value_for :visibility_level, gitlab_config_features.visibility_level
   default_value_for :container_registry_enabled, gitlab_config_features.container_registry
@@ -44,32 +46,24 @@ class Project < ActiveRecord::Base
   default_value_for :snippets_enabled, gitlab_config_features.snippets
   default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
 
-  after_create :ensure_storage_path_exist
-  after_create :create_project_feature, unless: :project_feature
-  after_save :update_project_statistics, if: :namespace_id_changed?
+  add_authentication_token_field :runners_token
+  before_save :ensure_runners_token
 
-  # set last_activity_at to the same as created_at
+  after_save :update_project_statistics, if: :namespace_id_changed?
+  after_create :create_project_feature, unless: :project_feature
   after_create :set_last_activity_at
-  def set_last_activity_at
-    update_column(:last_activity_at, self.created_at)
-  end
-
   after_create :set_last_repository_updated_at
-  def set_last_repository_updated_at
-    update_column(:last_repository_updated_at, self.created_at)
-  end
+  after_update :update_forks_visibility_level
 
   before_destroy :remove_private_deploy_keys
   after_destroy -> { run_after_commit { remove_pages } }
 
-  # update visibility_level of forks
-  after_update :update_forks_visibility_level
-
   after_validation :check_pending_delete
 
-  # Legacy Storage specific hooks
-
-  after_save :ensure_storage_path_exist, if: :namespace_id_changed?
+  # Storage specific hooks
+  after_initialize :use_hashed_storage
+  after_create :ensure_storage_path_exists
+  after_save :ensure_storage_path_exists, if: :namespace_id_changed?
 
   acts_as_taggable
 
@@ -238,9 +232,6 @@ class Project < ActiveRecord::Base
     presence: true,
     inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
 
-  add_authentication_token_field :runners_token
-  before_save :ensure_runners_token
-
   mount_uploader :avatar, AvatarUploader
   has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
@@ -487,6 +478,10 @@ class Project < ActiveRecord::Base
     @repository ||= Repository.new(full_path, self, disk_path: disk_path)
   end
 
+  def reload_repository!
+    @repository = nil
+  end
+
   def container_registry_url
     if Gitlab.config.registry.enabled
       "#{Gitlab.config.registry.host_port}/#{full_path.downcase}"
@@ -1004,6 +999,19 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def create_repository(force: false)
+    # Forked import is handled asynchronously
+    return if forked? && !force
+
+    if gitlab_shell.add_repository(repository_storage_path, disk_path)
+      repository.after_create
+      true
+    else
+      errors.add(:base, 'Failed to create repository via gitlab-shell')
+      false
+    end
+  end
+
   def hook_attrs(backward: true)
     attrs = {
       name: name,
@@ -1086,6 +1094,7 @@ class Project < ActiveRecord::Base
     !!repository.exists?
   end
 
+  # update visibility_level of forks
   def update_forks_visibility_level
     return unless visibility_level < visibility_level_was
 
@@ -1213,7 +1222,8 @@ class Project < ActiveRecord::Base
   end
 
   def pages_path
-    File.join(Settings.pages.path, disk_path)
+    # TODO: when we migrate Pages to work with new storage types, change here to use disk_path
+    File.join(Settings.pages.path, full_path)
   end
 
   def public_pages_path
@@ -1252,6 +1262,50 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def rename_repo
+    new_full_path = build_full_path
+
+    Rails.logger.error "Attempting to rename #{full_path_was} -> #{new_full_path}"
+
+    if has_container_registry_tags?
+      Rails.logger.error "Project #{full_path_was} cannot be renamed because container registry tags are present!"
+
+      # we currently doesn't support renaming repository if it contains images in container registry
+      raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
+    end
+
+    expire_caches_before_rename(full_path_was)
+
+    if storage.rename_repo
+      Gitlab::AppLogger.info "Project was renamed: #{full_path_was} -> #{new_full_path}"
+      rename_repo_notify!
+      after_rename_repo
+    else
+      Rails.logger.error "Repository could not be renamed: #{full_path_was} -> #{new_full_path}"
+
+      # if we cannot move namespace directory we should rollback
+      # db changes in order to prevent out of sync between db and fs
+      raise StandardError.new('repository cannot be renamed')
+    end
+  end
+
+  def rename_repo_notify!
+    send_move_instructions(full_path_was)
+    expires_full_path_cache
+
+    self.old_path_with_namespace = full_path_was
+    SystemHooksService.new.execute_hooks_for(self, :rename)
+
+    reload_repository!
+  end
+
+  def after_rename_repo
+    path_before_change = previous_changes['path'].first
+
+    Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+    Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+  end
+
   def running_or_pending_build_count(force: false)
     Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
       builds.running_or_pending.count(:all)
@@ -1410,6 +1464,10 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def full_path_was
+    File.join(namespace.full_path, previous_changes['path'].first)
+  end
+
   alias_method :name_with_namespace, :full_name
   alias_method :human_name, :full_name
   # @deprecated cannot remove yet because it has an index with its name in elasticsearch
@@ -1419,8 +1477,36 @@ class Project < ActiveRecord::Base
     Projects::ForksCountService.new(self).count
   end
 
+  def legacy_storage?
+    self.storage_version.nil?
+  end
+
   private
 
+  def storage
+    @storage ||=
+      if self.storage_version && self.storage_version >= 1
+        Storage::HashedProject.new(self)
+      else
+        Storage::LegacyProject.new(self)
+      end
+  end
+
+  def use_hashed_storage
+    if self.new_record? && current_application_settings.hashed_storage_enabled
+      self.storage_version = LATEST_STORAGE_VERSION
+    end
+  end
+
+  # set last_activity_at to the same as created_at
+  def set_last_activity_at
+    update_column(:last_activity_at, self.created_at)
+  end
+
+  def set_last_repository_updated_at
+    update_column(:last_repository_updated_at, self.created_at)
+  end
+
   def cross_namespace_reference?(from)
     case from
     when Project
diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb
new file mode 100644
index 00000000000..fae1b64961a
--- /dev/null
+++ b/app/models/storage/hashed_project.rb
@@ -0,0 +1,42 @@
+module Storage
+  class HashedProject
+    attr_accessor :project
+    delegate :gitlab_shell, :repository_storage_path, to: :project
+
+    ROOT_PATH_PREFIX = '@hashed'.freeze
+
+    def initialize(project)
+      @project = project
+    end
+
+    # Base directory
+    #
+    # @return [String] directory where repository is stored
+    def base_dir
+      "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}" if disk_hash
+    end
+
+    # Disk path is used to build repository and project's wiki path on disk
+    #
+    # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
+    def disk_path
+      "#{base_dir}/#{disk_hash}" if disk_hash
+    end
+
+    def ensure_storage_path_exists
+      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+    end
+
+    def rename_repo
+      true
+    end
+
+    private
+
+    # Generates the hash for the project path and name on disk
+    # If you need to refer to the repository on disk, use the `#disk_path`
+    def disk_hash
+      @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
+    end
+  end
+end
diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb
new file mode 100644
index 00000000000..9d9e5e1d352
--- /dev/null
+++ b/app/models/storage/legacy_project.rb
@@ -0,0 +1,51 @@
+module Storage
+  class LegacyProject
+    attr_accessor :project
+    delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project
+
+    def initialize(project)
+      @project = project
+    end
+
+    # Base directory
+    #
+    # @return [String] directory where repository is stored
+    def base_dir
+      namespace.full_path
+    end
+
+    # Disk path is used to build repository and project's wiki path on disk
+    #
+    # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
+    def disk_path
+      project.full_path
+    end
+
+    def ensure_storage_path_exists
+      return unless namespace
+
+      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+    end
+
+    def rename_repo
+      new_full_path = project.build_full_path
+
+      if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path)
+        # If repository moved successfully we need to send update instructions to users.
+        # However we cannot allow rollback since we moved repository
+        # So we basically we mute exceptions in next actions
+        begin
+          gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki")
+          return true
+        rescue => e
+          Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}"
+          # Returning false does not rollback after_* transaction but gives
+          # us information about failing some of tasks
+          return false
+        end
+      end
+
+      false
+    end
+  end
+end