Merge branch 'master' into backstage/gb/after-save-asynchronous-job-hooks

* master: (115 commits)
  Use event-based waiting in Gitlab::JobWaiter
  Make sure repository's removal work for legacy and hashed storages
  Use `@hashed` prefix for hashed paths on disk, to avoid collision with existing ones
  Refactor project and storage types
  Prevent using gitlab import task when hashed storage is enabled
  Some codestyle changes and fixes for GitLab pages
  Removed some useless code, codestyle changes and removed an index
  Fix repository reloading in some specs
  Changelog
  Moving away from the "extend" based factory to a more traditional one.
  Enable automatic hashed storage for new projects by application settings
  New storage is now "Hashed" instead of "UUID"
  Add UUID Storage to Project
  Move create_repository back to project model as we can use disk_path and share it
  Codestyle: move hooks to the same place and move dependent methods to private
  Use non-i18n values for setting new group-level issue/MR button text
  indexes external issue tracker
  copyedit
  indexes user/search/ from /user/index
  Correctly encode string params for Gitaly's TreeEntries RPC
  ...

37 files changed, 418 insertions, 155 deletions

diff --git a/app/assets/javascripts/project_select_combo_button.js b/app/assets/javascripts/project_select_combo_button.js
index f799d9d619a..46a26fb91f4 100644
--- a/app/assets/javascripts/project_select_combo_button.js
+++ b/app/assets/javascripts/project_select_combo_button.js
@@ -4,10 +4,10 @@ export default class ProjectSelectComboButton {
   constructor(select) {
     this.projectSelectInput = $(select);
     this.newItemBtn = $('.new-project-item-link');
-    this.newItemBtnBaseText = this.newItemBtn.data('label');
-    this.itemType = this.deriveItemTypeFromLabel();
+    this.resourceType = this.newItemBtn.data('type');
+    this.resourceLabel = this.newItemBtn.data('label');
+    this.formattedText = this.deriveTextVariants();
     this.groupId = this.projectSelectInput.data('groupId');
-
     this.bindEvents();
     this.initLocalStorage();
   }
@@ -23,9 +23,7 @@ export default class ProjectSelectComboButton {
     const localStorageIsSafe = AccessorUtilities.isLocalStorageAccessSafe();
 
     if (localStorageIsSafe) {
-      const itemTypeKebabed = this.newItemBtnBaseText.toLowerCase().split(' ').join('-');
-
-      this.localStorageKey = ['group', this.groupId, itemTypeKebabed, 'recent-project'].join('-');
+      this.localStorageKey = ['group', this.groupId, this.formattedText.localStorageItemType, 'recent-project'].join('-');
       this.setBtnTextFromLocalStorage();
     }
   }
@@ -57,19 +55,14 @@ export default class ProjectSelectComboButton {
   setNewItemBtnAttributes(project) {
     if (project) {
       this.newItemBtn.attr('href', project.url);
-      this.newItemBtn.text(`${this.newItemBtnBaseText} in ${project.name}`);
+      this.newItemBtn.text(`${this.formattedText.defaultTextPrefix} in ${project.name}`);
       this.newItemBtn.enable();
     } else {
-      this.newItemBtn.text(`Select project to create ${this.itemType}`);
+      this.newItemBtn.text(`Select project to create ${this.formattedText.presetTextSuffix}`);
       this.newItemBtn.disable();
     }
   }
 
-  deriveItemTypeFromLabel() {
-    // label is either 'New issue' or 'New merge request'
-    return this.newItemBtnBaseText.split(' ').slice(1).join(' ');
-  }
-
   getProjectFromLocalStorage() {
     const projectString = localStorage.getItem(this.localStorageKey);
 
@@ -81,5 +74,19 @@ export default class ProjectSelectComboButton {
 
     localStorage.setItem(this.localStorageKey, projectString);
   }
+
+  deriveTextVariants() {
+    const defaultTextPrefix = this.resourceLabel;
+
+    // the trailing slice call depluralizes each of these strings (e.g. new-issues -> new-issue)
+    const localStorageItemType = `new-${this.resourceType.split('_').join('-').slice(0, -1)}`;
+    const presetTextSuffix = this.resourceType.split('_').join(' ').slice(0, -1);
+
+    return {
+      localStorageItemType, // new-issue / new-merge-request
+      defaultTextPrefix, // New issue / New merge request
+      presetTextSuffix, // issue / merge request
+    };
+  }
 }
 
diff --git a/app/assets/stylesheets/framework/dropdowns.scss b/app/assets/stylesheets/framework/dropdowns.scss
index 5e768df972f..5f270e288ae 100644
--- a/app/assets/stylesheets/framework/dropdowns.scss
+++ b/app/assets/stylesheets/framework/dropdowns.scss
@@ -761,7 +761,7 @@
         &:hover,
         &:active,
         &:focus {
-          background-color: $gray-darker;
+          background-color: $dropdown-item-hover-bg;
           color: $gl-text-color;
         }
 
diff --git a/app/assets/stylesheets/framework/selects.scss b/app/assets/stylesheets/framework/selects.scss
index 40e654f4838..f7a0b355bf1 100644
--- a/app/assets/stylesheets/framework/selects.scss
+++ b/app/assets/stylesheets/framework/selects.scss
@@ -264,3 +264,41 @@
 .ajax-users-dropdown {
   min-width: 250px !important;
 }
+
+// TODO: change global style
+.ajax-project-dropdown {
+  &.select2-drop {
+    color: $gl-text-color;
+  }
+
+  .select2-results {
+    .select2-no-results,
+    .select2-searching,
+    .select2-ajax-error,
+    .select2-selection-limit {
+      background: transparent;
+    }
+
+    .select2-result {
+      padding: 0 1px;
+
+      .select2-match {
+        font-weight: bold;
+        text-decoration: none;
+      }
+
+      .select2-result-label {
+        padding: #{$gl-padding / 2} $gl-padding;
+      }
+
+      &.select2-highlighted {
+        background-color: transparent !important;
+        color: $gl-text-color;
+
+        .select2-result-label {
+          background-color: $dropdown-item-hover-bg;
+        }
+      }
+    }
+  }
+}
diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss
index 3c109a5a929..225d116e9c7 100644
--- a/app/assets/stylesheets/framework/variables.scss
+++ b/app/assets/stylesheets/framework/variables.scss
@@ -294,7 +294,7 @@ $dropdown-input-focus-shadow: rgba($dropdown-input-focus-border, .4);
 $dropdown-loading-bg: rgba(#fff, .6);
 $dropdown-chevron-size: 10px;
 $dropdown-toggle-active-border-color: darken($border-color, 14%);
-
+$dropdown-item-hover-bg: $gray-darker;
 
 /*
 * Filtered Search
diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb
index f76b3f69e9e..994e736d66e 100644
--- a/app/controllers/groups_controller.rb
+++ b/app/controllers/groups_controller.rb
@@ -26,6 +26,13 @@ class GroupsController < Groups::ApplicationController
 
   def new
     @group = Group.new
+
+    if params[:parent_id].present?
+      parent = Group.find_by(id: params[:parent_id])
+      if can?(current_user, :create_subgroup, parent)
+        @group.parent = parent
+      end
+    end
   end
 
   def create
diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb
index 150188f0b65..3b76da238e0 100644
--- a/app/helpers/application_settings_helper.rb
+++ b/app/helpers/application_settings_helper.rb
@@ -116,6 +116,7 @@ module ApplicationSettingsHelper
       :email_author_in_body,
       :enabled_git_access_protocol,
       :gravatar_enabled,
+      :hashed_storage_enabled,
       :help_page_hide_commercial_content,
       :help_page_support_url,
       :help_page_text,
diff --git a/app/helpers/events_helper.rb b/app/helpers/events_helper.rb
index 722a65eeb98..c6f98e7e782 100644
--- a/app/helpers/events_helper.rb
+++ b/app/helpers/events_helper.rb
@@ -176,7 +176,7 @@ module EventsHelper
     sanitize(
       text,
       tags: %w(a img gl-emoji b pre code p span),
-      attributes: Rails::Html::WhiteListSanitizer.allowed_attributes + ['style', 'data-name', 'data-unicode-version']
+      attributes: Rails::Html::WhiteListSanitizer.allowed_attributes + ['style', 'data-src', 'data-name', 'data-unicode-version']
     )
   end
 
diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb
index 085eeeae157..e7e02587759 100644
--- a/app/models/ci/pipeline_schedule.rb
+++ b/app/models/ci/pipeline_schedule.rb
@@ -9,7 +9,7 @@ module Ci
     belongs_to :owner, class_name: 'User'
     has_one :last_pipeline, -> { order(id: :desc) }, class_name: 'Ci::Pipeline'
     has_many :pipelines
-    has_many :variables, class_name: 'Ci::PipelineScheduleVariable'
+    has_many :variables, class_name: 'Ci::PipelineScheduleVariable', validate: false
 
     validates :cron, unless: :importing?, cron: true, presence: { unless: :importing? }
     validates :cron_timezone, cron_timezone: true, presence: { unless: :importing? }
diff --git a/app/models/ci/pipeline_schedule_variable.rb b/app/models/ci/pipeline_schedule_variable.rb
index 1ff177616e8..ee5b8733fac 100644
--- a/app/models/ci/pipeline_schedule_variable.rb
+++ b/app/models/ci/pipeline_schedule_variable.rb
@@ -4,5 +4,7 @@ module Ci
     include HasVariable
 
     belongs_to :pipeline_schedule
+
+    validates :key, uniqueness: { scope: :pipeline_schedule_id }
   end
 end
diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb
index 59570924c8d..4ee972fa68d 100644
--- a/app/models/ci/stage.rb
+++ b/app/models/ci/stage.rb
@@ -1,11 +1,66 @@
 module Ci
   class Stage < ActiveRecord::Base
     extend Ci::Model
+    include Importable
+    include HasStatus
+    include Gitlab::OptimisticLocking
+
+    enum status: HasStatus::STATUSES_ENUM
 
     belongs_to :project
     belongs_to :pipeline
 
-    has_many :statuses, class_name: 'CommitStatus', foreign_key: :commit_id
-    has_many :builds, foreign_key: :commit_id
+    has_many :statuses, class_name: 'CommitStatus', foreign_key: :stage_id
+    has_many :builds, foreign_key: :stage_id
+
+    validates :project, presence: true, unless: :importing?
+    validates :pipeline, presence: true, unless: :importing?
+    validates :name, presence: true, unless: :importing?
+
+    state_machine :status, initial: :created do
+      event :enqueue do
+        transition created: :pending
+        transition [:success, :failed, :canceled, :skipped] => :running
+      end
+
+      event :run do
+        transition any - [:running] => :running
+      end
+
+      event :skip do
+        transition any - [:skipped] => :skipped
+      end
+
+      event :drop do
+        transition any - [:failed] => :failed
+      end
+
+      event :succeed do
+        transition any - [:success] => :success
+      end
+
+      event :cancel do
+        transition any - [:canceled] => :canceled
+      end
+
+      event :block do
+        transition any - [:manual] => :manual
+      end
+    end
+
+    def update_status
+      retry_optimistic_lock(self) do
+        case statuses.latest.status
+        when 'pending' then enqueue
+        when 'running' then run
+        when 'success' then succeed
+        when 'failed' then drop
+        when 'canceled' then cancel
+        when 'manual' then block
+        when 'skipped' then skip
+        else skip
+        end
+      end
+    end
   end
 end
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 07cec63b939..842c6e5cb50 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -39,14 +39,14 @@ class CommitStatus < ActiveRecord::Base
   scope :after_stage, -> (index) { where('stage_idx > ?', index) }
 
   state_machine :status do
-    event :enqueue do
-      transition [:created, :skipped, :manual] => :pending
-    end
-
     event :process do
       transition [:skipped, :manual] => :created
     end
 
+    event :enqueue do
+      transition [:created, :skipped, :manual] => :pending
+    end
+
     event :run do
       transition pending: :running
     end
@@ -91,6 +91,7 @@ class CommitStatus < ActiveRecord::Base
           end
         end
 
+        StageUpdateWorker.perform_async(commit_status.stage_id)
         ExpireJobCacheWorker.perform_async(commit_status.id)
       end
     end
diff --git a/app/models/concerns/has_status.rb b/app/models/concerns/has_status.rb
index 32af5566135..3803e18a96e 100644
--- a/app/models/concerns/has_status.rb
+++ b/app/models/concerns/has_status.rb
@@ -8,6 +8,8 @@ module HasStatus
   ACTIVE_STATUSES = %w[pending running].freeze
   COMPLETED_STATUSES = %w[success failed canceled skipped].freeze
   ORDERED_STATUSES = %w[failed pending running manual canceled success skipped created].freeze
+  STATUSES_ENUM = { created: 0, pending: 1, running: 2, success: 3,
+                    failed: 4, canceled: 5, skipped: 6, manual: 7 }.freeze
 
   class_methods do
     def status_sql
diff --git a/app/models/concerns/storage/legacy_project.rb b/app/models/concerns/storage/legacy_project.rb
deleted file mode 100644
index 815db712285..00000000000
--- a/app/models/concerns/storage/legacy_project.rb
+++ /dev/null
@@ -1,76 +0,0 @@
-module Storage
-  module LegacyProject
-    extend ActiveSupport::Concern
-
-    def disk_path
-      full_path
-    end
-
-    def ensure_storage_path_exist
-      gitlab_shell.add_namespace(repository_storage_path, namespace.full_path)
-    end
-
-    def rename_repo
-      path_was = previous_changes['path'].first
-      old_path_with_namespace = File.join(namespace.full_path, path_was)
-      new_path_with_namespace = File.join(namespace.full_path, path)
-
-      Rails.logger.error "Attempting to rename #{old_path_with_namespace} -> #{new_path_with_namespace}"
-
-      if has_container_registry_tags?
-        Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!"
-
-        # we currently doesn't support renaming repository if it contains images in container registry
-        raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
-      end
-
-      expire_caches_before_rename(old_path_with_namespace)
-
-      if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace)
-        # If repository moved successfully we need to send update instructions to users.
-        # However we cannot allow rollback since we moved repository
-        # So we basically we mute exceptions in next actions
-        begin
-          gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki")
-          send_move_instructions(old_path_with_namespace)
-          expires_full_path_cache
-
-          @old_path_with_namespace = old_path_with_namespace
-
-          SystemHooksService.new.execute_hooks_for(self, :rename)
-
-          @repository = nil
-        rescue => e
-          Rails.logger.error "Exception renaming #{old_path_with_namespace} -> #{new_path_with_namespace}: #{e}"
-          # Returning false does not rollback after_* transaction but gives
-          # us information about failing some of tasks
-          false
-        end
-      else
-        Rails.logger.error "Repository could not be renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
-
-        # if we cannot move namespace directory we should rollback
-        # db changes in order to prevent out of sync between db and fs
-        raise StandardError.new('repository cannot be renamed')
-      end
-
-      Gitlab::AppLogger.info "Project was renamed: #{old_path_with_namespace} -> #{new_path_with_namespace}"
-
-      Gitlab::UploadsTransfer.new.rename_project(path_was, path, namespace.full_path)
-      Gitlab::PagesTransfer.new.rename_project(path_was, path, namespace.full_path)
-    end
-
-    def create_repository(force: false)
-      # Forked import is handled asynchronously
-      return if forked? && !force
-
-      if gitlab_shell.add_repository(repository_storage_path, path_with_namespace)
-        repository.after_create
-        true
-      else
-        errors.add(:base, 'Failed to create repository via gitlab-shell')
-        false
-      end
-    end
-  end
-end
diff --git a/app/models/event.rb b/app/models/event.rb
index f2a560a6b56..15ee170ca75 100644
--- a/app/models/event.rb
+++ b/app/models/event.rb
@@ -83,6 +83,10 @@ class Event < ActiveRecord::Base
   self.inheritance_column = 'action'
 
   class << self
+    def model_name
+      ActiveModel::Name.new(self, nil, 'event')
+    end
+
     def find_sti_class(action)
       if action.to_i == PUSHED
         PushEvent
@@ -438,6 +442,12 @@ class Event < ActiveRecord::Base
     EventForMigration.create!(new_attributes)
   end
 
+  def to_partial_path
+    # We are intentionally using `Event` rather than `self.class` so that
+    # subclasses also use the `Event` implementation.
+    Event._to_partial_path
+  end
+
   private
 
   def recent_update?
diff --git a/app/models/issue.rb b/app/models/issue.rb
index 1c948c8957e..043da9967a1 100644
--- a/app/models/issue.rb
+++ b/app/models/issue.rb
@@ -9,11 +9,8 @@ class Issue < ActiveRecord::Base
   include Spammable
   include FasterCacheKeys
   include RelativePositioning
-  include IgnorableColumn
   include CreatedAtFilterable
 
-  ignore_column :position
-
   DueDateStruct = Struct.new(:title, :name).freeze
   NoDueDate     = DueDateStruct.new('No Due Date', '0').freeze
   AnyDueDate    = DueDateStruct.new('Any Due Date', '').freeze
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index ac08dc0ee1f..f028d2395c1 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -7,7 +7,6 @@ class MergeRequest < ActiveRecord::Base
   include IgnorableColumn
   include CreatedAtFilterable
 
-  ignore_column :position
   ignore_column :locked_at
 
   belongs_to :target_project, class_name: "Project"
diff --git a/app/models/project.rb b/app/models/project.rb
index be248bc99e1..37f4dd08355 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -17,7 +17,6 @@ class Project < ActiveRecord::Base
   include ProjectFeaturesCompatibility
   include SelectForProjectAuthorization
   include Routable
-  include Storage::LegacyProject
 
   extend Gitlab::ConfigHelper
 
@@ -25,6 +24,7 @@ class Project < ActiveRecord::Base
 
   NUMBER_OF_PERMITTED_BOARDS = 1
   UNKNOWN_IMPORT_URL = 'http://unknown.git'.freeze
+  LATEST_STORAGE_VERSION = 1
 
   cache_markdown_field :description, pipeline: :description
 
@@ -32,6 +32,8 @@ class Project < ActiveRecord::Base
            :merge_requests_enabled?, :issues_enabled?, to: :project_feature,
                                                        allow_nil: true
 
+  delegate :base_dir, :disk_path, :ensure_storage_path_exists, to: :storage
+
   default_value_for :archived, false
   default_value_for :visibility_level, gitlab_config_features.visibility_level
   default_value_for :container_registry_enabled, gitlab_config_features.container_registry
@@ -44,32 +46,24 @@ class Project < ActiveRecord::Base
   default_value_for :snippets_enabled, gitlab_config_features.snippets
   default_value_for :only_allow_merge_if_all_discussions_are_resolved, false
 
-  after_create :ensure_storage_path_exist
-  after_create :create_project_feature, unless: :project_feature
-  after_save :update_project_statistics, if: :namespace_id_changed?
+  add_authentication_token_field :runners_token
+  before_save :ensure_runners_token
 
-  # set last_activity_at to the same as created_at
+  after_save :update_project_statistics, if: :namespace_id_changed?
+  after_create :create_project_feature, unless: :project_feature
   after_create :set_last_activity_at
-  def set_last_activity_at
-    update_column(:last_activity_at, self.created_at)
-  end
-
   after_create :set_last_repository_updated_at
-  def set_last_repository_updated_at
-    update_column(:last_repository_updated_at, self.created_at)
-  end
+  after_update :update_forks_visibility_level
 
   before_destroy :remove_private_deploy_keys
   after_destroy -> { run_after_commit { remove_pages } }
 
-  # update visibility_level of forks
-  after_update :update_forks_visibility_level
-
   after_validation :check_pending_delete
 
-  # Legacy Storage specific hooks
-
-  after_save :ensure_storage_path_exist, if: :namespace_id_changed?
+  # Storage specific hooks
+  after_initialize :use_hashed_storage
+  after_create :ensure_storage_path_exists
+  after_save :ensure_storage_path_exists, if: :namespace_id_changed?
 
   acts_as_taggable
 
@@ -238,9 +232,6 @@ class Project < ActiveRecord::Base
     presence: true,
     inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } }
 
-  add_authentication_token_field :runners_token
-  before_save :ensure_runners_token
-
   mount_uploader :avatar, AvatarUploader
   has_many :uploads, as: :model, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent
 
@@ -487,6 +478,10 @@ class Project < ActiveRecord::Base
     @repository ||= Repository.new(full_path, self, disk_path: disk_path)
   end
 
+  def reload_repository!
+    @repository = nil
+  end
+
   def container_registry_url
     if Gitlab.config.registry.enabled
       "#{Gitlab.config.registry.host_port}/#{full_path.downcase}"
@@ -1004,6 +999,19 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def create_repository(force: false)
+    # Forked import is handled asynchronously
+    return if forked? && !force
+
+    if gitlab_shell.add_repository(repository_storage_path, disk_path)
+      repository.after_create
+      true
+    else
+      errors.add(:base, 'Failed to create repository via gitlab-shell')
+      false
+    end
+  end
+
   def hook_attrs(backward: true)
     attrs = {
       name: name,
@@ -1086,6 +1094,7 @@ class Project < ActiveRecord::Base
     !!repository.exists?
   end
 
+  # update visibility_level of forks
   def update_forks_visibility_level
     return unless visibility_level < visibility_level_was
 
@@ -1213,7 +1222,8 @@ class Project < ActiveRecord::Base
   end
 
   def pages_path
-    File.join(Settings.pages.path, disk_path)
+    # TODO: when we migrate Pages to work with new storage types, change here to use disk_path
+    File.join(Settings.pages.path, full_path)
   end
 
   def public_pages_path
@@ -1252,6 +1262,50 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def rename_repo
+    new_full_path = build_full_path
+
+    Rails.logger.error "Attempting to rename #{full_path_was} -> #{new_full_path}"
+
+    if has_container_registry_tags?
+      Rails.logger.error "Project #{full_path_was} cannot be renamed because container registry tags are present!"
+
+      # we currently doesn't support renaming repository if it contains images in container registry
+      raise StandardError.new('Project cannot be renamed, because images are present in its container registry')
+    end
+
+    expire_caches_before_rename(full_path_was)
+
+    if storage.rename_repo
+      Gitlab::AppLogger.info "Project was renamed: #{full_path_was} -> #{new_full_path}"
+      rename_repo_notify!
+      after_rename_repo
+    else
+      Rails.logger.error "Repository could not be renamed: #{full_path_was} -> #{new_full_path}"
+
+      # if we cannot move namespace directory we should rollback
+      # db changes in order to prevent out of sync between db and fs
+      raise StandardError.new('repository cannot be renamed')
+    end
+  end
+
+  def rename_repo_notify!
+    send_move_instructions(full_path_was)
+    expires_full_path_cache
+
+    self.old_path_with_namespace = full_path_was
+    SystemHooksService.new.execute_hooks_for(self, :rename)
+
+    reload_repository!
+  end
+
+  def after_rename_repo
+    path_before_change = previous_changes['path'].first
+
+    Gitlab::UploadsTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+    Gitlab::PagesTransfer.new.rename_project(path_before_change, self.path, namespace.full_path)
+  end
+
   def running_or_pending_build_count(force: false)
     Rails.cache.fetch(['projects', id, 'running_or_pending_build_count'], force: force) do
       builds.running_or_pending.count(:all)
@@ -1410,6 +1464,10 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def full_path_was
+    File.join(namespace.full_path, previous_changes['path'].first)
+  end
+
   alias_method :name_with_namespace, :full_name
   alias_method :human_name, :full_name
   # @deprecated cannot remove yet because it has an index with its name in elasticsearch
@@ -1419,8 +1477,36 @@ class Project < ActiveRecord::Base
     Projects::ForksCountService.new(self).count
   end
 
+  def legacy_storage?
+    self.storage_version.nil?
+  end
+
   private
 
+  def storage
+    @storage ||=
+      if self.storage_version && self.storage_version >= 1
+        Storage::HashedProject.new(self)
+      else
+        Storage::LegacyProject.new(self)
+      end
+  end
+
+  def use_hashed_storage
+    if self.new_record? && current_application_settings.hashed_storage_enabled
+      self.storage_version = LATEST_STORAGE_VERSION
+    end
+  end
+
+  # set last_activity_at to the same as created_at
+  def set_last_activity_at
+    update_column(:last_activity_at, self.created_at)
+  end
+
+  def set_last_repository_updated_at
+    update_column(:last_repository_updated_at, self.created_at)
+  end
+
   def cross_namespace_reference?(from)
     case from
     when Project
diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb
new file mode 100644
index 00000000000..fae1b64961a
--- /dev/null
+++ b/app/models/storage/hashed_project.rb
@@ -0,0 +1,42 @@
+module Storage
+  class HashedProject
+    attr_accessor :project
+    delegate :gitlab_shell, :repository_storage_path, to: :project
+
+    ROOT_PATH_PREFIX = '@hashed'.freeze
+
+    def initialize(project)
+      @project = project
+    end
+
+    # Base directory
+    #
+    # @return [String] directory where repository is stored
+    def base_dir
+      "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}" if disk_hash
+    end
+
+    # Disk path is used to build repository and project's wiki path on disk
+    #
+    # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
+    def disk_path
+      "#{base_dir}/#{disk_hash}" if disk_hash
+    end
+
+    def ensure_storage_path_exists
+      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+    end
+
+    def rename_repo
+      true
+    end
+
+    private
+
+    # Generates the hash for the project path and name on disk
+    # If you need to refer to the repository on disk, use the `#disk_path`
+    def disk_hash
+      @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id
+    end
+  end
+end
diff --git a/app/models/storage/legacy_project.rb b/app/models/storage/legacy_project.rb
new file mode 100644
index 00000000000..9d9e5e1d352
--- /dev/null
+++ b/app/models/storage/legacy_project.rb
@@ -0,0 +1,51 @@
+module Storage
+  class LegacyProject
+    attr_accessor :project
+    delegate :namespace, :gitlab_shell, :repository_storage_path, to: :project
+
+    def initialize(project)
+      @project = project
+    end
+
+    # Base directory
+    #
+    # @return [String] directory where repository is stored
+    def base_dir
+      namespace.full_path
+    end
+
+    # Disk path is used to build repository and project's wiki path on disk
+    #
+    # @return [String] combination of base_dir and the repository own name without `.git` or `.wiki.git` extensions
+    def disk_path
+      project.full_path
+    end
+
+    def ensure_storage_path_exists
+      return unless namespace
+
+      gitlab_shell.add_namespace(repository_storage_path, base_dir)
+    end
+
+    def rename_repo
+      new_full_path = project.build_full_path
+
+      if gitlab_shell.mv_repository(repository_storage_path, project.full_path_was, new_full_path)
+        # If repository moved successfully we need to send update instructions to users.
+        # However we cannot allow rollback since we moved repository
+        # So we basically we mute exceptions in next actions
+        begin
+          gitlab_shell.mv_repository(repository_storage_path, "#{project.full_path_was}.wiki", "#{new_full_path}.wiki")
+          return true
+        rescue => e
+          Rails.logger.error "Exception renaming #{project.full_path_was} -> #{new_full_path}: #{e}"
+          # Returning false does not rollback after_* transaction but gives
+          # us information about failing some of tasks
+          return false
+        end
+      end
+
+      false
+    end
+  end
+end
diff --git a/app/policies/group_policy.rb b/app/policies/group_policy.rb
index 6defab75fce..8ada661e571 100644
--- a/app/policies/group_policy.rb
+++ b/app/policies/group_policy.rb
@@ -13,6 +13,8 @@ class GroupPolicy < BasePolicy
   condition(:master) { access_level >= GroupMember::MASTER }
   condition(:reporter) { access_level >= GroupMember::REPORTER }
 
+  condition(:nested_groups_supported, scope: :global) { Group.supports_nested_groups? }
+
   condition(:has_projects) do
     GroupProjectsFinder.new(group: @subject, current_user: @user).execute.any?
   end
@@ -42,7 +44,7 @@ class GroupPolicy < BasePolicy
     enable :change_visibility_level
   end
 
-  rule { owner & can_create_group }.enable :create_subgroup
+  rule { owner & can_create_group & nested_groups_supported }.enable :create_subgroup
 
   rule { public_group | logged_in_viewable }.enable :view_globally
 
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb
index 884b681ff81..d0ba9f89460 100644
--- a/app/services/ci/create_pipeline_service.rb
+++ b/app/services/ci/create_pipeline_service.rb
@@ -176,9 +176,14 @@ module Ci
     end
 
     def error(message, save: false)
-      pipeline.errors.add(:base, message)
-      pipeline.drop if save
-      pipeline
+      pipeline.tap do
+        pipeline.errors.add(:base, message)
+
+        if save
+          pipeline.drop
+          update_merge_requests_head_pipeline
+        end
+      end
     end
 
     def pipeline_created_counter
diff --git a/app/services/groups/create_service.rb b/app/services/groups/create_service.rb
index c4e9b8fd8e0..c7c27621085 100644
--- a/app/services/groups/create_service.rb
+++ b/app/services/groups/create_service.rb
@@ -13,9 +13,9 @@ module Groups
         return @group
       end
 
-      if @group.parent && !can?(current_user, :admin_group, @group.parent)
+      if @group.parent && !can?(current_user, :create_subgroup, @group.parent)
         @group.parent = nil
-        @group.errors.add(:parent_id, 'manage access required to create subgroup')
+        @group.errors.add(:parent_id, 'You don’t have permission to create a subgroup in this group.')
 
         return @group
       end
diff --git a/app/services/groups/destroy_service.rb b/app/services/groups/destroy_service.rb
index f565612a89d..e3f9d9ee95d 100644
--- a/app/services/groups/destroy_service.rb
+++ b/app/services/groups/destroy_service.rb
@@ -13,7 +13,7 @@ module Groups
         # Execute the destruction of the models immediately to ensure atomic cleanup.
         # Skip repository removal because we remove directory with namespace
         # that contain all these repositories
-        ::Projects::DestroyService.new(project, current_user, skip_repo: true).execute
+        ::Projects::DestroyService.new(project, current_user, skip_repo: project.legacy_storage?).execute
       end
 
       group.children.each do |group|
diff --git a/app/services/merge_requests/create_from_issue_service.rb b/app/services/merge_requests/create_from_issue_service.rb
index 738cedbaed7..da39a380451 100644
--- a/app/services/merge_requests/create_from_issue_service.rb
+++ b/app/services/merge_requests/create_from_issue_service.rb
@@ -3,6 +3,8 @@ module MergeRequests
     def execute
       return error('Invalid issue iid') unless issue_iid.present? && issue.present?
 
+      params[:label_ids] = issue.label_ids if issue.label_ids.any?
+
       result = CreateBranchService.new(project, current_user).execute(branch_name, ref)
       return result if result[:status] == :error
 
@@ -43,7 +45,8 @@ module MergeRequests
       {
         source_project_id: project.id,
         source_branch: branch_name,
-        target_project_id: project.id
+        target_project_id: project.id,
+        milestone_id: issue.milestone_id
       }
     end
 
diff --git a/app/services/users/destroy_service.rb b/app/services/users/destroy_service.rb
index 9d7237c2fbb..8e20de8dfa5 100644
--- a/app/services/users/destroy_service.rb
+++ b/app/services/users/destroy_service.rb
@@ -35,16 +35,18 @@ module Users
         Groups::DestroyService.new(group, current_user).execute
       end
 
+      namespace = user.namespace
+      namespace.prepare_for_destroy
+
       user.personal_projects.each do |project|
         # Skip repository removal because we remove directory with namespace
         # that contain all this repositories
-        ::Projects::DestroyService.new(project, current_user, skip_repo: true).execute
+        ::Projects::DestroyService.new(project, current_user, skip_repo: project.legacy_storage?).execute
       end
 
       MigrateToGhostUserService.new(user).execute unless options[:hard_delete]
 
       # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing
-      namespace = user.namespace
       user_data = user.destroy
       namespace.really_destroy!
 
diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml
index 8bf6556079b..959af5c0d13 100644
--- a/app/views/admin/application_settings/_form.html.haml
+++ b/app/views/admin/application_settings/_form.html.haml
@@ -362,7 +362,9 @@
   %fieldset
     %legend Background Jobs
     %p
-      These settings require a restart to take effect.
+      These settings require a
+      = link_to 'restart', help_page_path('administration/restart_gitlab')
+      to take effect.
     .form-group
       .col-sm-offset-2.col-sm-10
         .checkbox
@@ -491,6 +493,16 @@
   %fieldset
     %legend Repository Storage
     .form-group
+      .col-sm-offset-2.col-sm-10
+        .checkbox
+          = f.label :hashed_storage_enabled do
+            = f.check_box :hashed_storage_enabled
+            Create new projects using hashed storage paths
+          .help-block
+            Enable immutable, hash-based paths and repository names to store repositories on disk. This prevents
+            repositories from having to be moved or renamed when the Project URL changes and may improve disk I/O performance.
+            %em (EXPERIMENTAL)
+    .form-group
       = f.label :repository_storages, 'Storage paths for new projects', class: 'control-label col-sm-2'
       .col-sm-10
         = f.select :repository_storages, repository_storages_options_for_select, {include_hidden: false}, multiple: true, class: 'form-control'
@@ -499,6 +511,7 @@
           = succeed "." do
             = link_to "repository storages documentation", help_page_path("administration/repository_storages")
 
+
   %fieldset
     %legend Repository Checks
     .form-group
diff --git a/app/views/dashboard/issues.html.haml b/app/views/dashboard/issues.html.haml
index 52e0012fd7d..9ac44674b73 100644
--- a/app/views/dashboard/issues.html.haml
+++ b/app/views/dashboard/issues.html.haml
@@ -8,14 +8,14 @@
   - content_for :breadcrumbs_extra do
     = link_to params.merge(rss_url_options), class: 'btn has-tooltip append-right-10', title: 'Subscribe' do
       = icon('rss')
-    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues'
+    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues', type: :issues
 
 .top-area
   = render 'shared/issuable/nav', type: :issues
   .nav-controls{ class: ("visible-xs" if show_new_nav?) }
     = link_to params.merge(rss_url_options), class: 'btn has-tooltip', title: 'Subscribe' do
       = icon('rss')
-    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues'
+    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", with_feature_enabled: 'issues', type: :issues
 
 = render 'shared/issuable/filter', type: :issues
 = render 'shared/issues'
diff --git a/app/views/dashboard/merge_requests.html.haml b/app/views/dashboard/merge_requests.html.haml
index c3fe14da2b2..960e1e55f36 100644
--- a/app/views/dashboard/merge_requests.html.haml
+++ b/app/views/dashboard/merge_requests.html.haml
@@ -4,12 +4,12 @@
 
 - if show_new_nav?
   - content_for :breadcrumbs_extra do
-    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests'
+    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests', type: :merge_requests
 
 .top-area
   = render 'shared/issuable/nav', type: :merge_requests
   .nav-controls{ class: ("visible-xs" if show_new_nav?) }
-    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests'
+    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request", with_feature_enabled: 'merge_requests', type: :merge_requests
 
 = render 'shared/issuable/filter', type: :merge_requests
 = render 'shared/merge_requests'
diff --git a/app/views/dashboard/milestones/index.html.haml b/app/views/dashboard/milestones/index.html.haml
index 37dbcaf5cb8..cb8bf57cba1 100644
--- a/app/views/dashboard/milestones/index.html.haml
+++ b/app/views/dashboard/milestones/index.html.haml
@@ -4,13 +4,13 @@
 
 - if show_new_nav?
   - content_for :breadcrumbs_extra do
-    = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true
+    = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true, type: :milestones
 
 .top-area
   = render 'shared/milestones_filter', counts: @milestone_states
 
   .nav-controls{ class: ("visible-xs" if show_new_nav?) }
-    = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true
+    = render 'shared/new_project_item_select', path: 'milestones/new', label: 'New milestone', include_groups: true, type: :milestones
 
 .milestones
   %ul.content-list
diff --git a/app/views/groups/issues.html.haml b/app/views/groups/issues.html.haml
index f83ebbf09ef..12bc092d216 100644
--- a/app/views/groups/issues.html.haml
+++ b/app/views/groups/issues.html.haml
@@ -12,7 +12,7 @@
   - content_for :breadcrumbs_extra do
     = link_to params.merge(rss_url_options), class: 'btn btn-default append-right-10' do
       = icon('rss')
-    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue"
+    = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", type: :issues
 
 - if group_issues_exists
   .top-area
@@ -22,7 +22,7 @@
         = icon('rss')
         %span.icon-label
           Subscribe
-      = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue"
+      = render 'shared/new_project_item_select', path: 'issues/new', label: "New issue", type: :issues
 
   = render 'shared/issuable/search_bar', type: :issues
 
diff --git a/app/views/groups/merge_requests.html.haml b/app/views/groups/merge_requests.html.haml
index 997c82c77d9..569eef46e6e 100644
--- a/app/views/groups/merge_requests.html.haml
+++ b/app/views/groups/merge_requests.html.haml
@@ -2,7 +2,7 @@
 
 - if show_new_nav? && current_user
   - content_for :breadcrumbs_extra do
-    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request"
+    = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request",  type: :merge_requests
 
 - if @group_merge_requests.empty?
   = render 'shared/empty_states/merge_requests', project_select_button: true
@@ -11,7 +11,7 @@
     = render 'shared/issuable/nav', type: :merge_requests
     - if current_user
       .nav-controls{ class: ("visible-xs" if show_new_nav?) }
-        = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request"
+        = render 'shared/new_project_item_select', path: 'merge_requests/new', label: "New merge request",  type: :merge_requests
 
   = render 'shared/issuable/filter', type: :merge_requests
 
diff --git a/app/views/shared/_group_form.html.haml b/app/views/shared/_group_form.html.haml
index 8d5b5129454..2e1bd5a088c 100644
--- a/app/views/shared/_group_form.html.haml
+++ b/app/views/shared/_group_form.html.haml
@@ -1,6 +1,6 @@
 - content_for :page_specific_javascripts do
   = page_specific_javascript_bundle_tag('group')
-- parent = GroupFinder.new(current_user).execute(id: params[:parent_id] || @group.parent_id)
+- parent = @group.parent
 - group_path = root_url
 - group_path << parent.full_path + '/' if parent
 
@@ -13,13 +13,12 @@
         %span>= root_url
         - if parent
           %strong= parent.full_path + '/'
+      = f.hidden_field :parent_id
       = f.text_field :path, placeholder: 'open-source', class: 'form-control',
         autofocus: local_assigns[:autofocus] || false, required: true,
         pattern: Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX_JS,
         title: 'Please choose a group path with no special characters.',
         "data-bind-in" => "#{'create_chat_team' if Gitlab.config.mattermost.enabled}"
-      - if parent
-        = f.hidden_field :parent_id, value: parent.id
 
     - if @group.persisted?
       .alert.alert-warning.prepend-top-10
diff --git a/app/views/shared/_new_project_item_select.html.haml b/app/views/shared/_new_project_item_select.html.haml
index 96502d7ce93..dc912d800cf 100644
--- a/app/views/shared/_new_project_item_select.html.haml
+++ b/app/views/shared/_new_project_item_select.html.haml
@@ -1,6 +1,6 @@
 - if any_projects?(@projects)
   .project-item-select-holder.btn-group.pull-right
-    %a.btn.btn-new.new-project-item-link{ href: '', data: { label: local_assigns[:label] } }
+    %a.btn.btn-new.new-project-item-link{ href: '', data: { label: local_assigns[:label], type: local_assigns[:type] } }
       = icon('spinner spin')
     = project_select_tag :project_path, class: "project-item-select", data: { include_groups: local_assigns[:include_groups], order_by: 'last_activity_at', relative_path: local_assigns[:path] }, with_feature_enabled: local_assigns[:with_feature_enabled]
     %button.btn.btn-new.new-project-item-select-button
diff --git a/app/views/shared/empty_states/_issues.html.haml b/app/views/shared/empty_states/_issues.html.haml
index b0c0ab523c7..68737e8da66 100644
--- a/app/views/shared/empty_states/_issues.html.haml
+++ b/app/views/shared/empty_states/_issues.html.haml
@@ -15,7 +15,7 @@
           Issues can be bugs, tasks or ideas to be discussed.
           Also, issues are searchable and filterable.
         - if project_select_button
-          = render 'shared/new_project_item_select', path: 'issues/new', label: 'New issue'
+          = render 'shared/new_project_item_select', path: 'issues/new', label: 'New issue', type: :issues
         - else
           = link_to 'New issue', button_path, class: 'btn btn-new', title: 'New issue', id: 'new_issue_link'
       - else
diff --git a/app/views/shared/empty_states/_merge_requests.html.haml b/app/views/shared/empty_states/_merge_requests.html.haml
index 3e64f403b8b..ff5741b6d61 100644
--- a/app/views/shared/empty_states/_merge_requests.html.haml
+++ b/app/views/shared/empty_states/_merge_requests.html.haml
@@ -14,7 +14,7 @@
         %p
           Interested parties can even contribute by pushing commits if they want to.
         - if project_select_button
-          = render 'shared/new_project_item_select', path: 'merge_requests/new', label: 'New merge request'
+          = render 'shared/new_project_item_select', path: 'merge_requests/new', label: 'New merge request', type: :merge_requests
         - else
           = link_to 'New merge request', button_path, class: 'btn btn-new', title: 'New merge request', id: 'new_merge_request_link'
       - else
diff --git a/app/workers/authorized_projects_worker.rb b/app/workers/authorized_projects_worker.rb
index 13207a8bc71..be4c77503bb 100644
--- a/app/workers/authorized_projects_worker.rb
+++ b/app/workers/authorized_projects_worker.rb
@@ -4,18 +4,25 @@ class AuthorizedProjectsWorker
 
   # Schedules multiple jobs and waits for them to be completed.
   def self.bulk_perform_and_wait(args_list)
-    job_ids = bulk_perform_async(args_list)
+    waiter = Gitlab::JobWaiter.new(args_list.size)
 
-    Gitlab::JobWaiter.new(job_ids).wait
+    # Point all the bulk jobs at the same JobWaiter. Converts, [[1], [2], [3]]
+    # into [[1, "key"], [2, "key"], [3, "key"]]
+    waiting_args_list = args_list.map { |args| args << waiter.key }
+    bulk_perform_async(waiting_args_list)
+
+    waiter.wait
   end
 
   def self.bulk_perform_async(args_list)
     Sidekiq::Client.push_bulk('class' => self, 'queue' => sidekiq_options['queue'], 'args' => args_list)
   end
 
-  def perform(user_id)
+  def perform(user_id, notify_key = nil)
     user = User.find_by(id: user_id)
 
     user&.refresh_authorized_projects
+  ensure
+    Gitlab::JobWaiter.notify(notify_key, jid) if notify_key
   end
 end
diff --git a/app/workers/stage_update_worker.rb b/app/workers/stage_update_worker.rb
new file mode 100644
index 00000000000..eef0b11e70b
--- /dev/null
+++ b/app/workers/stage_update_worker.rb
@@ -0,0 +1,10 @@
+class StageUpdateWorker
+  include Sidekiq::Worker
+  include PipelineQueue
+
+  def perform(stage_id)
+    Ci::Stage.find_by(id: stage_id).try do |stage|
+      stage.update_status
+    end
+  end
+end