Merge branch 'master' into 38031-monitoring-hover-info-is-clipped38031-monitoring-hover-info-is-clipped

9 files changed, 153 insertions, 42 deletions

diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index acaa028eaa2..3d5acc00f8f 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -434,7 +434,7 @@ module Ci
     def update_duration
       return unless started_at
 
-      self.duration = Gitlab::Ci::PipelineDuration.from_pipeline(self)
+      self.duration = Gitlab::Ci::Pipeline::Duration.from_pipeline(self)
     end
 
     def execute_hooks
diff --git a/app/models/gpg_key.rb b/app/models/gpg_key.rb
index 44deae4234b..54bd5b68777 100644
--- a/app/models/gpg_key.rb
+++ b/app/models/gpg_key.rb
@@ -73,7 +73,7 @@ class GpgKey < ActiveRecord::Base
   end
 
   def verified_and_belongs_to_email?(email)
-    emails_with_verified_status.fetch(email, false)
+    emails_with_verified_status.fetch(email.downcase, false)
   end
 
   def update_invalid_gpg_signatures
diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb
index 8d9a30397a9..e85b83daf9e 100644
--- a/app/models/merge_request.rb
+++ b/app/models/merge_request.rb
@@ -524,6 +524,14 @@ class MergeRequest < ActiveRecord::Base
     true
   end
 
+  def ff_merge_possible?
+    project.repository.ancestor?(target_branch_sha, diff_head_sha)
+  end
+
+  def should_be_rebased?
+    project.ff_merge_must_be_possible? && !ff_merge_possible?
+  end
+
   def can_cancel_merge_when_pipeline_succeeds?(current_user)
     can_be_merged_by?(current_user) || self.author == current_user
   end
diff --git a/app/models/project.rb b/app/models/project.rb
index f7221e4f3b2..59b5a5b3cd7 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -64,6 +64,7 @@ class Project < ActiveRecord::Base
 
   # Storage specific hooks
   after_initialize :use_hashed_storage
+  after_create :check_repository_absence!
   after_create :ensure_storage_path_exists
   after_save :ensure_storage_path_exists, if: :namespace_id_changed?
 
@@ -72,6 +73,7 @@ class Project < ActiveRecord::Base
   attr_accessor :old_path_with_namespace
   attr_accessor :template_name
   attr_writer :pipeline_status
+  attr_accessor :skip_disk_validation
 
   alias_attribute :title, :name
 
@@ -227,7 +229,7 @@ class Project < ActiveRecord::Base
   validates :import_url, importable_url: true, if: [:external_import?, :import_url_changed?]
   validates :star_count, numericality: { greater_than_or_equal_to: 0 }
   validate :check_limit, on: :create
-  validate :can_create_repository?, on: [:create, :update], if: ->(project) { !project.persisted? || project.renamed? }
+  validate :check_repository_path_availability, on: :update, if: ->(project) { project.renamed? }
   validate :avatar_type,
     if: ->(project) { project.avatar.present? && project.avatar_changed? }
   validates :avatar, file_size: { maximum: 200.kilobytes.to_i }
@@ -245,6 +247,9 @@ class Project < ActiveRecord::Base
   scope :pending_delete, -> { where(pending_delete: true) }
   scope :without_deleted, -> { where(pending_delete: false) }
 
+  scope :with_hashed_storage, -> { where('storage_version >= 1') }
+  scope :with_legacy_storage, -> { where(storage_version: [nil, 0]) }
+
   scope :sorted_by_activity, -> { reorder(last_activity_at: :desc) }
   scope :sorted_by_stars, -> { reorder('projects.star_count DESC') }
 
@@ -1015,12 +1020,15 @@ class Project < ActiveRecord::Base
   end
 
   # Check if repository already exists on disk
-  def can_create_repository?
+  def check_repository_path_availability
+    return true if skip_disk_validation
     return false unless repository_storage_path
 
     expires_full_path_cache # we need to clear cache to validate renames correctly
 
-    if gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git")
+    # Check if repository with same path already exists on disk we can
+    # skip this for the hashed storage because the path does not change
+    if legacy_storage? && repository_with_same_path_already_exists?
       errors.add(:base, 'There is already a repository with that name on disk')
       return false
     end
@@ -1032,7 +1040,7 @@ class Project < ActiveRecord::Base
     # Forked import is handled asynchronously
     return if forked? && !force
 
-    if gitlab_shell.add_repository(repository_storage_path, disk_path)
+    if gitlab_shell.add_repository(repository_storage, disk_path)
       repository.after_create
       true
     else
@@ -1550,18 +1558,72 @@ class Project < ActiveRecord::Base
   end
 
   def legacy_storage?
-    self.storage_version.nil?
+    [nil, 0].include?(self.storage_version)
+  end
+
+  def hashed_storage?
+    self.storage_version && self.storage_version >= 1
   end
 
   def renamed?
     persisted? && path_changed?
   end
 
+  def merge_method
+    if self.merge_requests_ff_only_enabled
+      :ff
+    elsif self.merge_requests_rebase_enabled
+      :rebase_merge
+    else
+      :merge
+    end
+  end
+
+  def merge_method=(method)
+    case method.to_s
+    when "ff"
+      self.merge_requests_ff_only_enabled = true
+      self.merge_requests_rebase_enabled = true
+    when "rebase_merge"
+      self.merge_requests_ff_only_enabled = false
+      self.merge_requests_rebase_enabled = true
+    when "merge"
+      self.merge_requests_ff_only_enabled = false
+      self.merge_requests_rebase_enabled = false
+    end
+  end
+
+  def ff_merge_must_be_possible?
+    self.merge_requests_ff_only_enabled || self.merge_requests_rebase_enabled
+  end
+
+  def migrate_to_hashed_storage!
+    return if hashed_storage?
+
+    update!(repository_read_only: true)
+
+    if repo_reference_count > 0 || wiki_reference_count > 0
+      ProjectMigrateHashedStorageWorker.perform_in(Gitlab::ReferenceCounter::REFERENCE_EXPIRE_TIME, id)
+    else
+      ProjectMigrateHashedStorageWorker.perform_async(id)
+    end
+  end
+
+  def storage_version=(value)
+    super
+
+    @storage = nil if storage_version_changed?
+  end
+
+  def gl_repository(is_wiki:)
+    Gitlab::GlRepository.gl_repository(self, is_wiki)
+  end
+
   private
 
   def storage
     @storage ||=
-      if self.storage_version && self.storage_version >= 1
+      if hashed_storage?
         Storage::HashedProject.new(self)
       else
         Storage::LegacyProject.new(self)
@@ -1574,6 +1636,27 @@ class Project < ActiveRecord::Base
     end
   end
 
+  def repo_reference_count
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: false)).value
+  end
+
+  def wiki_reference_count
+    Gitlab::ReferenceCounter.new(gl_repository(is_wiki: true)).value
+  end
+
+  def check_repository_absence!
+    return if skip_disk_validation
+
+    if repository_storage_path.blank? || repository_with_same_path_already_exists?
+      errors.add(:base, 'There is already a repository with that name on disk')
+      throw :abort
+    end
+  end
+
+  def repository_with_same_path_already_exists?
+    gitlab_shell.exists?(repository_storage_path, "#{disk_path}.git")
+  end
+
   # set last_activity_at to the same as created_at
   def set_last_activity_at
     update_column(:last_activity_at, self.created_at)
diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb
index 698fdf7a20c..c4cc1c1cf22 100644
--- a/app/models/project_wiki.rb
+++ b/app/models/project_wiki.rb
@@ -174,7 +174,7 @@ class ProjectWiki
   private
 
   def init_repo(disk_path)
-    gitlab_shell.add_repository(project.repository_storage_path, disk_path)
+    gitlab_shell.add_repository(project.repository_storage, disk_path)
   end
 
   def commit_details(action, message = nil, title = nil)
diff --git a/app/models/repository.rb b/app/models/repository.rb
index b28fe79e19c..d47dc9a05cd 100644
--- a/app/models/repository.rb
+++ b/app/models/repository.rb
@@ -34,7 +34,10 @@ class Repository
   CACHED_METHODS = %i(size commit_count rendered_readme contribution_guide
                       changelog license_blob license_key gitignore koding_yml
                       gitlab_ci_yml branch_names tag_names branch_count
-                      tag_count avatar exists? empty? root_ref).freeze
+                      tag_count avatar exists? empty? root_ref has_visible_content?).freeze
+
+  # Methods that use cache_method but only memoize the value
+  MEMOIZED_CACHED_METHODS = %i(license empty_repo?).freeze
 
   # Certain method caches should be refreshed when certain types of files are
   # changed. This Hash maps file types (as returned by Gitlab::FileDetector) to
@@ -91,12 +94,6 @@ class Repository
     )
   end
 
-  # we need to have this method here because it is not cached in ::Git and
-  # the method is called multiple times for every request
-  def has_visible_content?
-    branch_count > 0
-  end
-
   def inspect
     "#<#{self.class.name}:#{@disk_path}>"
   end
@@ -275,7 +272,7 @@ class Repository
   end
 
   def expire_branches_cache
-    expire_method_caches(%i(branch_names branch_count))
+    expire_method_caches(%i(branch_names branch_count has_visible_content?))
     @local_branches = nil
     @branch_exists_memo = nil
   end
@@ -346,7 +343,7 @@ class Repository
   def expire_emptiness_caches
     return unless empty?
 
-    expire_method_caches(%i(empty?))
+    expire_method_caches(%i(empty? has_visible_content?))
   end
 
   def lookup_cache
@@ -489,13 +486,7 @@ class Repository
   def exists?
     return false unless full_path
 
-    Gitlab::GitalyClient.migrate(:repository_exists) do |enabled|
-      if enabled
-        raw_repository.exists?
-      else
-        refs_directory_exists?
-      end
-    end
+    raw_repository.exists?
   end
   cache_method :exists?
 
@@ -529,9 +520,10 @@ class Repository
   delegate :tag_names, to: :raw_repository
   cache_method :tag_names, fallback: []
 
-  delegate :branch_count, :tag_count, to: :raw_repository
+  delegate :branch_count, :tag_count, :has_visible_content?, to: :raw_repository
   cache_method :branch_count, fallback: 0
   cache_method :tag_count, fallback: 0
+  cache_method :has_visible_content?, fallback: false
 
   def avatar
     # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/38327
@@ -858,6 +850,25 @@ class Repository
     end
   end
 
+  def ff_merge(user, source, target_branch, merge_request: nil)
+    our_commit = rugged.branches[target_branch].target
+    their_commit =
+      if source.is_a?(Gitlab::Git::Commit)
+        source.raw_commit
+      else
+        rugged.lookup(source)
+      end
+
+    raise 'Invalid merge target' if our_commit.nil?
+    raise 'Invalid merge source' if their_commit.nil?
+
+    with_branch(user, target_branch) do |start_commit|
+      merge_request&.update(in_progress_merge_commit_sha: their_commit.oid)
+
+      their_commit.oid
+    end
+  end
+
   def revert(
     user, commit, branch_name, message,
     start_branch_name: nil, start_project: project)
@@ -1063,12 +1074,6 @@ class Repository
     blob.data
   end
 
-  def refs_directory_exists?
-    circuit_breaker.perform do
-      File.exist?(File.join(path_to_repo, 'refs'))
-    end
-  end
-
   def cache
     # TODO: should we use UUIDs here? We could move repositories without clearing this cache
     @cache ||= RepositoryCache.new(full_path, @project.id)
@@ -1120,10 +1125,6 @@ class Repository
     Gitlab::Git::Repository.new(project.repository_storage, disk_path + '.git', Gitlab::GlRepository.gl_repository(project, false))
   end
 
-  def circuit_breaker
-    @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(project.repository_storage)
-  end
-
   def find_commits_by_message_by_shelling_out(query, ref, path, limit, offset)
     ref ||= root_ref
 
diff --git a/app/models/storage/hashed_project.rb b/app/models/storage/hashed_project.rb
index fae1b64961a..f025f40994e 100644
--- a/app/models/storage/hashed_project.rb
+++ b/app/models/storage/hashed_project.rb
@@ -4,6 +4,7 @@ module Storage
     delegate :gitlab_shell, :repository_storage_path, to: :project
 
     ROOT_PATH_PREFIX = '@hashed'.freeze
+    STORAGE_VERSION = 1
 
     def initialize(project)
       @project = project
diff --git a/app/models/user.rb b/app/models/user.rb
index 09c9b3250eb..4e71a3e11c2 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -60,7 +60,7 @@ class User < ActiveRecord::Base
     lease = Gitlab::ExclusiveLease.new("user_update_tracked_fields:#{id}", timeout: 1.hour.to_i)
     return unless lease.try_obtain
 
-    Users::UpdateService.new(self).execute(validate: false)
+    Users::UpdateService.new(self, user: self).execute(validate: false)
   end
 
   attr_accessor :force_random_password
@@ -130,6 +130,8 @@ class User < ActiveRecord::Base
   has_many :assigned_issues, class_name: "Issue", through: :issue_assignees, source: :issue
   has_many :assigned_merge_requests,  dependent: :nullify, foreign_key: :assignee_id, class_name: "MergeRequest" # rubocop:disable Cop/ActiveRecordDependent
 
+  has_many :custom_attributes, class_name: 'UserCustomAttribute'
+
   #
   # Validations
   #
@@ -526,8 +528,8 @@ class User < ActiveRecord::Base
   def update_emails_with_primary_email
     primary_email_record = emails.find_by(email: email)
     if primary_email_record
-      Emails::DestroyService.new(self, email: email).execute
-      Emails::CreateService.new(self, email: email_was).execute
+      Emails::DestroyService.new(self, user: self, email: email).execute
+      Emails::CreateService.new(self, user: self, email: email_was).execute
     end
   end
 
@@ -690,7 +692,11 @@ class User < ActiveRecord::Base
   end
 
   def ldap_user?
-    identities.exists?(["provider LIKE ? AND extern_uid IS NOT NULL", "ldap%"])
+    if identities.loaded?
+      identities.find { |identity| identity.provider.start_with?('ldap') && !identity.extern_uid.nil? }
+    else
+      identities.exists?(["provider LIKE ? AND extern_uid IS NOT NULL", "ldap%"])
+    end
   end
 
   def ldap_identity
@@ -1000,7 +1006,7 @@ class User < ActiveRecord::Base
     if attempts_exceeded?
       lock_access! unless access_locked?
     else
-      Users::UpdateService.new(self).execute(validate: false)
+      Users::UpdateService.new(self, user: self).execute(validate: false)
     end
   end
 
@@ -1061,6 +1067,12 @@ class User < ActiveRecord::Base
     user_synced_attributes_metadata&.read_only?(attribute)
   end
 
+  # override, from Devise
+  def lock_access!
+    Gitlab::AppLogger.info("Account Locked: username=#{username}")
+    super
+  end
+
   protected
 
   # override, from Devise::Validatable
@@ -1186,7 +1198,7 @@ class User < ActiveRecord::Base
       &creation_block
     )
 
-    Users::UpdateService.new(user).execute(validate: false)
+    Users::UpdateService.new(user, user: user).execute(validate: false)
     user
   ensure
     Gitlab::ExclusiveLease.cancel(lease_key, uuid)
diff --git a/app/models/user_custom_attribute.rb b/app/models/user_custom_attribute.rb
new file mode 100644
index 00000000000..eff25b31f9b
--- /dev/null
+++ b/app/models/user_custom_attribute.rb
@@ -0,0 +1,6 @@
+class UserCustomAttribute < ActiveRecord::Base
+  belongs_to :user
+
+  validates :user_id, :key, :value, presence: true
+  validates :key, uniqueness: { scope: [:user_id] }
+end