Add latest changes from gitlab-org/gitlab@13-0-stable-ee

7 files changed, 718 insertions, 0 deletions

diff --git a/app/models/design_management/action.rb b/app/models/design_management/action.rb
new file mode 100644
index 00000000000..ecd7973a523
--- /dev/null
+++ b/app/models/design_management/action.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_dependency 'design_management'
+
+module DesignManagement
+  class Action < ApplicationRecord
+    include WithUploads
+
+    self.table_name = "#{DesignManagement.table_name_prefix}designs_versions"
+
+    mount_uploader :image_v432x230, DesignManagement::DesignV432x230Uploader
+
+    belongs_to :design, class_name: "DesignManagement::Design", inverse_of: :actions
+    belongs_to :version, class_name: "DesignManagement::Version", inverse_of: :actions
+
+    enum event: { creation: 0, modification: 1, deletion: 2 }
+
+    # we assume sequential ordering.
+    scope :ordered, -> { order(version_id: :asc) }
+
+    # For each design, only select the most recent action
+    scope :most_recent, -> do
+      selection = Arel.sql("DISTINCT ON (#{table_name}.design_id) #{table_name}.*")
+
+      order(arel_table[:design_id].asc, arel_table[:version_id].desc).select(selection)
+    end
+
+    # Find all records created before or at the given version, or all if nil
+    scope :up_to_version, ->(version = nil) do
+      case version
+      when nil
+        all
+      when DesignManagement::Version
+        where(arel_table[:version_id].lteq(version.id))
+      when ::Gitlab::Git::COMMIT_ID
+        versions = DesignManagement::Version.arel_table
+        subquery = versions.project(versions[:id]).where(versions[:sha].eq(version))
+        where(arel_table[:version_id].lteq(subquery))
+      else
+        raise ArgumentError, "Expected a DesignManagement::Version, got #{version}"
+      end
+    end
+  end
+end
diff --git a/app/models/design_management/design.rb b/app/models/design_management/design.rb
new file mode 100644
index 00000000000..e9b69eab7a7
--- /dev/null
+++ b/app/models/design_management/design.rb
@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  class Design < ApplicationRecord
+    include Importable
+    include Noteable
+    include Gitlab::FileTypeDetection
+    include Gitlab::Utils::StrongMemoize
+    include Referable
+    include Mentionable
+    include WhereComposite
+
+    belongs_to :project, inverse_of: :designs
+    belongs_to :issue
+
+    has_many :actions
+    has_many :versions, through: :actions, class_name: 'DesignManagement::Version', inverse_of: :designs
+    # This is a polymorphic association, so we can't count on FK's to delete the
+    # data
+    has_many :notes, as: :noteable, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
+    has_many :user_mentions, class_name: 'DesignUserMention', dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent
+
+    validates :project, :filename, presence: true
+    validates :issue, presence: true, unless: :importing?
+    validates :filename, uniqueness: { scope: :issue_id }
+    validate :validate_file_is_image
+
+    alias_attribute :title, :filename
+
+    # Pre-fetching scope to include the data necessary to construct a
+    # reference using `to_reference`.
+    scope :for_reference, -> { includes(issue: [{ project: [:route, :namespace] }]) }
+
+    # A design can be uniquely identified by issue_id and filename
+    # Takes one or more sets of composite IDs of the form:
+    # `{issue_id: Integer, filename: String}`.
+    #
+    # @see WhereComposite::where_composite
+    #
+    # e.g:
+    #
+    #   by_issue_id_and_filename(issue_id: 1, filename: 'homescreen.jpg')
+    #   by_issue_id_and_filename([]) # returns ActiveRecord::NullRelation
+    #   by_issue_id_and_filename([
+    #     { issue_id: 1, filename: 'homescreen.jpg' },
+    #     { issue_id: 2, filename: 'homescreen.jpg' },
+    #     { issue_id: 1, filename: 'menu.png' }
+    #   ])
+    #
+    scope :by_issue_id_and_filename, ->(composites) do
+      where_composite(%i[issue_id filename], composites)
+    end
+
+    # Find designs visible at the given version
+    #
+    # @param version [nil, DesignManagement::Version]:
+    #   the version at which the designs must be visible
+    #   Passing `nil` is the same as passing the most current version
+    #
+    # Restricts to designs
+    # - created at least *before* the given version
+    # - not deleted as of the given version.
+    #
+    # As a query, we ascertain this by finding the last event prior to
+    # (or equal to) the cut-off, and seeing whether that version was a deletion.
+    scope :visible_at_version, -> (version) do
+      deletion = ::DesignManagement::Action.events[:deletion]
+      designs = arel_table
+      actions = ::DesignManagement::Action
+        .most_recent.up_to_version(version)
+        .arel.as('most_recent_actions')
+
+      join = designs.join(actions)
+        .on(actions[:design_id].eq(designs[:id]))
+
+      joins(join.join_sources).where(actions[:event].not_eq(deletion)).order(:id)
+    end
+
+    scope :with_filename, -> (filenames) { where(filename: filenames) }
+    scope :on_issue, ->(issue) { where(issue_id: issue) }
+
+    # Scope called by our REST API to avoid N+1 problems
+    scope :with_api_entity_associations, -> { preload(:issue) }
+
+    # A design is current if the most recent event is not a deletion
+    scope :current, -> { visible_at_version(nil) }
+
+    def status
+      if new_design?
+        :new
+      elsif deleted?
+        :deleted
+      else
+        :current
+      end
+    end
+
+    def deleted?
+      most_recent_action&.deletion?
+    end
+
+    # A design is visible_in? a version if:
+    #   * it was created before that version
+    #   * the most recent action before the version was not a deletion
+    def visible_in?(version)
+      map = strong_memoize(:visible_in) do
+        Hash.new do |h, k|
+          h[k] = self.class.visible_at_version(k).where(id: id).exists?
+        end
+      end
+
+      map[version]
+    end
+
+    def most_recent_action
+      strong_memoize(:most_recent_action) { actions.ordered.last }
+    end
+
+    # A reference for a design is the issue reference, indexed by the filename
+    # with an optional infix when full.
+    #
+    # e.g.
+    #   #123[homescreen.png]
+    #   other-project#72[sidebar.jpg]
+    #   #38/designs[transition.gif]
+    #   #12["filename with [] in it.jpg"]
+    def to_reference(from = nil, full: false)
+      infix = full ? '/designs' : ''
+      totally_simple = %r{ \A #{self.class.simple_file_name} \z }x
+      safe_name = if totally_simple.match?(filename)
+                    filename
+                  elsif filename =~ /[<>]/
+                    %Q{base64:#{Base64.strict_encode64(filename)}}
+                  else
+                    escaped = filename.gsub(%r{[\\"]}) { |x| "\\#{x}" }
+                    %Q{"#{escaped}"}
+                  end
+
+      "#{issue.to_reference(from, full: full)}#{infix}[#{safe_name}]"
+    end
+
+    def self.reference_pattern
+      @reference_pattern ||= begin
+        # Filenames can be escaped with double quotes to name filenames
+        # that include square brackets, or other special characters
+        %r{
+          #{Issue.reference_pattern}
+          (\/designs)?
+          \[
+            (?<design> #{simple_file_name} | #{quoted_file_name} | #{base_64_encoded_name})
+          \]
+        }x
+      end
+    end
+
+    def self.simple_file_name
+      %r{
+          (?<simple_file_name>
+           ( \w | [_:,'-] | \. | \s )+
+           \.
+           \w+
+          )
+      }x
+    end
+
+    def self.base_64_encoded_name
+      %r{
+          base64:
+          (?<base_64_encoded_name>
+           [A-Za-z0-9+\n]+
+           =?
+          )
+      }x
+    end
+
+    def self.quoted_file_name
+      %r{
+          "
+          (?<escaped_filename>
+            (\\ \\ | \\ " | [^"\\])+
+          )
+          "
+      }x
+    end
+
+    def self.link_reference_pattern
+      @link_reference_pattern ||= begin
+        exts = SAFE_IMAGE_EXT + DANGEROUS_IMAGE_EXT
+        path_segment = %r{issues/#{Gitlab::Regex.issue}/designs}
+        filename_pattern = %r{(?<simple_file_name>[a-z0-9_=-]+\.(#{exts.join('|')}))}i
+
+        super(path_segment, filename_pattern)
+      end
+    end
+
+    def to_ability_name
+      'design'
+    end
+
+    def description
+      ''
+    end
+
+    def new_design?
+      strong_memoize(:new_design) { actions.none? }
+    end
+
+    def full_path
+      @full_path ||= File.join(DesignManagement.designs_directory, "issue-#{issue.iid}", filename)
+    end
+
+    def diff_refs
+      strong_memoize(:diff_refs) { head_version&.diff_refs }
+    end
+
+    def clear_version_cache
+      [versions, actions].each(&:reset)
+      %i[new_design diff_refs head_sha visible_in most_recent_action].each do |key|
+        clear_memoization(key)
+      end
+    end
+
+    def repository
+      project.design_repository
+    end
+
+    def user_notes_count
+      user_notes_count_service.count
+    end
+
+    def after_note_changed(note)
+      user_notes_count_service.delete_cache unless note.system?
+    end
+    alias_method :after_note_created,   :after_note_changed
+    alias_method :after_note_destroyed, :after_note_changed
+
+    private
+
+    def head_version
+      strong_memoize(:head_sha) { versions.ordered.first }
+    end
+
+    def allow_dangerous_images?
+      Feature.enabled?(:design_management_allow_dangerous_images, project)
+    end
+
+    def valid_file_extensions
+      allow_dangerous_images? ? (SAFE_IMAGE_EXT + DANGEROUS_IMAGE_EXT) : SAFE_IMAGE_EXT
+    end
+
+    def validate_file_is_image
+      unless image? || (dangerous_image? && allow_dangerous_images?)
+        message = _('does not have a supported extension. Only %{extension_list} are supported') % {
+          extension_list: valid_file_extensions.to_sentence
+        }
+        errors.add(:filename, message)
+      end
+    end
+
+    def user_notes_count_service
+      strong_memoize(:user_notes_count_service) do
+        ::DesignManagement::DesignUserNotesCountService.new(self) # rubocop: disable CodeReuse/ServiceClass
+      end
+    end
+  end
+end
diff --git a/app/models/design_management/design_action.rb b/app/models/design_management/design_action.rb
new file mode 100644
index 00000000000..22baa916296
--- /dev/null
+++ b/app/models/design_management/design_action.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  # Parameter object which is a tuple of the database record and the
+  # last gitaly call made to change it. This serves to perform the
+  # logical mapping from git action to database representation.
+  class DesignAction
+    include ActiveModel::Validations
+
+    EVENT_FOR_GITALY_ACTION = {
+      create: DesignManagement::Action.events[:creation],
+      update: DesignManagement::Action.events[:modification],
+      delete: DesignManagement::Action.events[:deletion]
+    }.freeze
+
+    attr_reader :design, :action, :content
+
+    delegate :issue_id, to: :design
+
+    validates :design, presence: true
+    validates :action, presence: true, inclusion: { in: EVENT_FOR_GITALY_ACTION.keys }
+    validates :content,
+      absence: { if: :forbids_content?,
+                  message: 'this action forbids content' },
+      presence: { if: :needs_content?,
+                  message: 'this action needs content' }
+
+    # Parameters:
+    # - design [DesignManagement::Design]: the design that was changed
+    # - action [Symbol]: the action that gitaly performed
+    def initialize(design, action, content = nil)
+      @design, @action, @content = design, action, content
+      validate!
+    end
+
+    def row_attrs(version)
+      { design_id: design.id, version_id: version.id, event: event }
+    end
+
+    def gitaly_action
+      { action: action, file_path: design.full_path, content: content }.compact
+    end
+
+    # This action has been performed - do any post-creation actions
+    # such as clearing method caches.
+    def performed
+      design.clear_version_cache
+    end
+
+    private
+
+    def needs_content?
+      action != :delete
+    end
+
+    def forbids_content?
+      action == :delete
+    end
+
+    def event
+      EVENT_FOR_GITALY_ACTION[action]
+    end
+  end
+end
diff --git a/app/models/design_management/design_at_version.rb b/app/models/design_management/design_at_version.rb
new file mode 100644
index 00000000000..b4cafb93c2c
--- /dev/null
+++ b/app/models/design_management/design_at_version.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+# Tuple of design and version
+# * has a composite ID, with lazy_find
+module DesignManagement
+  class DesignAtVersion
+    include ActiveModel::Validations
+    include GlobalID::Identification
+    include Gitlab::Utils::StrongMemoize
+
+    attr_reader :version
+    attr_reader :design
+
+    validates :version, presence: true
+    validates :design, presence: true
+
+    validate :design_and_version_belong_to_the_same_issue
+    validate :design_and_version_have_issue_id
+
+    def initialize(design: nil, version: nil)
+      @design, @version = design, version
+    end
+
+    def self.instantiate(attrs)
+      new(attrs).tap { |obj| obj.validate! }
+    end
+
+    # The ID, needed by GraphQL types and as part of the Lazy-fetch
+    # protocol, includes information about both the design and the version.
+    #
+    # The particular format is not interesting, and should be treated as opaque
+    # by all callers.
+    def id
+      "#{design.id}.#{version.id}"
+    end
+
+    def ==(other)
+      return false unless other && self.class == other.class
+
+      other.id == id
+    end
+
+    alias_method :eql?, :==
+
+    def self.lazy_find(id)
+      BatchLoader.for(id).batch do |ids, callback|
+        find(ids).each do |record|
+          callback.call(record.id, record)
+        end
+      end
+    end
+
+    def self.find(ids)
+      pairs = ids.map { |id| id.split('.').map(&:to_i) }
+
+      design_ids = pairs.map(&:first).uniq
+      version_ids = pairs.map(&:second).uniq
+
+      designs = ::DesignManagement::Design
+        .where(id: design_ids)
+        .index_by(&:id)
+
+      versions = ::DesignManagement::Version
+        .where(id: version_ids)
+        .index_by(&:id)
+
+      pairs.map do |(design_id, version_id)|
+        design = designs[design_id]
+        version = versions[version_id]
+
+        obj = new(design: design, version: version)
+
+        obj if obj.valid?
+      end.compact
+    end
+
+    def status
+      if not_created_yet?
+        :not_created_yet
+      elsif deleted?
+        :deleted
+      else
+        :current
+      end
+    end
+
+    def deleted?
+      action&.deletion?
+    end
+
+    def not_created_yet?
+      action.nil?
+    end
+
+    private
+
+    def action
+      strong_memoize(:most_recent_action) do
+        ::DesignManagement::Action
+          .most_recent.up_to_version(version)
+          .find_by(design: design)
+      end
+    end
+
+    def design_and_version_belong_to_the_same_issue
+      id_a, id_b = [design, version].map { |obj| obj&.issue_id }
+
+      return if id_a == id_b
+
+      errors.add(:issue, 'must be the same on design and version')
+    end
+
+    def design_and_version_have_issue_id
+      return if [design, version].all? { |obj| obj.try(:issue_id).present? }
+
+      errors.add(:issue, 'must be present on both design and version')
+    end
+  end
+end
diff --git a/app/models/design_management/design_collection.rb b/app/models/design_management/design_collection.rb
new file mode 100644
index 00000000000..18d1541e9c7
--- /dev/null
+++ b/app/models/design_management/design_collection.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  class DesignCollection
+    attr_reader :issue
+
+    delegate :designs, :project, to: :issue
+
+    def initialize(issue)
+      @issue = issue
+    end
+
+    def find_or_create_design!(filename:)
+      designs.find { |design| design.filename == filename } ||
+        designs.safe_find_or_create_by!(project: project, filename: filename)
+    end
+
+    def versions
+      @versions ||= DesignManagement::Version.for_designs(designs)
+    end
+
+    def repository
+      project.design_repository
+    end
+
+    def designs_by_filename(filenames)
+      designs.current.where(filename: filenames)
+    end
+  end
+end
diff --git a/app/models/design_management/repository.rb b/app/models/design_management/repository.rb
new file mode 100644
index 00000000000..985d6317d5d
--- /dev/null
+++ b/app/models/design_management/repository.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  class Repository < ::Repository
+    extend ::Gitlab::Utils::Override
+
+    # We define static git attributes for the design repository as this
+    # repository is entirely GitLab-managed rather than user-facing.
+    #
+    # Enable all uploaded files to be stored in LFS.
+    MANAGED_GIT_ATTRIBUTES = <<~GA.freeze
+      /#{DesignManagement.designs_directory}/* filter=lfs diff=lfs merge=lfs -text
+    GA
+
+    def initialize(project)
+      full_path = project.full_path + Gitlab::GlRepository::DESIGN.path_suffix
+      disk_path = project.disk_path + Gitlab::GlRepository::DESIGN.path_suffix
+
+      super(full_path, project, shard: project.repository_storage, disk_path: disk_path, repo_type: Gitlab::GlRepository::DESIGN)
+    end
+
+    # Override of a method called on Repository instances but sent via
+    # method_missing to Gitlab::Git::Repository where it is defined
+    def info_attributes
+      @info_attributes ||= Gitlab::Git::AttributesParser.new(MANAGED_GIT_ATTRIBUTES)
+    end
+
+    # Override of a method called on Repository instances but sent via
+    # method_missing to Gitlab::Git::Repository where it is defined
+    def attributes(path)
+      info_attributes.attributes(path)
+    end
+
+    # Override of a method called on Repository instances but sent via
+    # method_missing to Gitlab::Git::Repository where it is defined
+    def gitattribute(path, name)
+      attributes(path)[name]
+    end
+
+    # Override of a method called on Repository instances but sent via
+    # method_missing to Gitlab::Git::Repository where it is defined
+    def attributes_at(_ref = nil)
+      info_attributes
+    end
+
+    override :copy_gitattributes
+    def copy_gitattributes(_ref = nil)
+      true
+    end
+  end
+end
diff --git a/app/models/design_management/version.rb b/app/models/design_management/version.rb
new file mode 100644
index 00000000000..6be98fe3d44
--- /dev/null
+++ b/app/models/design_management/version.rb
@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module DesignManagement
+  class Version < ApplicationRecord
+    include Importable
+    include ShaAttribute
+    include AfterCommitQueue
+    include Gitlab::Utils::StrongMemoize
+    extend Gitlab::ExclusiveLeaseHelpers
+
+    NotSameIssue = Class.new(StandardError)
+
+    class CouldNotCreateVersion < StandardError
+      attr_reader :sha, :issue_id, :actions
+
+      def initialize(sha, issue_id, actions)
+        @sha, @issue_id, @actions = sha, issue_id, actions
+      end
+
+      def message
+        "could not create version from commit: #{sha}"
+      end
+
+      def sentry_extra_data
+        {
+          sha: sha,
+          issue_id: issue_id,
+          design_ids: actions.map { |a| a.design.id }
+        }
+      end
+    end
+
+    belongs_to :issue
+    belongs_to :author, class_name: 'User'
+    has_many :actions
+    has_many :designs,
+             through: :actions,
+             class_name: "DesignManagement::Design",
+             source: :design,
+             inverse_of: :versions
+
+    validates :designs, presence: true, unless: :importing?
+    validates :sha, presence: true
+    validates :sha, uniqueness: { case_sensitive: false, scope: :issue_id }
+    validates :author, presence: true
+    # We are not validating the issue object as it incurs an extra query to fetch
+    # the record from the DB. Instead, we rely on the foreign key constraint to
+    # ensure referential integrity.
+    validates :issue_id, presence: true, unless: :importing?
+
+    sha_attribute :sha
+
+    delegate :project, to: :issue
+
+    scope :for_designs, -> (designs) do
+      where(id: ::DesignManagement::Action.where(design_id: designs).select(:version_id)).distinct
+    end
+    scope :earlier_or_equal_to, -> (version) { where("(#{table_name}.id) <= ?", version) } # rubocop:disable GitlabSecurity/SqlInjection
+    scope :ordered, -> { order(id: :desc) }
+    scope :for_issue, -> (issue) { where(issue: issue) }
+    scope :by_sha, -> (sha) { where(sha: sha) }
+
+    # This is the one true way to create a Version.
+    #
+    # This method means you can avoid the paradox of versions being invalid without
+    # designs, and not being able to add designs without a saved version. Also this
+    # method inserts designs in bulk, rather than one by one.
+    #
+    # Before calling this method, callers must guard against concurrent
+    # modification by obtaining the lock on the design repository. See:
+    # `DesignManagement::Version.with_lock`.
+    #
+    # Parameters:
+    # - design_actions [DesignManagement::DesignAction]:
+    #     the actions that have been performed in the repository.
+    # - sha [String]:
+    #     the SHA of the commit that performed them
+    # - author [User]:
+    #     the user who performed the commit
+    # returns [DesignManagement::Version]
+    def self.create_for_designs(design_actions, sha, author)
+      issue_id, not_uniq = design_actions.map(&:issue_id).compact.uniq
+      raise NotSameIssue, 'All designs must belong to the same issue!' if not_uniq
+
+      transaction do
+        version = new(sha: sha, issue_id: issue_id, author: author)
+        version.save(validate: false) # We need it to have an ID. Validate later when designs are present
+
+        rows = design_actions.map { |action| action.row_attrs(version) }
+
+        Gitlab::Database.bulk_insert(::DesignManagement::Action.table_name, rows)
+        version.designs.reset
+        version.validate!
+        design_actions.each(&:performed)
+
+        version
+      end
+    rescue
+      raise CouldNotCreateVersion.new(sha, issue_id, design_actions)
+    end
+
+    CREATION_TTL = 5.seconds
+    RETRY_DELAY = ->(num) { 0.2.seconds * num**2 }
+
+    def self.with_lock(project_id, repository, &block)
+      key = "with_lock:#{name}:{#{project_id}}"
+
+      in_lock(key, ttl: CREATION_TTL, retries: 5, sleep_sec: RETRY_DELAY) do |_retried|
+        repository.create_if_not_exists
+        yield
+      end
+    end
+
+    def designs_by_event
+      actions
+        .includes(:design)
+        .group_by(&:event)
+        .transform_values { |group| group.map(&:design) }
+    end
+
+    def author
+      super || (commit_author if persisted?)
+    end
+
+    def diff_refs
+      strong_memoize(:diff_refs) { commit&.diff_refs }
+    end
+
+    def reset
+      %i[diff_refs commit].each { |k| clear_memoization(k) }
+      super
+    end
+
+    private
+
+    def commit_author
+      commit&.author
+    end
+
+    def commit
+      strong_memoize(:commit) { issue.project.design_repository.commit(sha) }
+    end
+  end
+end