Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42

9 files changed, 684 insertions, 0 deletions

diff --git a/lib/gitlab/changelog/ast.rb b/lib/gitlab/changelog/ast.rb
new file mode 100644
index 00000000000..2c787d396f5
--- /dev/null
+++ b/lib/gitlab/changelog/ast.rb
@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # AST nodes to evaluate when rendering a template.
+    #
+    # Evaluating an AST is done by walking over the nodes and calling
+    # `evaluate`. This method takes two arguments:
+    #
+    # 1. An instance of `EvalState`, used for tracking data such as the number
+    #    of nested loops.
+    # 2. An object used as the data for the current scope. This can be an Array,
+    #    Hash, String, or something else. It's up to the AST node to determine
+    #    what to do with it.
+    #
+    # While tree walking interpreters (such as implemented here) aren't usually
+    # the fastest type of interpreter, they are:
+    #
+    # 1. Fast enough for our use case
+    # 2. Easy to implement and maintain
+    #
+    # In addition, our AST interpreter doesn't allow for arbitrary code
+    # execution, unlike existing template engines such as Mustache
+    # (https://github.com/mustache/mustache/issues/244) or ERB.
+    #
+    # Our interpreter also takes care of limiting the number of nested loops.
+    # And unlike Liquid, our interpreter is much smaller and thus has a smaller
+    # attack surface. Liquid isn't without its share of issues, such as
+    # https://github.com/Shopify/liquid/pull/1071.
+    #
+    # We also evaluated using Handlebars using the project
+    # https://github.com/SmartBear/ruby-handlebars. Sadly, this implementation
+    # of Handlebars doesn't support control of whitespace
+    # (https://github.com/SmartBear/ruby-handlebars/issues/37), and the project
+    # didn't appear to be maintained that much.
+    #
+    # This doesn't mean these template engines aren't good, instead it means
+    # they won't work for our use case. For more information, refer to the
+    # comment https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50063#note_469293322.
+    module AST
+      # An identifier in a selector.
+      Identifier = Struct.new(:name) do
+        def evaluate(state, data)
+          return data if name == 'it'
+
+          data[name] if data.is_a?(Hash)
+        end
+      end
+
+      # An integer used in a selector.
+      Integer = Struct.new(:value) do
+        def evaluate(state, data)
+          data[value] if data.is_a?(Array)
+        end
+      end
+
+      # A selector used for loading a value.
+      Selector = Struct.new(:steps) do
+        def evaluate(state, data)
+          steps.reduce(data) do |current, step|
+            break if current.nil?
+
+            step.evaluate(state, current)
+          end
+        end
+      end
+
+      # A tag used for displaying a value in the output.
+      Variable = Struct.new(:selector) do
+        def evaluate(state, data)
+          selector.evaluate(state, data).to_s
+        end
+      end
+
+      # A collection of zero or more expressions.
+      Expressions = Struct.new(:nodes) do
+        def evaluate(state, data)
+          nodes.map { |node| node.evaluate(state, data) }.join('')
+        end
+      end
+
+      # A single text node.
+      Text = Struct.new(:text) do
+        def evaluate(*)
+          text
+        end
+      end
+
+      # An `if` expression, with an optional `else` clause.
+      If = Struct.new(:condition, :true_body, :false_body) do
+        def evaluate(state, data)
+          result =
+            if truthy?(condition.evaluate(state, data))
+              true_body.evaluate(state, data)
+            elsif false_body
+              false_body.evaluate(state, data)
+            end
+
+          result.to_s
+        end
+
+        def truthy?(value)
+          # We treat empty collections and such as false, removing the need for
+          # some sort of `if length(x) > 0` expression.
+          value.respond_to?(:empty?) ? !value.empty? : !!value
+        end
+      end
+
+      # An `each` expression.
+      Each = Struct.new(:collection, :body) do
+        def evaluate(state, data)
+          values = collection.evaluate(state, data)
+
+          return '' unless values.respond_to?(:each)
+
+          # While unlikely to happen, it's possible users attempt to nest many
+          # loops in order to negatively impact the GitLab instance. To make
+          # this more difficult, we limit the number of nested loops a user can
+          # create.
+          state.enter_loop do
+            values.map { |value| body.evaluate(state, value) }.join('')
+          end
+        end
+      end
+
+      # A class for transforming a raw Parslet AST into a more structured/easier
+      # to work with AST.
+      #
+      # For more information about Parslet transformations, refer to the
+      # documentation at http://kschiess.github.io/parslet/transform.html.
+      class Transformer < Parslet::Transform
+        rule(ident: simple(:name)) { Identifier.new(name.to_s) }
+        rule(int: simple(:name)) { Integer.new(name.to_i) }
+        rule(text: simple(:text)) { Text.new(text.to_s) }
+        rule(exprs: subtree(:nodes)) { Expressions.new(nodes) }
+        rule(selector: sequence(:steps)) { Selector.new(steps) }
+        rule(selector: simple(:step)) { Selector.new([step]) }
+        rule(variable: simple(:selector)) { Variable.new(selector) }
+        rule(each: simple(:values), body: simple(:body)) do
+          Each.new(values, body)
+        end
+
+        rule(if: simple(:cond), true_body: simple(:true_body)) do
+          If.new(cond, true_body)
+        end
+
+        rule(
+          if: simple(:cond),
+          true_body: simple(:true_body),
+          false_body: simple(:false_body)
+        ) do
+          If.new(cond, true_body, false_body)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/committer.rb b/lib/gitlab/changelog/committer.rb
new file mode 100644
index 00000000000..31661650eff
--- /dev/null
+++ b/lib/gitlab/changelog/committer.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # A class used for committing a release's changelog to a Git repository.
+    class Committer
+      def initialize(project, user)
+        @project = project
+        @user = user
+      end
+
+      # Commits a release's changelog to a file on a branch.
+      #
+      # The `release` argument is a `Gitlab::Changelog::Release` for which to
+      # update the changelog.
+      #
+      # The `file` argument specifies the path to commit the changes to.
+      #
+      # The `branch` argument specifies the branch to commit the changes on.
+      #
+      # The `message` argument specifies the commit message to use.
+      def commit(release:, file:, branch:, message:)
+        # When retrying, we need to reprocess the existing changelog from
+        # scratch, otherwise we may end up throwing away changes. As such, all
+        # the logic is contained within the retry block.
+        Retriable.retriable(on: Error) do
+          commit = Gitlab::Git::Commit.last_for_path(
+            @project.repository,
+            branch,
+            file,
+            literal_pathspec: true
+          )
+
+          content = blob_content(file, commit)
+
+          # If the release has already been added (e.g. concurrently by another
+          # API call), we don't want to add it again.
+          break if content&.match?(release.header_start_pattern)
+
+          service = Files::MultiService.new(
+            @project,
+            @user,
+            commit_message: message,
+            branch_name: branch,
+            start_branch: branch,
+            actions: [
+              {
+                action: content ? 'update' : 'create',
+                content: Generator.new(content.to_s).add(release),
+                file_path: file,
+                last_commit_id: commit&.sha
+              }
+            ]
+          )
+
+          result = service.execute
+
+          raise Error.new(result[:message]) if result[:status] != :success
+        end
+      end
+
+      def blob_content(file, commit = nil)
+        return unless commit
+
+        @project.repository.blob_at(commit.sha, file)&.data
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb
new file mode 100644
index 00000000000..105050936ce
--- /dev/null
+++ b/lib/gitlab/changelog/config.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # Configuration settings used when generating changelogs.
+    class Config
+      # When rendering changelog entries, authors are not included.
+      AUTHORS_NONE = 'none'
+
+      # The path to the configuration file as stored in the project's Git
+      # repository.
+      FILE_PATH = '.gitlab/changelog_config.yml'
+
+      # The default date format to use for formatting release dates.
+      DEFAULT_DATE_FORMAT = '%Y-%m-%d'
+
+      # The default template to use for generating release sections.
+      DEFAULT_TEMPLATE = File.read(File.join(__dir__, 'template.tpl'))
+
+      attr_accessor :date_format, :categories, :template
+
+      def self.from_git(project)
+        if (yaml = project.repository.changelog_config)
+          from_hash(project, YAML.safe_load(yaml))
+        else
+          new(project)
+        end
+      end
+
+      def self.from_hash(project, hash)
+        config = new(project)
+
+        if (date = hash['date_format'])
+          config.date_format = date
+        end
+
+        if (template = hash['template'])
+          config.template = Parser.new.parse_and_transform(template)
+        end
+
+        if (categories = hash['categories'])
+          if categories.is_a?(Hash)
+            config.categories = categories
+          else
+            raise Error, 'The "categories" configuration key must be a Hash'
+          end
+        end
+
+        config
+      end
+
+      def initialize(project)
+        @project = project
+        @date_format = DEFAULT_DATE_FORMAT
+        @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE)
+        @categories = {}
+      end
+
+      def contributor?(user)
+        @project.team.contributor?(user)
+      end
+
+      def category(name)
+        @categories[name] || name
+      end
+
+      def format_date(date)
+        date.strftime(@date_format)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/error.rb b/lib/gitlab/changelog/error.rb
new file mode 100644
index 00000000000..0bd886fbdb7
--- /dev/null
+++ b/lib/gitlab/changelog/error.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # An error raised when a changelog couldn't be generated.
+    Error = Class.new(StandardError)
+  end
+end
diff --git a/lib/gitlab/changelog/eval_state.rb b/lib/gitlab/changelog/eval_state.rb
new file mode 100644
index 00000000000..a0439df60cf
--- /dev/null
+++ b/lib/gitlab/changelog/eval_state.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # A class for tracking state when evaluating a template
+    class EvalState
+      MAX_LOOPS = 4
+
+      def initialize
+        @loops = 0
+      end
+
+      def enter_loop
+        if @loops == MAX_LOOPS
+          raise Error, "You can only nest up to #{MAX_LOOPS} loops"
+        end
+
+        @loops += 1
+        retval = yield
+        @loops -= 1
+
+        retval
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/generator.rb b/lib/gitlab/changelog/generator.rb
new file mode 100644
index 00000000000..a80ca0728f9
--- /dev/null
+++ b/lib/gitlab/changelog/generator.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # Parsing and generating of Markdown changelogs.
+    class Generator
+      # The regex used to parse a release header.
+      RELEASE_REGEX =
+        /^##\s+(?<version>#{Gitlab::Regex.unbounded_semver_regex})/.freeze
+
+      # The `input` argument must be a `String` containing the existing
+      # changelog Markdown. If no changelog exists, this should be an empty
+      # `String`.
+      def initialize(input = '')
+        @lines = input.lines
+        @locations = {}
+
+        @lines.each_with_index do |line, index|
+          matches = line.match(RELEASE_REGEX)
+
+          next if !matches || !matches[:version]
+
+          @locations[matches[:version]] = index
+        end
+      end
+
+      # Generates the Markdown for the given release and returns the new
+      # changelog Markdown content.
+      #
+      # The `release` argument must be an instance of
+      # `Gitlab::Changelog::Release`.
+      def add(release)
+        versions = [release.version, *@locations.keys]
+
+        VersionSorter.rsort!(versions)
+
+        new_index = versions.index(release.version)
+        new_lines = @lines.dup
+        markdown = release.to_markdown
+
+        if (insert_after = versions[new_index + 1])
+          line_index = @locations[insert_after]
+
+          new_lines.insert(line_index, markdown)
+        else
+          # When adding to the end of the changelog, the previous section only
+          # has a single newline, resulting in the release section title
+          # following it immediately. When this is the case, we insert an extra
+          # empty line to keep the changelog readable in its raw form.
+          new_lines.push("\n") if versions.length > 1
+          new_lines.push(markdown.rstrip)
+          new_lines.push("\n")
+        end
+
+        new_lines.join
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/parser.rb b/lib/gitlab/changelog/parser.rb
new file mode 100644
index 00000000000..a4c8da283cd
--- /dev/null
+++ b/lib/gitlab/changelog/parser.rb
@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # A parser for the template syntax used for generating changelogs.
+    #
+    # As a quick primer on the template syntax, a basic template looks like
+    # this:
+    #
+    #     {% each users %}
+    #     Name: {{name}}
+    #     Age: {{age}}
+    #
+    #     {% if birthday %}
+    #     This user is celebrating their birthday today! Yay!
+    #     {% end %}
+    #     {% end %}
+    #
+    # For more information, refer to the Parslet documentation found at
+    # http://kschiess.github.io/parslet/.
+    class Parser < Parslet::Parser
+      root(:exprs)
+
+      rule(:exprs) do
+        (
+          variable | if_expr | each_expr | escaped | text | newline
+        ).repeat.as(:exprs)
+      end
+
+      rule(:space) { match('[ \\t]') }
+      rule(:whitespace) { match('\s').repeat }
+      rule(:lf) { str("\n") }
+      rule(:newline) { lf.as(:text) }
+
+      # Escaped newlines are ignored, allowing the user to control the
+      # whitespace in the output. All other escape sequences are treated as
+      # literal text.
+      #
+      # For example, this:
+      #
+      #     foo \
+      #     bar
+      #
+      # Is parsed into this:
+      #
+      #     foo bar
+      rule(:escaped) do
+        backslash = str('\\')
+
+        (backslash >> lf).ignore | (backslash >> chars).as(:text)
+      end
+
+      # A sequence of regular characters, with the exception of newlines and
+      # escaped newlines.
+      rule(:chars) do
+        char = match("[^{\\\\\n]")
+
+        # The rules here are such that we do treat single curly braces or
+        # non-opening tags (e.g. `{foo}`) as text, but not opening tags
+        # themselves (e.g. `{{`).
+        (
+          char.repeat(1) | curly_open >> (curly_open | percent).absent?
+        ).repeat(1)
+      end
+
+      rule(:text) { chars.as(:text) }
+
+      # An integer, limited to 10 digits (= a 32 bits integer).
+      #
+      # The size is limited to prevents users from creating integers that are
+      # too large, as this may result in runtime errors.
+      rule(:integer) { match('\d').repeat(1, 10).as(:int) }
+
+      # An identifier to look up in a data structure.
+      #
+      # We only support simple ASCII identifiers as we simply don't have a need
+      # for more complex identifiers (e.g. those containing multibyte
+      # characters).
+      rule(:ident) { match('[a-zA-Z_]').repeat(1).as(:ident) }
+
+      # A selector is used for reading a value, consisting of one or more
+      # "steps".
+      #
+      # Examples:
+      #
+      #     name
+      #     users.0.name
+      #     0
+      #     it
+      rule(:selector) do
+        step = ident | integer
+
+        whitespace >>
+          (step >> (str('.') >> step).repeat).as(:selector) >>
+          whitespace
+      end
+
+      rule(:curly_open) { str('{') }
+      rule(:curly_close) { str('}') }
+      rule(:percent) { str('%') }
+
+      # A variable tag.
+      #
+      # Examples:
+      #
+      #     {{name}}
+      #     {{users.0.name}}
+      rule(:variable) do
+        curly_open.repeat(2) >> selector.as(:variable) >> curly_close.repeat(2)
+      end
+
+      rule(:expr_open) { curly_open >> percent >> whitespace }
+      rule(:expr_close) do
+        # Since whitespace control is important (as Markdown is whitespace
+        # sensitive), we default to stripping a newline that follows a %} tag.
+        # This is less annoying compared to having to opt-in to this behaviour.
+        whitespace >> percent >> curly_close >> lf.maybe.ignore
+      end
+
+      rule(:end_tag) { expr_open >> str('end') >> expr_close }
+
+      # An `if` expression, with an optional `else` clause.
+      #
+      # Examples:
+      #
+      #     {% if foo %}
+      #     yes
+      #     {% end %}
+      #
+      #     {% if foo %}
+      #     yes
+      #     {% else %}
+      #     no
+      #     {% end %}
+      rule(:if_expr) do
+        else_tag =
+          expr_open >> str('else') >> expr_close >> exprs.as(:false_body)
+
+        expr_open >>
+          str('if') >>
+          space.repeat(1) >>
+          selector.as(:if) >>
+          expr_close >>
+          exprs.as(:true_body) >>
+          else_tag.maybe >>
+          end_tag
+      end
+
+      # An `each` expression, used for iterating over collections.
+      #
+      # Example:
+      #
+      #     {% each users %}
+      #     * {{name}}
+      #     {% end %}
+      rule(:each_expr) do
+        expr_open >>
+          str('each') >>
+          space.repeat(1) >>
+          selector.as(:each) >>
+          expr_close >>
+          exprs.as(:body) >>
+          end_tag
+      end
+
+      def parse_and_transform(input)
+        AST::Transformer.new.apply(parse(input))
+      rescue Parslet::ParseFailed => ex
+        # We raise a custom error so it's easier to catch different changelog
+        # related errors. In addition, this ensures the caller of this method
+        # doesn't depend on a Parslet specific error class.
+        raise Error.new("Failed to parse the template: #{ex.message}")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/release.rb b/lib/gitlab/changelog/release.rb
new file mode 100644
index 00000000000..f2a01c2b0dc
--- /dev/null
+++ b/lib/gitlab/changelog/release.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # A release to add to a changelog.
+    class Release
+      attr_reader :version
+
+      def initialize(version:, date:, config:)
+        @version = version
+        @date = date
+        @config = config
+        @entries = Hash.new { |h, k| h[k] = [] }
+
+        # This ensures that entries are presented in the same order as the
+        # categories Hash in the user's configuration.
+        @config.categories.values.each do |category|
+          @entries[category] = []
+        end
+      end
+
+      def add_entry(
+        title:,
+        commit:,
+        category:,
+        author: nil,
+        merge_request: nil
+      )
+        # When changing these fields, keep in mind that this needs to be
+        # backwards compatible. For example, you can't just remove a field as
+        # this will break the changelog generation process for existing users.
+        entry = {
+          'title' => title,
+          'commit' => {
+            'reference' => commit.to_reference(full: true),
+            'trailers' => commit.trailers
+          }
+        }
+
+        if author
+          entry['author'] = {
+            'reference' => author.to_reference(full: true),
+            'contributor' => @config.contributor?(author)
+          }
+        end
+
+        if merge_request
+          entry['merge_request'] = {
+            'reference' => merge_request.to_reference(full: true)
+          }
+        end
+
+        @entries[@config.category(category)] << entry
+      end
+
+      def to_markdown
+        state = EvalState.new
+        data = { 'categories' => entries_for_template }
+
+        # While not critical, we would like release sections to be separated by
+        # an empty line in the changelog; ensuring it's readable even in its
+        # raw form.
+        #
+        # Since it can be a bit tricky to get this right in a template, we
+        # enforce an empty line separator ourselves.
+        markdown = @config.template.evaluate(state, data).strip
+
+        # The release header can't be changed using the Liquid template, as we
+        # need this to be in a known format. Without this restriction, we won't
+        # know where to insert a new release section in an existing changelog.
+        "## #{@version} (#{release_date})\n\n#{markdown}\n\n"
+      end
+
+      def header_start_pattern
+        /^##\s*#{Regexp.escape(@version)}/
+      end
+
+      private
+
+      def release_date
+        @config.format_date(@date)
+      end
+
+      def entries_for_template
+        rows = []
+
+        @entries.each do |category, entries|
+          next if entries.empty?
+
+          rows << {
+            'title' => category,
+            'count' => entries.length,
+            'single_change' => entries.length == 1,
+            'entries' => entries
+          }
+        end
+
+        rows
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/template.tpl b/lib/gitlab/changelog/template.tpl
new file mode 100644
index 00000000000..584939dff51
--- /dev/null
+++ b/lib/gitlab/changelog/template.tpl
@@ -0,0 +1,15 @@
+{% if categories %}
+{% each categories %}
+### {{ title }} ({% if single_change %}1 change{% else %}{{ count }} changes{% end %})
+
+{% each entries %}
+- [{{ title }}]({{ commit.reference }})\
+{% if author.contributor %} by {{ author.reference }}{% end %}\
+{% if merge_request %} ([merge request]({{ merge_request.reference }})){% end %}
+
+{% end %}
+
+{% end %}
+{% else %}
+No changes.
+{% end %}