Add latest changes from gitlab-org/gitlab@master

19 files changed, 474 insertions, 290 deletions

diff --git a/lib/api/project_templates.rb b/lib/api/project_templates.rb
index 19244ed697f..f8b5ac65323 100644
--- a/lib/api/project_templates.rb
+++ b/lib/api/project_templates.rb
@@ -26,9 +26,7 @@ module API
         use :pagination
       end
       get ':id/templates/:type' do
-        templates = TemplateFinder
-          .build(params[:type], user_project)
-          .execute
+        templates = TemplateFinder.all_template_names_array(user_project, params[:type])
 
         present paginate(::Kaminari.paginate_array(templates)), with: Entities::TemplatesList
       end
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 353f2ed1c25..d0e81a947d9 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -228,7 +228,7 @@ module API
 
         service.execute
         status(200)
-      rescue => ex
+      rescue Gitlab::Changelog::Error => ex
         render_api_error!("Failed to generate the changelog: #{ex.message}", 500)
       end
     end
diff --git a/lib/gitlab/changelog/ast.rb b/lib/gitlab/changelog/ast.rb
new file mode 100644
index 00000000000..2c787d396f5
--- /dev/null
+++ b/lib/gitlab/changelog/ast.rb
@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # AST nodes to evaluate when rendering a template.
+    #
+    # Evaluating an AST is done by walking over the nodes and calling
+    # `evaluate`. This method takes two arguments:
+    #
+    # 1. An instance of `EvalState`, used for tracking data such as the number
+    #    of nested loops.
+    # 2. An object used as the data for the current scope. This can be an Array,
+    #    Hash, String, or something else. It's up to the AST node to determine
+    #    what to do with it.
+    #
+    # While tree walking interpreters (such as implemented here) aren't usually
+    # the fastest type of interpreter, they are:
+    #
+    # 1. Fast enough for our use case
+    # 2. Easy to implement and maintain
+    #
+    # In addition, our AST interpreter doesn't allow for arbitrary code
+    # execution, unlike existing template engines such as Mustache
+    # (https://github.com/mustache/mustache/issues/244) or ERB.
+    #
+    # Our interpreter also takes care of limiting the number of nested loops.
+    # And unlike Liquid, our interpreter is much smaller and thus has a smaller
+    # attack surface. Liquid isn't without its share of issues, such as
+    # https://github.com/Shopify/liquid/pull/1071.
+    #
+    # We also evaluated using Handlebars using the project
+    # https://github.com/SmartBear/ruby-handlebars. Sadly, this implementation
+    # of Handlebars doesn't support control of whitespace
+    # (https://github.com/SmartBear/ruby-handlebars/issues/37), and the project
+    # didn't appear to be maintained that much.
+    #
+    # This doesn't mean these template engines aren't good, instead it means
+    # they won't work for our use case. For more information, refer to the
+    # comment https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50063#note_469293322.
+    module AST
+      # An identifier in a selector.
+      Identifier = Struct.new(:name) do
+        def evaluate(state, data)
+          return data if name == 'it'
+
+          data[name] if data.is_a?(Hash)
+        end
+      end
+
+      # An integer used in a selector.
+      Integer = Struct.new(:value) do
+        def evaluate(state, data)
+          data[value] if data.is_a?(Array)
+        end
+      end
+
+      # A selector used for loading a value.
+      Selector = Struct.new(:steps) do
+        def evaluate(state, data)
+          steps.reduce(data) do |current, step|
+            break if current.nil?
+
+            step.evaluate(state, current)
+          end
+        end
+      end
+
+      # A tag used for displaying a value in the output.
+      Variable = Struct.new(:selector) do
+        def evaluate(state, data)
+          selector.evaluate(state, data).to_s
+        end
+      end
+
+      # A collection of zero or more expressions.
+      Expressions = Struct.new(:nodes) do
+        def evaluate(state, data)
+          nodes.map { |node| node.evaluate(state, data) }.join('')
+        end
+      end
+
+      # A single text node.
+      Text = Struct.new(:text) do
+        def evaluate(*)
+          text
+        end
+      end
+
+      # An `if` expression, with an optional `else` clause.
+      If = Struct.new(:condition, :true_body, :false_body) do
+        def evaluate(state, data)
+          result =
+            if truthy?(condition.evaluate(state, data))
+              true_body.evaluate(state, data)
+            elsif false_body
+              false_body.evaluate(state, data)
+            end
+
+          result.to_s
+        end
+
+        def truthy?(value)
+          # We treat empty collections and such as false, removing the need for
+          # some sort of `if length(x) > 0` expression.
+          value.respond_to?(:empty?) ? !value.empty? : !!value
+        end
+      end
+
+      # An `each` expression.
+      Each = Struct.new(:collection, :body) do
+        def evaluate(state, data)
+          values = collection.evaluate(state, data)
+
+          return '' unless values.respond_to?(:each)
+
+          # While unlikely to happen, it's possible users attempt to nest many
+          # loops in order to negatively impact the GitLab instance. To make
+          # this more difficult, we limit the number of nested loops a user can
+          # create.
+          state.enter_loop do
+            values.map { |value| body.evaluate(state, value) }.join('')
+          end
+        end
+      end
+
+      # A class for transforming a raw Parslet AST into a more structured/easier
+      # to work with AST.
+      #
+      # For more information about Parslet transformations, refer to the
+      # documentation at http://kschiess.github.io/parslet/transform.html.
+      class Transformer < Parslet::Transform
+        rule(ident: simple(:name)) { Identifier.new(name.to_s) }
+        rule(int: simple(:name)) { Integer.new(name.to_i) }
+        rule(text: simple(:text)) { Text.new(text.to_s) }
+        rule(exprs: subtree(:nodes)) { Expressions.new(nodes) }
+        rule(selector: sequence(:steps)) { Selector.new(steps) }
+        rule(selector: simple(:step)) { Selector.new([step]) }
+        rule(variable: simple(:selector)) { Variable.new(selector) }
+        rule(each: simple(:values), body: simple(:body)) do
+          Each.new(values, body)
+        end
+
+        rule(if: simple(:cond), true_body: simple(:true_body)) do
+          If.new(cond, true_body)
+        end
+
+        rule(
+          if: simple(:cond),
+          true_body: simple(:true_body),
+          false_body: simple(:false_body)
+        ) do
+          If.new(cond, true_body, false_body)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/committer.rb b/lib/gitlab/changelog/committer.rb
index 617017faa58..31661650eff 100644
--- a/lib/gitlab/changelog/committer.rb
+++ b/lib/gitlab/changelog/committer.rb
@@ -4,8 +4,6 @@ module Gitlab
   module Changelog
     # A class used for committing a release's changelog to a Git repository.
     class Committer
-      CommitError = Class.new(StandardError)
-
       def initialize(project, user)
         @project = project
         @user = user
@@ -25,7 +23,7 @@ module Gitlab
         # When retrying, we need to reprocess the existing changelog from
         # scratch, otherwise we may end up throwing away changes. As such, all
         # the logic is contained within the retry block.
-        Retriable.retriable(on: CommitError) do
+        Retriable.retriable(on: Error) do
           commit = Gitlab::Git::Commit.last_for_path(
             @project.repository,
             branch,
@@ -57,7 +55,7 @@ module Gitlab
 
           result = service.execute
 
-          raise CommitError.new(result[:message]) if result[:status] != :success
+          raise Error.new(result[:message]) if result[:status] != :success
         end
       end
 
diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb
index 3f06b612687..105050936ce 100644
--- a/lib/gitlab/changelog/config.rb
+++ b/lib/gitlab/changelog/config.rb
@@ -4,8 +4,6 @@ module Gitlab
   module Changelog
     # Configuration settings used when generating changelogs.
     class Config
-      ConfigError = Class.new(StandardError)
-
       # When rendering changelog entries, authors are not included.
       AUTHORS_NONE = 'none'
 
@@ -37,17 +35,14 @@ module Gitlab
         end
 
         if (template = hash['template'])
-          # We use the full namespace here (and further down) as otherwise Rails
-          # may use the wrong constant when autoloading is used.
-          config.template =
-            ::Gitlab::Changelog::Template::Compiler.new.compile(template)
+          config.template = Parser.new.parse_and_transform(template)
         end
 
         if (categories = hash['categories'])
           if categories.is_a?(Hash)
             config.categories = categories
           else
-            raise ConfigError, 'The "categories" configuration key must be a Hash'
+            raise Error, 'The "categories" configuration key must be a Hash'
           end
         end
 
@@ -57,8 +52,7 @@ module Gitlab
       def initialize(project)
         @project = project
         @date_format = DEFAULT_DATE_FORMAT
-        @template =
-          ::Gitlab::Changelog::Template::Compiler.new.compile(DEFAULT_TEMPLATE)
+        @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE)
         @categories = {}
       end
 
diff --git a/lib/gitlab/changelog/error.rb b/lib/gitlab/changelog/error.rb
new file mode 100644
index 00000000000..0bd886fbdb7
--- /dev/null
+++ b/lib/gitlab/changelog/error.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # An error raised when a changelog couldn't be generated.
+    Error = Class.new(StandardError)
+  end
+end
diff --git a/lib/gitlab/changelog/eval_state.rb b/lib/gitlab/changelog/eval_state.rb
new file mode 100644
index 00000000000..a0439df60cf
--- /dev/null
+++ b/lib/gitlab/changelog/eval_state.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # A class for tracking state when evaluating a template
+    class EvalState
+      MAX_LOOPS = 4
+
+      def initialize
+        @loops = 0
+      end
+
+      def enter_loop
+        if @loops == MAX_LOOPS
+          raise Error, "You can only nest up to #{MAX_LOOPS} loops"
+        end
+
+        @loops += 1
+        retval = yield
+        @loops -= 1
+
+        retval
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/parser.rb b/lib/gitlab/changelog/parser.rb
new file mode 100644
index 00000000000..a4c8da283cd
--- /dev/null
+++ b/lib/gitlab/changelog/parser.rb
@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Changelog
+    # A parser for the template syntax used for generating changelogs.
+    #
+    # As a quick primer on the template syntax, a basic template looks like
+    # this:
+    #
+    #     {% each users %}
+    #     Name: {{name}}
+    #     Age: {{age}}
+    #
+    #     {% if birthday %}
+    #     This user is celebrating their birthday today! Yay!
+    #     {% end %}
+    #     {% end %}
+    #
+    # For more information, refer to the Parslet documentation found at
+    # http://kschiess.github.io/parslet/.
+    class Parser < Parslet::Parser
+      root(:exprs)
+
+      rule(:exprs) do
+        (
+          variable | if_expr | each_expr | escaped | text | newline
+        ).repeat.as(:exprs)
+      end
+
+      rule(:space) { match('[ \\t]') }
+      rule(:whitespace) { match('\s').repeat }
+      rule(:lf) { str("\n") }
+      rule(:newline) { lf.as(:text) }
+
+      # Escaped newlines are ignored, allowing the user to control the
+      # whitespace in the output. All other escape sequences are treated as
+      # literal text.
+      #
+      # For example, this:
+      #
+      #     foo \
+      #     bar
+      #
+      # Is parsed into this:
+      #
+      #     foo bar
+      rule(:escaped) do
+        backslash = str('\\')
+
+        (backslash >> lf).ignore | (backslash >> chars).as(:text)
+      end
+
+      # A sequence of regular characters, with the exception of newlines and
+      # escaped newlines.
+      rule(:chars) do
+        char = match("[^{\\\\\n]")
+
+        # The rules here are such that we do treat single curly braces or
+        # non-opening tags (e.g. `{foo}`) as text, but not opening tags
+        # themselves (e.g. `{{`).
+        (
+          char.repeat(1) | curly_open >> (curly_open | percent).absent?
+        ).repeat(1)
+      end
+
+      rule(:text) { chars.as(:text) }
+
+      # An integer, limited to 10 digits (= a 32 bits integer).
+      #
+      # The size is limited to prevents users from creating integers that are
+      # too large, as this may result in runtime errors.
+      rule(:integer) { match('\d').repeat(1, 10).as(:int) }
+
+      # An identifier to look up in a data structure.
+      #
+      # We only support simple ASCII identifiers as we simply don't have a need
+      # for more complex identifiers (e.g. those containing multibyte
+      # characters).
+      rule(:ident) { match('[a-zA-Z_]').repeat(1).as(:ident) }
+
+      # A selector is used for reading a value, consisting of one or more
+      # "steps".
+      #
+      # Examples:
+      #
+      #     name
+      #     users.0.name
+      #     0
+      #     it
+      rule(:selector) do
+        step = ident | integer
+
+        whitespace >>
+          (step >> (str('.') >> step).repeat).as(:selector) >>
+          whitespace
+      end
+
+      rule(:curly_open) { str('{') }
+      rule(:curly_close) { str('}') }
+      rule(:percent) { str('%') }
+
+      # A variable tag.
+      #
+      # Examples:
+      #
+      #     {{name}}
+      #     {{users.0.name}}
+      rule(:variable) do
+        curly_open.repeat(2) >> selector.as(:variable) >> curly_close.repeat(2)
+      end
+
+      rule(:expr_open) { curly_open >> percent >> whitespace }
+      rule(:expr_close) do
+        # Since whitespace control is important (as Markdown is whitespace
+        # sensitive), we default to stripping a newline that follows a %} tag.
+        # This is less annoying compared to having to opt-in to this behaviour.
+        whitespace >> percent >> curly_close >> lf.maybe.ignore
+      end
+
+      rule(:end_tag) { expr_open >> str('end') >> expr_close }
+
+      # An `if` expression, with an optional `else` clause.
+      #
+      # Examples:
+      #
+      #     {% if foo %}
+      #     yes
+      #     {% end %}
+      #
+      #     {% if foo %}
+      #     yes
+      #     {% else %}
+      #     no
+      #     {% end %}
+      rule(:if_expr) do
+        else_tag =
+          expr_open >> str('else') >> expr_close >> exprs.as(:false_body)
+
+        expr_open >>
+          str('if') >>
+          space.repeat(1) >>
+          selector.as(:if) >>
+          expr_close >>
+          exprs.as(:true_body) >>
+          else_tag.maybe >>
+          end_tag
+      end
+
+      # An `each` expression, used for iterating over collections.
+      #
+      # Example:
+      #
+      #     {% each users %}
+      #     * {{name}}
+      #     {% end %}
+      rule(:each_expr) do
+        expr_open >>
+          str('each') >>
+          space.repeat(1) >>
+          selector.as(:each) >>
+          expr_close >>
+          exprs.as(:body) >>
+          end_tag
+      end
+
+      def parse_and_transform(input)
+        AST::Transformer.new.apply(parse(input))
+      rescue Parslet::ParseFailed => ex
+        # We raise a custom error so it's easier to catch different changelog
+        # related errors. In addition, this ensures the caller of this method
+        # doesn't depend on a Parslet specific error class.
+        raise Error.new("Failed to parse the template: #{ex.message}")
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/changelog/release.rb b/lib/gitlab/changelog/release.rb
index 4c78eb5080c..f2a01c2b0dc 100644
--- a/lib/gitlab/changelog/release.rb
+++ b/lib/gitlab/changelog/release.rb
@@ -54,14 +54,16 @@ module Gitlab
       end
 
       def to_markdown
+        state = EvalState.new
+        data = { 'categories' => entries_for_template }
+
         # While not critical, we would like release sections to be separated by
         # an empty line in the changelog; ensuring it's readable even in its
         # raw form.
         #
-        # Since it can be a bit tricky to get this right using Liquid, we
+        # Since it can be a bit tricky to get this right in a template, we
         # enforce an empty line separator ourselves.
-        markdown =
-          @config.template.render('categories' => entries_for_template).strip
+        markdown = @config.template.evaluate(state, data).strip
 
         # The release header can't be changed using the Liquid template, as we
         # need this to be in a known format. Without this restriction, we won't
@@ -80,14 +82,20 @@ module Gitlab
       end
 
       def entries_for_template
-        @entries.map do |category, entries|
-          {
+        rows = []
+
+        @entries.each do |category, entries|
+          next if entries.empty?
+
+          rows << {
             'title' => category,
             'count' => entries.length,
             'single_change' => entries.length == 1,
             'entries' => entries
           }
         end
+
+        rows
       end
     end
   end
diff --git a/lib/gitlab/changelog/template.tpl b/lib/gitlab/changelog/template.tpl
index 838b7080f68..584939dff51 100644
--- a/lib/gitlab/changelog/template.tpl
+++ b/lib/gitlab/changelog/template.tpl
@@ -6,6 +6,7 @@
 - [{{ title }}]({{ commit.reference }})\
 {% if author.contributor %} by {{ author.reference }}{% end %}\
 {% if merge_request %} ([merge request]({{ merge_request.reference }})){% end %}
+
 {% end %}
 
 {% end %}
diff --git a/lib/gitlab/changelog/template/compiler.rb b/lib/gitlab/changelog/template/compiler.rb
deleted file mode 100644
index fa7724aa2da..00000000000
--- a/lib/gitlab/changelog/template/compiler.rb
+++ /dev/null
@@ -1,154 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Changelog
-    module Template
-      # Compiler is used for turning a minimal user templating language into an
-      # ERB template, without giving the user access to run arbitrary code.
-      #
-      # The template syntax is deliberately made as minimal as possible, and
-      # only supports the following:
-      #
-      # * Printing a value
-      # * Iterating over collections
-      # * if/else
-      #
-      # The syntax looks as follows:
-      #
-      #     {% each users %}
-      #
-      #     Name: {{user}}
-      #     Likes cats: {% if likes_cats %}yes{% else %}no{% end %}
-      #
-      #     {% end %}
-      #
-      # Newlines can be escaped by ending a line with a backslash. So this:
-      #
-      #     foo \
-      #     bar
-      #
-      # Is the same as this:
-      #
-      #     foo bar
-      #
-      # Templates are compiled into ERB templates, while taking care to make
-      # sure the user can't run arbitrary code. By using ERB we can let it do
-      # the heavy lifting of rendering data; all we need to provide is a
-      # translation layer.
-      #
-      # # Security
-      #
-      # The template syntax this compiler exposes is safe to be used by
-      # untrusted users. Not only are they unable to run arbitrary code, the
-      # compiler also enforces a limit on the integer sizes and the number of
-      # nested loops. ERB tags added by the user are also disabled.
-      class Compiler
-        # A pattern to match a single integer, with an upper size limit.
-        #
-        # We enforce a limit of 10 digits (= a 32 bits integer) so users can't
-        # trigger the allocation of infinitely large bignums, or trigger
-        # RangeError errors when using such integers to access an array value.
-        INTEGER = /^\d{1,10}$/.freeze
-
-        # The name/path of a variable, such as `user.address.city`.
-        #
-        # It's important that this regular expression _doesn't_ allow for
-        # anything but letters, numbers, and underscores, otherwise a user may
-        # use those to "escape" our template and run arbirtary Ruby code. For
-        # example, take this variable:
-        #
-        #     {{') ::Kernel.exit #'}}
-        #
-        # This would then be compiled into:
-        #
-        #     <%= read(variables, '') ::Kernel.exit #'') %>
-        #
-        # Restricting the allowed characters makes this impossible.
-        VAR_NAME = /([\w\.]+)/.freeze
-
-        # A variable tag, such as `{{username}}`.
-        VAR = /{{ \s* #{VAR_NAME} \s* }}/x.freeze
-
-        # The opening tag for a statement.
-        STM_START = /{% \s*/x.freeze
-
-        # The closing tag for a statement.
-        STM_END = /\s* %}/x.freeze
-
-        # A regular `end` closing tag.
-        NORMAL_END = /#{STM_START} end #{STM_END}/x.freeze
-
-        # An `end` closing tag on its own line, without any non-whitespace
-        # preceding or following it.
-        #
-        # These tags need some special care to make it easier to control
-        # whitespace.
-        LONELY_END = /^\s*#{NORMAL_END}\s$/x.freeze
-
-        # An `else` tag.
-        ELSE = /#{STM_START} else #{STM_END}/x.freeze
-
-        # The start of an `each` tag.
-        EACH = /#{STM_START} each \s+ #{VAR_NAME} #{STM_END}/x.freeze
-
-        # The start of an `if` tag.
-        IF = /#{STM_START} if \s+ #{VAR_NAME} #{STM_END}/x.freeze
-
-        # The pattern to use for escaping newlines.
-        ESCAPED_NEWLINE = /\\\n$/.freeze
-
-        # The start tag for ERB tags. These tags will be escaped, preventing
-        # users from using ERB directly.
-        ERB_START_TAG = /<\\?\s*\\?\s*%/.freeze
-
-        def compile(template)
-          transformed_lines = ['<% it = variables %>']
-
-          # ERB tags must be stripped here, otherwise a user may introduce ERB
-          # tags by making clever use of whitespace. See
-          # https://gitlab.com/gitlab-org/gitlab/-/issues/300224 for more
-          # information.
-          template = template.gsub(ERB_START_TAG, '<%%')
-
-          template.each_line { |line| transformed_lines << transform(line) }
-
-          # We use the full namespace here as otherwise Rails may use the wrong
-          # constant when autoloading is used.
-          ::Gitlab::Changelog::Template::Template.new(transformed_lines.join)
-        end
-
-        def transform(line)
-          line.gsub!(ESCAPED_NEWLINE, '')
-
-          # This replacement ensures that "end" blocks on their own lines
-          # don't add extra newlines. Using an ERB -%> tag sadly swallows too
-          # many newlines.
-          line.gsub!(LONELY_END, '<% end %>')
-          line.gsub!(NORMAL_END, '<% end %>')
-          line.gsub!(ELSE, '<% else -%>')
-
-          line.gsub!(EACH) do
-            # No, `it; variables` isn't a syntax error. Using `;` marks
-            # `variables` as block-local, making it possible to re-assign it
-            # without affecting outer definitions of this variable. We use
-            # this to scope template variables to the right input Hash.
-            "<% each(#{read_path(Regexp.last_match(1))}) do |it; variables| -%><% variables = it -%>"
-          end
-
-          line.gsub!(IF) { "<% if truthy?(#{read_path(Regexp.last_match(1))}) -%>" }
-          line.gsub!(VAR) { "<%= #{read_path(Regexp.last_match(1))} %>" }
-          line
-        end
-
-        def read_path(path)
-          return path if path == 'it'
-
-          args = path.split('.')
-          args.map! { |arg| arg.match?(INTEGER) ? "#{arg}" : "'#{arg}'" }
-
-          "read(variables, #{args.join(', ')})"
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/changelog/template/context.rb b/lib/gitlab/changelog/template/context.rb
deleted file mode 100644
index 8a0796d767e..00000000000
--- a/lib/gitlab/changelog/template/context.rb
+++ /dev/null
@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Changelog
-    module Template
-      # Context is used to provide a binding/context to ERB templates used for
-      # rendering changelogs.
-      #
-      # This class extends BasicObject so that we only expose the bare minimum
-      # needed to render the ERB template.
-      class Context < BasicObject
-        MAX_NESTED_LOOPS = 4
-
-        def initialize(variables)
-          @variables = variables
-          @loop_nesting = 0
-        end
-
-        def get_binding
-          ::Kernel.binding
-        end
-
-        def each(value, &block)
-          max = MAX_NESTED_LOOPS
-
-          if @loop_nesting == max
-            ::Kernel.raise(
-              ::Template::TemplateError.new("You can only nest up to #{max} loops")
-            )
-          end
-
-          @loop_nesting += 1
-          result = value.each(&block) if value.respond_to?(:each)
-          @loop_nesting -= 1
-
-          result
-        end
-
-        # rubocop: disable Style/TrivialAccessors
-        def variables
-          @variables
-        end
-        # rubocop: enable Style/TrivialAccessors
-
-        def read(source, *steps)
-          current = source
-
-          steps.each do |step|
-            case current
-            when ::Hash
-              current = current[step]
-            when ::Array
-              return '' unless step.is_a?(::Integer)
-
-              current = current[step]
-            else
-              break
-            end
-          end
-
-          current
-        end
-
-        def truthy?(value)
-          value.respond_to?(:any?) ? value.any? : !!value
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/changelog/template/template.rb b/lib/gitlab/changelog/template/template.rb
deleted file mode 100644
index 0ff2852d6d4..00000000000
--- a/lib/gitlab/changelog/template/template.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Changelog
-    module Template
-      # A wrapper around an ERB template user for rendering changelogs.
-      class Template
-        TemplateError = Class.new(StandardError)
-
-        def initialize(erb)
-          # Don't change the trim mode, as this may require changes to the
-          # regular expressions used to turn the template syntax into ERB
-          # tags.
-          @erb = ERB.new(erb, trim_mode: '-')
-        end
-
-        def render(data)
-          context = Context.new(data).get_binding
-
-          # ERB produces a SyntaxError when processing templates, as it
-          # internally uses eval() for this.
-          @erb.result(context)
-        rescue SyntaxError
-          raise TemplateError.new("The template's syntax is invalid")
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index c51349b9113..02c80923973 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -226,6 +226,7 @@ module Gitlab
       metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil)
       metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil)
       metadata.merge!(Feature::Gitaly.server_feature_flags)
+      metadata.merge!(route_to_primary)
 
       deadline_info = request_deadline(timeout)
       metadata.merge!(deadline_info.slice(:deadline_type))
@@ -233,6 +234,24 @@ module Gitlab
       { metadata: metadata, deadline: deadline_info[:deadline] }
     end
 
+    # Gitlab::Git::HookEnv will set the :gitlab_git_env variable in case we're
+    # running in the context of a Gitaly hook call, which may make use of
+    # quarantined object directories. We thus need to pass along the path of
+    # the quarantined object directory to Gitaly, otherwise it won't be able to
+    # find these quarantined objects. Given that the quarantine directory is
+    # generated with a random name, they'll have different names when multiple
+    # Gitaly nodes take part in a single transaction. As a result, we are
+    # forced to route all requests to the primary node which has injected the
+    # quarantine object directory to us.
+    def self.route_to_primary
+      return {} unless Gitlab::SafeRequestStore.active?
+
+      return {} unless Gitlab::SafeRequestStore[:gitlab_git_env]
+
+      { 'gitaly-route-repository-accessor-policy' => 'primary-only' }
+    end
+    private_class_method :route_to_primary
+
     def self.request_deadline(timeout)
       # timeout being 0 means the request is allowed to run indefinitely.
       # We can't allow that inside a request, but this won't count towards Gitaly
diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb
index b659bff52ad..02b3da54685 100644
--- a/lib/gitlab/template/base_template.rb
+++ b/lib/gitlab/template/base_template.rb
@@ -95,19 +95,29 @@ module Gitlab
           File.join(base_dir, categories[category])
         end
 
-        # If template is organized by category it returns { category_name: [{ name: template_name }, { name: template2_name }] }
-        # If no category is present returns [{ name: template_name }, { name: template2_name}]
-        def dropdown_names(project = nil)
-          return [] if project && !project.repository.exists?
+        # `repository_template_names` - reads through Gitaly the actual templates names within a
+        # given project's repository. This is only used by issue and merge request templates,
+        # that need to call this once and then cache the returned value.
+        #
+        # `template_names` - is an alias to `repository_template_names`. It would read through
+        # Gitaly the actual template names within a given project's repository for all file templates
+        # other than `issue` and `merge request` description templates, which would instead
+        # overwrite the `template_names` method to return a redis cached version, by reading cached values
+        # from `repository.issue_template_names_by_category` and `repository.merge_request_template_names_by_category`
+        # methods.
+        def repository_template_names(project)
+          template_names_by_category(self.all(project))
+        end
+        alias_method :template_names, :repository_template_names
 
-          if categories.any?
-            categories.keys.map do |category|
-              files = self.by_category(category, project)
-              [category, files.map { |t| { name: t.name } }]
-            end.to_h
-          else
-            files = self.all(project)
-            files.map { |t| { name: t.name } }
+        def template_names_by_category(items)
+          grouped = items.group_by(&:category)
+          categories = grouped.keys
+
+          categories.each_with_object({}) do |category, hash|
+            hash[category] = grouped[category].map do |item|
+              { name: item.name, id: item.key, key: item.key, project_id: item.try(:project_id) }
+            end
           end
         end
 
diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb
index 8e234148a63..9f0ba97bcdf 100644
--- a/lib/gitlab/template/finders/repo_template_finder.rb
+++ b/lib/gitlab/template/finders/repo_template_finder.rb
@@ -11,8 +11,8 @@ module Gitlab
         def initialize(project, base_dir, extension, categories = {})
           @categories     = categories
           @extension      = extension
-          @repository     = project.repository
-          @commit         = @repository.head_commit if @repository.exists?
+          @repository     = project&.repository
+          @commit         = @repository.head_commit if @repository&.exists?
 
           super(base_dir)
         end
@@ -51,7 +51,7 @@ module Gitlab
         private
 
         def select_directory(file_name)
-          return [] unless @commit
+          return unless @commit
 
           # Insert root as directory
           directories = ["", *@categories.keys]
diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb
index 01b191733d4..3049f43b322 100644
--- a/lib/gitlab/template/issue_template.rb
+++ b/lib/gitlab/template/issue_template.rb
@@ -15,6 +15,16 @@ module Gitlab
         def finder(project)
           Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories)
         end
+
+        def template_names(project)
+          return {} unless project&.repository&.exists?
+
+          # here we rely on project.repository caching mechanism. Ideally we would want the template finder to have its
+          # own caching mechanism to avoid the back and forth call jumps between finder and model.
+          #
+          # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279
+          project.repository.issue_template_names_by_category
+        end
       end
     end
   end
diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb
index 357b31cd82e..9442f3b13fb 100644
--- a/lib/gitlab/template/merge_request_template.rb
+++ b/lib/gitlab/template/merge_request_template.rb
@@ -15,6 +15,16 @@ module Gitlab
         def finder(project)
           Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories)
         end
+
+        def template_names(project)
+          return {} unless project&.repository&.exists?
+
+          # here we rely on project.repository caching mechanism. Ideally we would want the template finder to have its
+          # own caching mechanism to avoid the back and forth call jumps between finder and model.
+          #
+          # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279
+          project.repository.merge_request_template_names_by_category
+        end
       end
     end
   end
diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml
new file mode 100644
index 00000000000..3fd02164f74
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml
@@ -0,0 +1,22 @@
+---
+# Ecosystem category
+- name: i_ecosystem_jira_service_close_issue
+  category: ecosystem
+  redis_slot: ecosystem
+  aggregation: weekly
+  feature_flag: usage_data_track_ecosystem_jira_service
+- name: i_ecosystem_jira_service_cross_reference
+  category: ecosystem
+  redis_slot: ecosystem
+  aggregation: weekly
+  feature_flag: usage_data_track_ecosystem_jira_service
+- name: i_ecosystem_jira_service_list_issues
+  category: ecosystem
+  redis_slot: ecosystem
+  aggregation: weekly
+  feature_flag: usage_data_track_ecosystem_jira_service
+- name: i_ecosystem_jira_service_create_issue
+  category: ecosystem
+  redis_slot: ecosystem
+  aggregation: weekly
+  feature_flag: usage_data_track_ecosystem_jira_service