Add latest changes from gitlab-org/gitlab@master

6 files changed, 170 insertions, 33 deletions

diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index f97e28de628..9095aba7340 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -251,21 +251,14 @@ module API
       end
       params do
         requires :id, type: Integer, desc: %q(Job's ID)
+        requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact file to store (generated by Multipart middleware))
         optional :token, type: String, desc: %q(Job's authentication token)
         optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
         optional :artifact_type, type: String, desc: %q(The type of artifact),
                                  default: 'archive', values: Ci::JobArtifact.file_types.keys
         optional :artifact_format, type: String, desc: %q(The format of artifact),
                                    default: 'zip', values: Ci::JobArtifact.file_formats.keys
-        optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
-        optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
-        optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
-        optional 'file.size', type: Integer, desc: %q(real size of file (generated by Workhorse))
-        optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file (generated by Workhorse))
-        optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
-        optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
-        optional 'metadata.size', type: Integer, desc: %q(real size of metadata (generated by Workhorse))
-        optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of metadata (generated by Workhorse))
+        optional :metadata, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact metadata to store (generated by Multipart middleware))
       end
       post '/:id/artifacts' do
         not_allowed! unless Gitlab.config.artifacts.enabled
@@ -274,10 +267,9 @@ module API
         job = authenticate_job!
         forbidden!('Job is not running!') unless job.running?
 
-        artifacts = UploadedFile.from_params(params, :file, JobArtifactUploader.workhorse_local_upload_path)
-        metadata = UploadedFile.from_params(params, :metadata, JobArtifactUploader.workhorse_local_upload_path)
+        artifacts = params[:file]
+        metadata = params[:metadata]
 
-        bad_request!('Missing artifacts file!') unless artifacts
         file_too_large! unless artifacts.size < max_artifacts_size(job)
 
         result = Ci::CreateJobArtifactsService.new(job.project).execute(job, artifacts, params, metadata_file: metadata)
diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb
index 544231adea4..5900e762244 100644
--- a/lib/banzai/filter/front_matter_filter.rb
+++ b/lib/banzai/filter/front_matter_filter.rb
@@ -3,28 +3,11 @@
 module Banzai
   module Filter
     class FrontMatterFilter < HTML::Pipeline::Filter
-      DELIM_LANG = {
-        '---' => 'yaml',
-        '+++' => 'toml',
-        ';;;' => 'json'
-      }.freeze
-
-      DELIM = Regexp.union(DELIM_LANG.keys)
-
-      PATTERN = %r{
-        \A(?:[^\r\n]*coding:[^\r\n]*)?         # optional encoding line
-        \s*
-        ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*)  # opening front matter marker (optional language specifier)
-        \s*
-        ^(?<front_matter>.*?)                  # front matter (not greedy)
-        \s*
-        ^\k<delim>                             # closing front matter marker
-        \s*
-      }mx.freeze
-
       def call
-        html.sub(PATTERN) do |_match|
-          lang = $~[:lang].presence || DELIM_LANG[$~[:delim]]
+        lang_mapping = Gitlab::FrontMatter::DELIM_LANG
+
+        html.sub(Gitlab::FrontMatter::PATTERN) do |_match|
+          lang = $~[:lang].presence || lang_mapping[$~[:delim]]
 
           ["```#{lang}", $~[:front_matter], "```", "\n"].join("\n")
         end
diff --git a/lib/gitlab/front_matter.rb b/lib/gitlab/front_matter.rb
new file mode 100644
index 00000000000..7612bd36aca
--- /dev/null
+++ b/lib/gitlab/front_matter.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module FrontMatter
+    DELIM_LANG = {
+      '---' => 'yaml',
+      '+++' => 'toml',
+      ';;;' => 'json'
+    }.freeze
+
+    DELIM = Regexp.union(DELIM_LANG.keys)
+
+    PATTERN = %r{
+      \A(?:[^\r\n]*coding:[^\r\n]*)?         # optional encoding line
+      \s*
+      ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*)  # opening front matter marker (optional language specifier)
+      \s*
+      ^(?<front_matter>.*?)                  # front matter block content (not greedy)
+      \s*
+      ^(\k<delim> | \.{3})                   # closing front matter marker
+      \s*
+    }mx.freeze
+  end
+end
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index cb4213d23a4..c82c05e7319 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -107,6 +107,7 @@ module Gitlab
           [
             ::FileUploader.root,
             Gitlab.config.uploads.storage_path,
+            JobArtifactUploader.workhorse_upload_path,
             File.join(Rails.root, 'public/uploads/tmp')
           ]
         end
@@ -125,6 +126,8 @@ module Gitlab
         Handler.new(env, message).with_open_files do
           @app.call(env)
         end
+      rescue UploadedFile::InvalidPathError => e
+        [400, { 'Content-Type' => 'text/plain' }, e.message]
       end
     end
   end
diff --git a/lib/gitlab/wiki_pages.rb b/lib/gitlab/wiki_pages.rb
new file mode 100644
index 00000000000..47f9aa1117f
--- /dev/null
+++ b/lib/gitlab/wiki_pages.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module WikiPages
+    # Many common file systems have a limit of 255 bytes for file and
+    # directory names, and while Git and GitLab both support paths exceeding
+    # those limits, the presence of them makes it impossible for users on
+    # those file systems to checkout a wiki repository locally.
+
+    # To avoid this situation, we enforce these limits when editing pages
+    # through the GitLab web interface and API:
+    MAX_TITLE_BYTES = 245 # reserving 10 bytes for the file extension
+    MAX_DIRECTORY_BYTES = 255
+  end
+end
diff --git a/lib/gitlab/wiki_pages/front_matter_parser.rb b/lib/gitlab/wiki_pages/front_matter_parser.rb
new file mode 100644
index 00000000000..45dc6cf7fd1
--- /dev/null
+++ b/lib/gitlab/wiki_pages/front_matter_parser.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module WikiPages
+    class FrontMatterParser
+      FEATURE_FLAG = :wiki_front_matter
+
+      # We limit the maximum length of text we are prepared to parse as YAML, to
+      # avoid exploitations and attempts to consume memory and CPU. We allow for:
+      #  - a title line
+      #  - a "slugs:" line
+      #  - and up to 50 slugs
+      #
+      # This limit does not take comments into account.
+      MAX_SLUGS = 50
+      SLUG_LINE_LENGTH = (4 + Gitlab::WikiPages::MAX_DIRECTORY_BYTES + 1 + Gitlab::WikiPages::MAX_TITLE_BYTES)
+      MAX_FRONT_MATTER_LENGTH = (8 + Gitlab::WikiPages::MAX_TITLE_BYTES) + 7 + (SLUG_LINE_LENGTH * MAX_SLUGS)
+
+      ParseError = Class.new(StandardError)
+
+      class Result
+        attr_reader :front_matter, :content, :reason, :error
+
+        def initialize(content:, front_matter: {}, reason: nil, error: nil)
+          @content      = content
+          @front_matter = front_matter.freeze
+          @reason       = reason
+          @error        = error
+        end
+      end
+
+      # @param [String] wiki_content
+      # @param [FeatureGate] feature_gate The scope for feature availability
+      def initialize(wiki_content, feature_gate)
+        @wiki_content = wiki_content
+        @feature_gate = feature_gate
+      end
+
+      def self.enabled?(gate = nil)
+        Feature.enabled?(FEATURE_FLAG, gate)
+      end
+
+      def parse
+        return empty_result unless enabled? && wiki_content.present?
+        return empty_result(block.error) unless block.valid?
+
+        Result.new(front_matter: block.data, content: strip_front_matter_block)
+      rescue ParseError => error
+        empty_result(:parse_error, error)
+      end
+
+      class Block
+        include Gitlab::Utils::StrongMemoize
+
+        def initialize(delim = nil, lang = '', text = nil)
+          @lang = lang.downcase.presence || Gitlab::FrontMatter::DELIM_LANG[delim]
+          @text = text
+        end
+
+        def data
+          @data ||= YAML.safe_load(text, symbolize_names: true)
+        rescue Psych::DisallowedClass, Psych::SyntaxError => error
+          raise ParseError, error.message
+        end
+
+        def valid?
+          error.nil?
+        end
+
+        def error
+          strong_memoize(:error) { no_match? || too_long? || not_yaml? || not_mapping? }
+        end
+
+        private
+
+        attr_reader :lang, :text
+
+        def no_match?
+          :no_match if text.nil?
+        end
+
+        def not_yaml?
+          :not_yaml if lang != 'yaml'
+        end
+
+        def too_long?
+          :too_long if text.size > MAX_FRONT_MATTER_LENGTH
+        end
+
+        def not_mapping?
+          :not_mapping unless data.is_a?(Hash)
+        end
+      end
+
+      private
+
+      attr_reader :wiki_content, :feature_gate
+
+      def empty_result(reason = nil, error = nil)
+        Result.new(content: wiki_content, reason: reason, error: error)
+      end
+
+      def enabled?
+        self.class.enabled?(feature_gate)
+      end
+
+      def block
+        @block ||= parse_front_matter_block
+      end
+
+      def parse_front_matter_block
+        wiki_content.match(Gitlab::FrontMatter::PATTERN) { |m| Block.new(*m.captures) } || Block.new
+      end
+
+      def strip_front_matter_block
+        wiki_content.gsub(Gitlab::FrontMatter::PATTERN, '')
+      end
+    end
+  end
+end