diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/runner.rb | 16 | ||||
| -rw-r--r-- | lib/banzai/filter/front_matter_filter.rb | 25 | ||||
| -rw-r--r-- | lib/gitlab/front_matter.rb | 24 | ||||
| -rw-r--r-- | lib/gitlab/middleware/multipart.rb | 3 | ||||
| -rw-r--r-- | lib/gitlab/wiki_pages.rb | 15 | ||||
| -rw-r--r-- | lib/gitlab/wiki_pages/front_matter_parser.rb | 120 |
6 files changed, 170 insertions, 33 deletions
diff --git a/lib/api/runner.rb b/lib/api/runner.rb index f97e28de628..9095aba7340 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -251,21 +251,14 @@ module API end params do requires :id, type: Integer, desc: %q(Job's ID) + requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact file to store (generated by Multipart middleware)) optional :token, type: String, desc: %q(Job's authentication token) optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) optional :artifact_type, type: String, desc: %q(The type of artifact), default: 'archive', values: Ci::JobArtifact.file_types.keys optional :artifact_format, type: String, desc: %q(The format of artifact), default: 'zip', values: Ci::JobArtifact.file_formats.keys - optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) - optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) - optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) - optional 'file.size', type: Integer, desc: %q(real size of file (generated by Workhorse)) - optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file (generated by Workhorse)) - optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) - optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) - optional 'metadata.size', type: Integer, desc: %q(real size of metadata (generated by Workhorse)) - optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of metadata (generated by Workhorse)) + optional :metadata, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact metadata to store (generated by Multipart middleware)) end post '/:id/artifacts' do not_allowed! unless Gitlab.config.artifacts.enabled @@ -274,10 +267,9 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - artifacts = UploadedFile.from_params(params, :file, JobArtifactUploader.workhorse_local_upload_path) - metadata = UploadedFile.from_params(params, :metadata, JobArtifactUploader.workhorse_local_upload_path) + artifacts = params[:file] + metadata = params[:metadata] - bad_request!('Missing artifacts file!') unless artifacts file_too_large! unless artifacts.size < max_artifacts_size(job) result = Ci::CreateJobArtifactsService.new(job.project).execute(job, artifacts, params, metadata_file: metadata) diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb index 544231adea4..5900e762244 100644 --- a/lib/banzai/filter/front_matter_filter.rb +++ b/lib/banzai/filter/front_matter_filter.rb @@ -3,28 +3,11 @@ module Banzai module Filter class FrontMatterFilter < HTML::Pipeline::Filter - DELIM_LANG = { - '---' => 'yaml', - '+++' => 'toml', - ';;;' => 'json' - }.freeze - - DELIM = Regexp.union(DELIM_LANG.keys) - - PATTERN = %r{ - \A(?:[^\r\n]*coding:[^\r\n]*)? # optional encoding line - \s* - ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*) # opening front matter marker (optional language specifier) - \s* - ^(?<front_matter>.*?) # front matter (not greedy) - \s* - ^\k<delim> # closing front matter marker - \s* - }mx.freeze - def call - html.sub(PATTERN) do |_match| - lang = $~[:lang].presence || DELIM_LANG[$~[:delim]] + lang_mapping = Gitlab::FrontMatter::DELIM_LANG + + html.sub(Gitlab::FrontMatter::PATTERN) do |_match| + lang = $~[:lang].presence || lang_mapping[$~[:delim]] ["```#{lang}", $~[:front_matter], "```", "\n"].join("\n") end diff --git a/lib/gitlab/front_matter.rb b/lib/gitlab/front_matter.rb new file mode 100644 index 00000000000..7612bd36aca --- /dev/null +++ b/lib/gitlab/front_matter.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module FrontMatter + DELIM_LANG = { + '---' => 'yaml', + '+++' => 'toml', + ';;;' => 'json' + }.freeze + + DELIM = Regexp.union(DELIM_LANG.keys) + + PATTERN = %r{ + \A(?:[^\r\n]*coding:[^\r\n]*)? # optional encoding line + \s* + ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*) # opening front matter marker (optional language specifier) + \s* + ^(?<front_matter>.*?) # front matter block content (not greedy) + \s* + ^(\k<delim> | \.{3}) # closing front matter marker + \s* + }mx.freeze + end +end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index cb4213d23a4..c82c05e7319 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -107,6 +107,7 @@ module Gitlab [ ::FileUploader.root, Gitlab.config.uploads.storage_path, + JobArtifactUploader.workhorse_upload_path, File.join(Rails.root, 'public/uploads/tmp') ] end @@ -125,6 +126,8 @@ module Gitlab Handler.new(env, message).with_open_files do @app.call(env) end + rescue UploadedFile::InvalidPathError => e + [400, { 'Content-Type' => 'text/plain' }, e.message] end end end diff --git a/lib/gitlab/wiki_pages.rb b/lib/gitlab/wiki_pages.rb new file mode 100644 index 00000000000..47f9aa1117f --- /dev/null +++ b/lib/gitlab/wiki_pages.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module WikiPages + # Many common file systems have a limit of 255 bytes for file and + # directory names, and while Git and GitLab both support paths exceeding + # those limits, the presence of them makes it impossible for users on + # those file systems to checkout a wiki repository locally. + + # To avoid this situation, we enforce these limits when editing pages + # through the GitLab web interface and API: + MAX_TITLE_BYTES = 245 # reserving 10 bytes for the file extension + MAX_DIRECTORY_BYTES = 255 + end +end diff --git a/lib/gitlab/wiki_pages/front_matter_parser.rb b/lib/gitlab/wiki_pages/front_matter_parser.rb new file mode 100644 index 00000000000..45dc6cf7fd1 --- /dev/null +++ b/lib/gitlab/wiki_pages/front_matter_parser.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module WikiPages + class FrontMatterParser + FEATURE_FLAG = :wiki_front_matter + + # We limit the maximum length of text we are prepared to parse as YAML, to + # avoid exploitations and attempts to consume memory and CPU. We allow for: + # - a title line + # - a "slugs:" line + # - and up to 50 slugs + # + # This limit does not take comments into account. + MAX_SLUGS = 50 + SLUG_LINE_LENGTH = (4 + Gitlab::WikiPages::MAX_DIRECTORY_BYTES + 1 + Gitlab::WikiPages::MAX_TITLE_BYTES) + MAX_FRONT_MATTER_LENGTH = (8 + Gitlab::WikiPages::MAX_TITLE_BYTES) + 7 + (SLUG_LINE_LENGTH * MAX_SLUGS) + + ParseError = Class.new(StandardError) + + class Result + attr_reader :front_matter, :content, :reason, :error + + def initialize(content:, front_matter: {}, reason: nil, error: nil) + @content = content + @front_matter = front_matter.freeze + @reason = reason + @error = error + end + end + + # @param [String] wiki_content + # @param [FeatureGate] feature_gate The scope for feature availability + def initialize(wiki_content, feature_gate) + @wiki_content = wiki_content + @feature_gate = feature_gate + end + + def self.enabled?(gate = nil) + Feature.enabled?(FEATURE_FLAG, gate) + end + + def parse + return empty_result unless enabled? && wiki_content.present? + return empty_result(block.error) unless block.valid? + + Result.new(front_matter: block.data, content: strip_front_matter_block) + rescue ParseError => error + empty_result(:parse_error, error) + end + + class Block + include Gitlab::Utils::StrongMemoize + + def initialize(delim = nil, lang = '', text = nil) + @lang = lang.downcase.presence || Gitlab::FrontMatter::DELIM_LANG[delim] + @text = text + end + + def data + @data ||= YAML.safe_load(text, symbolize_names: true) + rescue Psych::DisallowedClass, Psych::SyntaxError => error + raise ParseError, error.message + end + + def valid? + error.nil? + end + + def error + strong_memoize(:error) { no_match? || too_long? || not_yaml? || not_mapping? } + end + + private + + attr_reader :lang, :text + + def no_match? + :no_match if text.nil? + end + + def not_yaml? + :not_yaml if lang != 'yaml' + end + + def too_long? + :too_long if text.size > MAX_FRONT_MATTER_LENGTH + end + + def not_mapping? + :not_mapping unless data.is_a?(Hash) + end + end + + private + + attr_reader :wiki_content, :feature_gate + + def empty_result(reason = nil, error = nil) + Result.new(content: wiki_content, reason: reason, error: error) + end + + def enabled? + self.class.enabled?(feature_gate) + end + + def block + @block ||= parse_front_matter_block + end + + def parse_front_matter_block + wiki_content.match(Gitlab::FrontMatter::PATTERN) { |m| Block.new(*m.captures) } || Block.new + end + + def strip_front_matter_block + wiki_content.gsub(Gitlab::FrontMatter::PATTERN, '') + end + end + end +end |