summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/runner.rb16
-rw-r--r--lib/banzai/filter/front_matter_filter.rb25
-rw-r--r--lib/gitlab/front_matter.rb24
-rw-r--r--lib/gitlab/middleware/multipart.rb3
-rw-r--r--lib/gitlab/wiki_pages.rb15
-rw-r--r--lib/gitlab/wiki_pages/front_matter_parser.rb120
6 files changed, 170 insertions, 33 deletions
diff --git a/lib/api/runner.rb b/lib/api/runner.rb
index f97e28de628..9095aba7340 100644
--- a/lib/api/runner.rb
+++ b/lib/api/runner.rb
@@ -251,21 +251,14 @@ module API
end
params do
requires :id, type: Integer, desc: %q(Job's ID)
+ requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact file to store (generated by Multipart middleware))
optional :token, type: String, desc: %q(Job's authentication token)
optional :expire_in, type: String, desc: %q(Specify when artifacts should expire)
optional :artifact_type, type: String, desc: %q(The type of artifact),
default: 'archive', values: Ci::JobArtifact.file_types.keys
optional :artifact_format, type: String, desc: %q(The format of artifact),
default: 'zip', values: Ci::JobArtifact.file_formats.keys
- optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
- optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse))
- optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse))
- optional 'file.size', type: Integer, desc: %q(real size of file (generated by Workhorse))
- optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file (generated by Workhorse))
- optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse))
- optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse))
- optional 'metadata.size', type: Integer, desc: %q(real size of metadata (generated by Workhorse))
- optional 'metadata.sha256', type: String, desc: %q(sha256 checksum of metadata (generated by Workhorse))
+ optional :metadata, type: ::API::Validations::Types::WorkhorseFile, desc: %(The artifact metadata to store (generated by Multipart middleware))
end
post '/:id/artifacts' do
not_allowed! unless Gitlab.config.artifacts.enabled
@@ -274,10 +267,9 @@ module API
job = authenticate_job!
forbidden!('Job is not running!') unless job.running?
- artifacts = UploadedFile.from_params(params, :file, JobArtifactUploader.workhorse_local_upload_path)
- metadata = UploadedFile.from_params(params, :metadata, JobArtifactUploader.workhorse_local_upload_path)
+ artifacts = params[:file]
+ metadata = params[:metadata]
- bad_request!('Missing artifacts file!') unless artifacts
file_too_large! unless artifacts.size < max_artifacts_size(job)
result = Ci::CreateJobArtifactsService.new(job.project).execute(job, artifacts, params, metadata_file: metadata)
diff --git a/lib/banzai/filter/front_matter_filter.rb b/lib/banzai/filter/front_matter_filter.rb
index 544231adea4..5900e762244 100644
--- a/lib/banzai/filter/front_matter_filter.rb
+++ b/lib/banzai/filter/front_matter_filter.rb
@@ -3,28 +3,11 @@
module Banzai
module Filter
class FrontMatterFilter < HTML::Pipeline::Filter
- DELIM_LANG = {
- '---' => 'yaml',
- '+++' => 'toml',
- ';;;' => 'json'
- }.freeze
-
- DELIM = Regexp.union(DELIM_LANG.keys)
-
- PATTERN = %r{
- \A(?:[^\r\n]*coding:[^\r\n]*)? # optional encoding line
- \s*
- ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*) # opening front matter marker (optional language specifier)
- \s*
- ^(?<front_matter>.*?) # front matter (not greedy)
- \s*
- ^\k<delim> # closing front matter marker
- \s*
- }mx.freeze
-
def call
- html.sub(PATTERN) do |_match|
- lang = $~[:lang].presence || DELIM_LANG[$~[:delim]]
+ lang_mapping = Gitlab::FrontMatter::DELIM_LANG
+
+ html.sub(Gitlab::FrontMatter::PATTERN) do |_match|
+ lang = $~[:lang].presence || lang_mapping[$~[:delim]]
["```#{lang}", $~[:front_matter], "```", "\n"].join("\n")
end
diff --git a/lib/gitlab/front_matter.rb b/lib/gitlab/front_matter.rb
new file mode 100644
index 00000000000..7612bd36aca
--- /dev/null
+++ b/lib/gitlab/front_matter.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module FrontMatter
+ DELIM_LANG = {
+ '---' => 'yaml',
+ '+++' => 'toml',
+ ';;;' => 'json'
+ }.freeze
+
+ DELIM = Regexp.union(DELIM_LANG.keys)
+
+ PATTERN = %r{
+ \A(?:[^\r\n]*coding:[^\r\n]*)? # optional encoding line
+ \s*
+ ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*) # opening front matter marker (optional language specifier)
+ \s*
+ ^(?<front_matter>.*?) # front matter block content (not greedy)
+ \s*
+ ^(\k<delim> | \.{3}) # closing front matter marker
+ \s*
+ }mx.freeze
+ end
+end
diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb
index cb4213d23a4..c82c05e7319 100644
--- a/lib/gitlab/middleware/multipart.rb
+++ b/lib/gitlab/middleware/multipart.rb
@@ -107,6 +107,7 @@ module Gitlab
[
::FileUploader.root,
Gitlab.config.uploads.storage_path,
+ JobArtifactUploader.workhorse_upload_path,
File.join(Rails.root, 'public/uploads/tmp')
]
end
@@ -125,6 +126,8 @@ module Gitlab
Handler.new(env, message).with_open_files do
@app.call(env)
end
+ rescue UploadedFile::InvalidPathError => e
+ [400, { 'Content-Type' => 'text/plain' }, e.message]
end
end
end
diff --git a/lib/gitlab/wiki_pages.rb b/lib/gitlab/wiki_pages.rb
new file mode 100644
index 00000000000..47f9aa1117f
--- /dev/null
+++ b/lib/gitlab/wiki_pages.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module WikiPages
+ # Many common file systems have a limit of 255 bytes for file and
+ # directory names, and while Git and GitLab both support paths exceeding
+ # those limits, the presence of them makes it impossible for users on
+ # those file systems to checkout a wiki repository locally.
+
+ # To avoid this situation, we enforce these limits when editing pages
+ # through the GitLab web interface and API:
+ MAX_TITLE_BYTES = 245 # reserving 10 bytes for the file extension
+ MAX_DIRECTORY_BYTES = 255
+ end
+end
diff --git a/lib/gitlab/wiki_pages/front_matter_parser.rb b/lib/gitlab/wiki_pages/front_matter_parser.rb
new file mode 100644
index 00000000000..45dc6cf7fd1
--- /dev/null
+++ b/lib/gitlab/wiki_pages/front_matter_parser.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module WikiPages
+ class FrontMatterParser
+ FEATURE_FLAG = :wiki_front_matter
+
+ # We limit the maximum length of text we are prepared to parse as YAML, to
+ # avoid exploitations and attempts to consume memory and CPU. We allow for:
+ # - a title line
+ # - a "slugs:" line
+ # - and up to 50 slugs
+ #
+ # This limit does not take comments into account.
+ MAX_SLUGS = 50
+ SLUG_LINE_LENGTH = (4 + Gitlab::WikiPages::MAX_DIRECTORY_BYTES + 1 + Gitlab::WikiPages::MAX_TITLE_BYTES)
+ MAX_FRONT_MATTER_LENGTH = (8 + Gitlab::WikiPages::MAX_TITLE_BYTES) + 7 + (SLUG_LINE_LENGTH * MAX_SLUGS)
+
+ ParseError = Class.new(StandardError)
+
+ class Result
+ attr_reader :front_matter, :content, :reason, :error
+
+ def initialize(content:, front_matter: {}, reason: nil, error: nil)
+ @content = content
+ @front_matter = front_matter.freeze
+ @reason = reason
+ @error = error
+ end
+ end
+
+ # @param [String] wiki_content
+ # @param [FeatureGate] feature_gate The scope for feature availability
+ def initialize(wiki_content, feature_gate)
+ @wiki_content = wiki_content
+ @feature_gate = feature_gate
+ end
+
+ def self.enabled?(gate = nil)
+ Feature.enabled?(FEATURE_FLAG, gate)
+ end
+
+ def parse
+ return empty_result unless enabled? && wiki_content.present?
+ return empty_result(block.error) unless block.valid?
+
+ Result.new(front_matter: block.data, content: strip_front_matter_block)
+ rescue ParseError => error
+ empty_result(:parse_error, error)
+ end
+
+ class Block
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(delim = nil, lang = '', text = nil)
+ @lang = lang.downcase.presence || Gitlab::FrontMatter::DELIM_LANG[delim]
+ @text = text
+ end
+
+ def data
+ @data ||= YAML.safe_load(text, symbolize_names: true)
+ rescue Psych::DisallowedClass, Psych::SyntaxError => error
+ raise ParseError, error.message
+ end
+
+ def valid?
+ error.nil?
+ end
+
+ def error
+ strong_memoize(:error) { no_match? || too_long? || not_yaml? || not_mapping? }
+ end
+
+ private
+
+ attr_reader :lang, :text
+
+ def no_match?
+ :no_match if text.nil?
+ end
+
+ def not_yaml?
+ :not_yaml if lang != 'yaml'
+ end
+
+ def too_long?
+ :too_long if text.size > MAX_FRONT_MATTER_LENGTH
+ end
+
+ def not_mapping?
+ :not_mapping unless data.is_a?(Hash)
+ end
+ end
+
+ private
+
+ attr_reader :wiki_content, :feature_gate
+
+ def empty_result(reason = nil, error = nil)
+ Result.new(content: wiki_content, reason: reason, error: error)
+ end
+
+ def enabled?
+ self.class.enabled?(feature_gate)
+ end
+
+ def block
+ @block ||= parse_front_matter_block
+ end
+
+ def parse_front_matter_block
+ wiki_content.match(Gitlab::FrontMatter::PATTERN) { |m| Block.new(*m.captures) } || Block.new
+ end
+
+ def strip_front_matter_block
+ wiki_content.gsub(Gitlab::FrontMatter::PATTERN, '')
+ end
+ end
+ end
+end