diff options
Diffstat (limited to 'lib')
29 files changed, 813 insertions, 61 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index c49c52213bf..8e259961828 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -143,6 +143,7 @@ module API mount ::API::Settings mount ::API::SidekiqMetrics mount ::API::Snippets + mount ::API::Submodules mount ::API::Subscriptions mount ::API::SystemHooks mount ::API::Tags diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9f7be27b047..61d57c643f0 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -607,6 +607,22 @@ module API end class MergeRequestBasic < ProjectEntity + expose :merged_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.merged_by + end + + expose :merged_at do |merge_request, _options| + merge_request.metrics&.merged_at + end + + expose :closed_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.latest_closed_by + end + + expose :closed_at do |merge_request, _options| + merge_request.metrics&.latest_closed_at + end + expose :title_html, if: -> (_, options) { options[:render_html] } do |entity| MarkupHelper.markdown_field(entity, :title) end @@ -676,22 +692,6 @@ module API merge_request.merge_request_diff.real_size end - expose :merged_by, using: Entities::UserBasic do |merge_request, _options| - merge_request.metrics&.merged_by - end - - expose :merged_at do |merge_request, _options| - merge_request.metrics&.merged_at - end - - expose :closed_by, using: Entities::UserBasic do |merge_request, _options| - merge_request.metrics&.latest_closed_by - end - - expose :closed_at do |merge_request, _options| - merge_request.metrics&.latest_closed_at - end - expose :latest_build_started_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options| merge_request.metrics&.latest_build_started_at end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 7909f9c7a00..491b5085bb8 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -28,7 +28,7 @@ module API args[:scope] = args[:scope].underscore if args[:scope] issues = IssuesFinder.new(current_user, args).execute - .preload(:assignees, :labels, :notes, :timelogs, :project, :author) + .preload(:assignees, :labels, :notes, :timelogs, :project, :author, :closed_by) issues.reorder(args[:order_by] => args[:sort]) end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index a617efaaa4c..16f07f16387 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -45,7 +45,7 @@ module API return merge_requests if args[:view] == 'simple' merge_requests - .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs) + .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs, metrics: [:latest_closed_by, :merged_by]) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index f1786c15f4f..1ae144ca9c1 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -14,7 +14,7 @@ module API end def public_snippets - SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute + SnippetsFinder.new(current_user, scope: :are_public).execute end end diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb new file mode 100644 index 00000000000..72d7d994102 --- /dev/null +++ b/lib/api/submodules.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module API + class Submodules < Grape::API + before { authenticate! } + + helpers do + def commit_params(attrs) + { + submodule: attrs[:submodule], + commit_sha: attrs[:commit_sha], + branch_name: attrs[:branch], + commit_message: attrs[:commit_message] + } + end + end + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects, requirements: Files::FILE_ENDPOINT_REQUIREMENTS do + desc 'Update existing submodule reference in repository' do + success Entities::Commit + end + params do + requires :submodule, type: String, desc: 'Url encoded full path to submodule.' + requires :commit_sha, type: String, desc: 'Commit sha to update the submodule to.' + requires :branch, type: String, desc: 'Name of the branch to commit into.' + optional :commit_message, type: String, desc: 'Commit message. If no message is provided a default one will be set.' + end + put ":id/repository/submodules/:submodule", requirements: Files::FILE_ENDPOINT_REQUIREMENTS do + authorize! :push_code, user_project + + submodule_params = declared_params(include_missing: false) + + result = ::Submodules::UpdateService.new(user_project, current_user, commit_params(submodule_params)).execute + + if result[:status] == :success + commit_detail = user_project.repository.commit(result[:result]) + present commit_detail, with: Entities::CommitDetail + else + render_api_error!(result[:message], result[:http_status] || 400) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb index 9941c2fe6d9..47579d46c1b 100644 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -67,7 +67,7 @@ module Gitlab .where('access_level >= ?', 20) confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id) - confidential_issues.each_batch(of: 100) do |batch| + confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch| batch.each do |issue| assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index e4610faa327..8e8c979f973 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when start_in artifacts cache dependencies before_script after_script variables - environment coverage retry extends].freeze + environment coverage retry parallel extends].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -26,14 +26,12 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true - validates :retry, numericality: { only_integer: true, - greater_than_or_equal_to: 0, - less_than_or_equal_to: 2 } + validates :parallel, numericality: { only_integer: true, + greater_than_or_equal_to: 2 } validates :when, inclusion: { in: %w[on_success on_failure always manual delayed], message: 'should be on_success, on_failure, ' \ 'always, manual or delayed' } - validates :dependencies, array_of_strings: true validates :extends, type: String end @@ -79,17 +77,21 @@ module Gitlab description: 'Artifacts configuration for this job.' entry :environment, Entry::Environment, - description: 'Environment configuration for this job.' + description: 'Environment configuration for this job.' entry :coverage, Entry::Coverage, - description: 'Coverage configuration for this job.' + description: 'Coverage configuration for this job.' + + entry :retry, Entry::Retry, + description: 'Retry configuration for this job.' helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage, :retry + :artifacts, :commands, :environment, :coverage, :retry, + :parallel attributes :script, :tags, :allow_failure, :when, :dependencies, - :retry, :extends, :start_in + :retry, :parallel, :extends, :start_in def compose!(deps = nil) super do @@ -157,7 +159,8 @@ module Gitlab environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, - retry: retry_defined? ? retry_value.to_i : nil, + retry: retry_defined? ? retry_value : nil, + parallel: parallel_defined? ? parallel_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored? } diff --git a/lib/gitlab/ci/config/entry/retry.rb b/lib/gitlab/ci/config/entry/retry.rb new file mode 100644 index 00000000000..e39cc5de229 --- /dev/null +++ b/lib/gitlab/ci/config/entry/retry.rb @@ -0,0 +1,90 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a retry config for a job. + # + class Retry < Simplifiable + strategy :SimpleRetry, if: -> (config) { config.is_a?(Integer) } + strategy :FullRetry, if: -> (config) { config.is_a?(Hash) } + + class SimpleRetry < Entry::Node + include Entry::Validatable + + validations do + validates :config, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } + end + + def value + { + max: config + } + end + + def location + 'retry' + end + end + + class FullRetry < Entry::Node + include Entry::Validatable + include Entry::Attributable + + ALLOWED_KEYS = %i[max when].freeze + attributes :max, :when + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :max, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } + + validates :when, array_of_strings_or_string: true + validates :when, + allowed_array_values: { in: FullRetry.possible_retry_when_values }, + if: -> (config) { config.when.is_a?(Array) } + validates :when, + inclusion: { in: FullRetry.possible_retry_when_values }, + if: -> (config) { config.when.is_a?(String) } + end + end + + def self.possible_retry_when_values + @possible_retry_when_values ||= ::Ci::Build.failure_reasons.keys.map(&:to_s) + ['always'] + end + + def value + super.tap do |config| + # make sure that `when` is an array, because we allow it to + # be passed as a String in config for simplicity + config[:when] = Array.wrap(config[:when]) if config[:when] + end + end + + def location + 'retry' + end + end + + class UnknownStrategy < Entry::Node + def errors + ["#{location} has to be either an integer or a hash"] + end + + def location + 'retry config' + end + end + + def self.default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index 805d26ca8d8..a1d552fb2e5 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -7,11 +7,11 @@ module Gitlab module Validators class AllowedKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - unknown_keys = record.config.try(:keys).to_a - options[:in] + unknown_keys = value.try(:keys).to_a - options[:in] if unknown_keys.any? - record.errors.add(:config, 'contains unknown keys: ' + - unknown_keys.join(', ')) + record.errors.add(attribute, "contains unknown keys: " + + unknown_keys.join(', ')) end end end @@ -24,6 +24,16 @@ module Gitlab end end + class AllowedArrayValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unkown_values = value - options[:in] + unless unkown_values.empty? + record.errors.add(attribute, "contains unknown values: " + + unkown_values.join(', ')) + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers @@ -68,6 +78,14 @@ module Gitlab end end + class HashOrIntegerValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) || value.is_a?(Integer) + record.errors.add(attribute, 'should be a hash or an integer') + end + end + end + class KeyValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb new file mode 100644 index 00000000000..b7743bd2090 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + def initialize(jobs_config) + @jobs_config = jobs_config + end + + def normalize_jobs + extract_parallelized_jobs! + return @jobs_config if @parallelized_jobs.empty? + + parallelized_config = parallelize_jobs + parallelize_dependencies(parallelized_config) + end + + private + + def extract_parallelized_jobs! + @parallelized_jobs = {} + + @jobs_config.each do |job_name, config| + if config[:parallel] + @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + end + end + + @parallelized_jobs + end + + def parallelize_jobs + @jobs_config.each_with_object({}) do |(job_name, config), hash| + if @parallelized_jobs.key?(job_name) + @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) } + else + hash[job_name] = config + end + + hash + end + end + + def parallelize_dependencies(parallelized_config) + parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) + parallelized_config.each_with_object({}) do |(job_name, config), hash| + if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? + deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten + hash[job_name] = config.merge(dependencies: deps) + else + hash[job_name] = config + end + + hash + end + end + + def self.parallelize_job_names(name, total) + Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] } + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index f443dbee120..b3452eae189 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -9,7 +9,7 @@ module Gitlab { image: 'illustrations/illustrations_scheduled-job_countdown.svg', size: 'svg-394', - title: _("This is a delayed to run in ") + " #{execute_in}", + title: _("This is a delayed job to run in %{remainingTime}"), content: _("This job will automatically run after it's timer finishes. " \ "Often they are used for incremental roll-out deploys " \ "to production environments. When unscheduled it converts " \ @@ -18,21 +18,12 @@ module Gitlab end def status_tooltip - "delayed manual action (#{execute_in})" + "delayed manual action (%{remainingTime})" end def self.matches?(build, user) build.scheduled? && build.scheduled_at end - - private - - include TimeHelper - - def execute_in - remaining_seconds = [0, subject.scheduled_at - Time.now].max - duration_in_numbers(remaining_seconds) - end end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 39a1b52e531..e6ec400e476 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -52,6 +52,8 @@ module Gitlab after_script: job[:after_script], environment: job[:environment], retry: job[:retry], + parallel: job[:parallel], + instance: job[:instance], start_in: job[:start_in] }.compact } end @@ -104,7 +106,7 @@ module Gitlab ## # Jobs # - @jobs = @ci_config.jobs + @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs @jobs.each do |name, job| # logical validation for job diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index fb117baca9e..84595f8afd7 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -26,6 +26,7 @@ module Gitlab @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + @unfolded = false # Ensure items are collected in the the batch new_blob_lazy @@ -135,6 +136,24 @@ module Gitlab Gitlab::Diff::Parser.new.parse(raw_diff.each_line, diff_file: self).to_a end + # Changes diff_lines according to the given position. That is, + # it checks whether the position requires blob lines into the diff + # in order to be presented. + def unfold_diff_lines(position) + return unless position + + unfolder = Gitlab::Diff::LinesUnfolder.new(self, position) + + if unfolder.unfold_required? + @diff_lines = unfolder.unfolded_diff_lines + @unfolded = true + end + end + + def unfolded? + @unfolded + end + def highlighted_diff_lines @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 5b67cd46c48..70063071ee7 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -3,9 +3,9 @@ module Gitlab class Line SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code, :type, :index, :old_pos, :new_pos + attr_reader :line_code, :type, :old_pos, :new_pos attr_writer :rich_text - attr_accessor :text + attr_accessor :text, :index def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text, @type, @index = text, type, index @@ -19,7 +19,14 @@ module Gitlab end def self.init_from_hash(hash) - new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos], line_code: hash[:line_code], rich_text: hash[:rich_text]) + new(hash[:text], + hash[:type], + hash[:index], + hash[:old_pos], + hash[:new_pos], + parent_file: hash[:parent_file], + line_code: hash[:line_code], + rich_text: hash[:rich_text]) end def to_hash diff --git a/lib/gitlab/diff/lines_unfolder.rb b/lib/gitlab/diff/lines_unfolder.rb new file mode 100644 index 00000000000..9306b7e16a2 --- /dev/null +++ b/lib/gitlab/diff/lines_unfolder.rb @@ -0,0 +1,235 @@ +# frozen_string_literal: true + +# Given a position, calculates which Blob lines should be extracted, treated and +# injected in the current diff file lines in order to present a "unfolded" diff. +module Gitlab + module Diff + class LinesUnfolder + include Gitlab::Utils::StrongMemoize + + UNFOLD_CONTEXT_SIZE = 3 + + def initialize(diff_file, position) + @diff_file = diff_file + @blob = diff_file.old_blob + @position = position + @generate_top_match_line = true + @generate_bottom_match_line = true + + # These methods update `@generate_top_match_line` and + # `@generate_bottom_match_line`. + @from_blob_line = calculate_from_blob_line! + @to_blob_line = calculate_to_blob_line! + end + + # Returns merged diff lines with required blob lines with correct + # positions. + def unfolded_diff_lines + strong_memoize(:unfolded_diff_lines) do + next unless unfold_required? + + merged_diff_with_blob_lines + end + end + + # Returns the extracted lines from the old blob which should be merged + # with the current diff lines. + def blob_lines + strong_memoize(:blob_lines) do + # Blob lines, unlike diffs, doesn't start with an empty space for + # unchanged line, so the parsing and highlighting step can get fuzzy + # without the following change. + line_prefix = ' ' + blob_as_diff_lines = @blob.data.each_line.map { |line| "#{line_prefix}#{line}" } + + lines = Gitlab::Diff::Parser.new.parse(blob_as_diff_lines, diff_file: @diff_file).to_a + + from = from_blob_line - 1 + to = to_blob_line - 1 + + lines[from..to] + end + end + + def unfold_required? + strong_memoize(:unfold_required) do + next false unless @diff_file.text? + next false unless @position.unchanged? + next false if @diff_file.new_file? || @diff_file.deleted_file? + next false unless @position.old_line + # Invalid position (MR import scenario) + next false if @position.old_line > @blob.lines.size + next false if @diff_file.diff_lines.empty? + next false if @diff_file.line_for_position(@position) + next false unless unfold_line + + true + end + end + + private + + attr_reader :from_blob_line, :to_blob_line + + def merged_diff_with_blob_lines + lines = @diff_file.diff_lines + match_line = unfold_line + insert_index = bottom? ? -1 : match_line.index + + lines -= [match_line] unless bottom? + + lines.insert(insert_index, *blob_lines_with_matches) + + # The inserted blob lines have invalid indexes, so we need + # to reindex them. + reindex(lines) + + lines + end + + # Returns 'unchanged' blob lines with recalculated `old_pos` and + # `new_pos` and the recalculated new match line (needed if we for instance + # we unfolded once, but there are still folded lines). + def blob_lines_with_matches + old_pos = from_blob_line + new_pos = from_blob_line + offset + + new_blob_lines = [] + + new_blob_lines.push(top_blob_match_line) if top_blob_match_line + + blob_lines.each do |line| + new_blob_lines << Gitlab::Diff::Line.new(line.text, line.type, nil, old_pos, new_pos, + parent_file: @diff_file) + + old_pos += 1 + new_pos += 1 + end + + new_blob_lines.push(bottom_blob_match_line) if bottom_blob_match_line + + new_blob_lines + end + + def reindex(lines) + lines.each_with_index { |line, i| line.index = i } + end + + def top_blob_match_line + strong_memoize(:top_blob_match_line) do + next unless @generate_top_match_line + + old_pos = from_blob_line + new_pos = from_blob_line + offset + + build_match_line(old_pos, new_pos) + end + end + + def bottom_blob_match_line + strong_memoize(:bottom_blob_match_line) do + # The bottom line match addition is already handled on + # Diff::File#diff_lines_for_serializer + next if bottom? + next unless @generate_bottom_match_line + + position = line_after_unfold_position.old_pos + + old_pos = position + new_pos = position + offset + + build_match_line(old_pos, new_pos) + end + end + + def build_match_line(old_pos, new_pos) + blob_lines_length = blob_lines.length + old_line_ref = [old_pos, blob_lines_length].join(',') + new_line_ref = [new_pos, blob_lines_length].join(',') + new_match_line_str = "@@ -#{old_line_ref}+#{new_line_ref} @@" + + Gitlab::Diff::Line.new(new_match_line_str, 'match', nil, old_pos, new_pos) + end + + # Returns the first line position that should be extracted + # from `blob_lines`. + def calculate_from_blob_line! + return unless unfold_required? + + from = comment_position - UNFOLD_CONTEXT_SIZE + + # There's no line before the match if it's in the top-most + # position. + prev_line_number = line_before_unfold_position&.old_pos || 0 + + if from <= prev_line_number + 1 + @generate_top_match_line = false + from = prev_line_number + 1 + end + + from + end + + # Returns the last line position that should be extracted + # from `blob_lines`. + def calculate_to_blob_line! + return unless unfold_required? + + to = comment_position + UNFOLD_CONTEXT_SIZE + + return to if bottom? + + next_line_number = line_after_unfold_position.old_pos + + if to >= next_line_number - 1 + @generate_bottom_match_line = false + to = next_line_number - 1 + end + + to + end + + def offset + unfold_line.new_pos - unfold_line.old_pos + end + + def line_before_unfold_position + return unless index = unfold_line&.index + + @diff_file.diff_lines[index - 1] if index > 0 + end + + def line_after_unfold_position + return unless index = unfold_line&.index + + @diff_file.diff_lines[index + 1] if index >= 0 + end + + def bottom? + strong_memoize(:bottom) do + @position.old_line > last_line.old_pos + end + end + + # Returns the line which needed to be expanded in order to send a comment + # in `@position`. + def unfold_line + strong_memoize(:unfold_line) do + next last_line if bottom? + + @diff_file.diff_lines.find do |line| + line.old_pos > comment_position && line.type == 'match' + end + end + end + + def comment_position + @position.old_line + end + + def last_line + @diff_file.diff_lines.last + end + end + end +end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index f967494199e..7bfab2d808f 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -101,6 +101,10 @@ module Gitlab @diff_refs ||= DiffRefs.new(base_sha: base_sha, start_sha: start_sha, head_sha: head_sha) end + def unfolded_diff?(repository) + diff_file(repository)&.unfolded? + end + def diff_file(repository) return @diff_file if defined?(@diff_file) @@ -134,7 +138,13 @@ module Gitlab return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) - comparison.diffs(diff_options).diff_files.first + file = comparison.diffs(diff_options).diff_files.first + + # We need to unfold diff lines according to the position in order + # to correctly calculate the line code and trace position changes. + file&.unfold_diff_lines(self) + + file end def get_formatter_class(type) diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index e68ae60ff98..5772727e855 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -44,10 +44,26 @@ module Gitlab @project ||= Project.find_by_full_path(project_path) end + def metrics_params + super.merge(includes_patches: patch_attachments.any?) + end + private + def build_merge_request + MergeRequests::BuildService.new(project, author, merge_request_params).execute + end + def create_merge_request - merge_request = MergeRequests::BuildService.new(project, author, merge_request_params).execute + merge_request = build_merge_request + + if patch_attachments.any? + apply_patches_to_source_branch(start_branch: merge_request.target_branch) + remove_patch_attachments + # Rebuild the merge request as the source branch might just have + # been created, so we should re-validate. + merge_request = build_merge_request + end if merge_request.errors.any? merge_request @@ -59,12 +75,42 @@ module Gitlab def merge_request_params params = { source_project_id: project.id, - source_branch: mail.subject, + source_branch: source_branch, target_project_id: project.id } params[:description] = message if message.present? params end + + def apply_patches_to_source_branch(start_branch:) + patches = patch_attachments.map { |patch| patch.body.decoded } + + result = Commits::CommitPatchService + .new(project, author, branch_name: source_branch, patches: patches, start_branch: start_branch) + .execute + + if result[:status] != :success + message = "Could not apply patches to #{source_branch}:\n#{result[:message]}" + raise InvalidAttachment, message + end + end + + def remove_patch_attachments + patch_attachments.each { |patch| mail.parts.delete(patch) } + # reset the message, so it needs to be reporocessed when the attachments + # have been modified + @message = nil + end + + def patch_attachments + @patches ||= mail.attachments + .select { |attachment| attachment.filename.ends_with?('.patch') } + .sort_by(&:filename) + end + + def source_branch + @source_branch ||= mail.subject + end end end end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index d8c594ad0e7..3a689967a64 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -18,6 +18,7 @@ module Gitlab InvalidIssueError = Class.new(InvalidRecordError) InvalidMergeRequestError = Class.new(InvalidRecordError) UnknownIncomingEmail = Class.new(ProcessingError) + InvalidAttachment = Class.new(ProcessingError) class Receiver def initialize(raw) diff --git a/lib/gitlab/git/patches/collection.rb b/lib/gitlab/git/patches/collection.rb new file mode 100644 index 00000000000..ad6b5d32abc --- /dev/null +++ b/lib/gitlab/git/patches/collection.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class Collection + MAX_PATCH_SIZE = 2.megabytes + + def initialize(one_or_more_patches) + @patches = Array(one_or_more_patches).map do |patch_content| + Gitlab::Git::Patches::Patch.new(patch_content) + end + end + + def content + @patches.map(&:content).join("\n") + end + + def valid_size? + size < MAX_PATCH_SIZE + end + + # rubocop: disable CodeReuse/ActiveRecord + # `@patches` is not an `ActiveRecord` relation, but an `Enumerable` + # We're using sum from `ActiveSupport` + def size + @size ||= @patches.sum(&:size) + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/git/patches/commit_patches.rb b/lib/gitlab/git/patches/commit_patches.rb new file mode 100644 index 00000000000..c62994432d3 --- /dev/null +++ b/lib/gitlab/git/patches/commit_patches.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class CommitPatches + include Gitlab::Git::WrapsGitalyErrors + + def initialize(user, repository, branch, patch_collection) + @user, @repository, @branch, @patches = user, repository, branch, patch_collection + end + + def commit + repository.with_cache_hooks do + wrapped_gitaly_errors do + operation_service.user_commit_patches(user, branch, patches.content) + end + end + end + + private + + attr_reader :user, :repository, :branch, :patches + + def operation_service + repository.raw.gitaly_operation_client + end + end + end + end +end diff --git a/lib/gitlab/git/patches/patch.rb b/lib/gitlab/git/patches/patch.rb new file mode 100644 index 00000000000..fe6ae1b5b00 --- /dev/null +++ b/lib/gitlab/git/patches/patch.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class Patch + attr_reader :content + + def initialize(content) + @content = content + end + + def size + content.bytesize + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 20cd257bb98..1642c4c5687 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -571,6 +571,20 @@ module Gitlab end end + def update_submodule(user:, submodule:, commit_sha:, message:, branch:) + args = { + user: user, + submodule: submodule, + commit_sha: commit_sha, + branch: branch, + message: message + } + + wrapped_gitaly_errors do + gitaly_operation_client.user_update_submodule(args) + end + end + # Delete the specified branch from the repository def delete_branch(branch_name) wrapped_gitaly_errors do diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 0f148614b20..4c78b790ce5 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -230,6 +230,32 @@ module Gitlab response.squash_sha end + def user_update_submodule(user:, submodule:, commit_sha:, branch:, message:) + request = Gitaly::UserUpdateSubmoduleRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + commit_sha: commit_sha, + branch: encode_binary(branch), + submodule: encode_binary(submodule), + commit_message: encode_binary(message) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_update_submodule, + request + ) + + if response.pre_receive_error.present? + raise Gitlab::Git::PreReceiveError, response.pre_receive_error + elsif response.commit_error.present? + raise Gitlab::Git::CommitError, response.commit_error + else + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + end + def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, start_branch_name, start_repository) @@ -273,6 +299,29 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end + def user_commit_patches(user, branch_name, patches) + header = Gitaly::UserApplyPatchRequest::Header.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + target_branch: encode_binary(branch_name) + ) + reader = binary_stringio(patches) + + chunks = Enumerator.new do |chunk| + chunk.yield Gitaly::UserApplyPatchRequest.new(header: header) + + until reader.eof? + patch_chunk = reader.read(MAX_MSG_SIZE) + + chunk.yield(Gitaly::UserApplyPatchRequest.new(patches: patch_chunk)) + end + end + + response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks) + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + private def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 1be7924d6ac..55add06bdb4 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -4,22 +4,27 @@ module Gitlab class InstallCommand include BaseCommand - attr_reader :name, :files, :chart, :version, :repository + attr_reader :name, :files, :chart, :version, :repository, :preinstall, :postinstall - def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil) + def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil, preinstall: nil, postinstall: nil) @name = name @chart = chart @version = version @rbac = rbac @files = files @repository = repository + @preinstall = preinstall + @postinstall = postinstall end def generate_script super + [ init_command, repository_command, - script_command + repository_update_command, + preinstall_command, + install_command, + postinstall_command ].compact.join("\n") end @@ -37,12 +42,24 @@ module Gitlab ['helm', 'repo', 'add', name, repository].shelljoin if repository end - def script_command + def repository_update_command + 'helm repo update >/dev/null' if repository + end + + def install_command command = ['helm', 'install', chart] + install_command_flags command.shelljoin + " >/dev/null\n" end + def preinstall_command + preinstall.join("\n") if preinstall + end + + def postinstall_command + postinstall.join("\n") if postinstall + end + def install_command_flags name_flag = ['--name', name] namespace_flag = ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] diff --git a/lib/gitlab/private_commit_email.rb b/lib/gitlab/private_commit_email.rb new file mode 100644 index 00000000000..bade2248ccd --- /dev/null +++ b/lib/gitlab/private_commit_email.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module PrivateCommitEmail + TOKEN = "_private".freeze + + class << self + def regex + hostname_regexp = Regexp.escape(Gitlab::CurrentSettings.current_application_settings.commit_email_hostname) + + /\A(?<id>([0-9]+))\-([^@]+)@#{hostname_regexp}\z/ + end + + def user_id_for_email(email) + match = email&.match(regex) + return unless match + + match[:id].to_i + end + + def for_user(user) + hostname = Gitlab::CurrentSettings.current_application_settings.commit_email_hostname + + "#{user.id}-#{user.username}@#{hostname}" + end + end + end +end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 30c6806b68e..59f8dd889aa 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -29,7 +29,7 @@ module Gitlab # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" # ``` - def extract_commands(content) + def extract_commands(content, only: nil) return [content, []] unless content content = content.dup @@ -37,7 +37,7 @@ module Gitlab commands = [] content.delete!("\r") - content.gsub!(commands_regex) do + content.gsub!(commands_regex(only: only)) do if $~[:cmd] commands << [$~[:cmd].downcase, $~[:arg]].reject(&:blank?) '' @@ -60,8 +60,8 @@ module Gitlab # It looks something like: # # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ - def commands_regex - names = command_names.map(&:to_s) + def commands_regex(only:) + names = command_names(limit_to_commands: only).map(&:to_s) @commands_regex ||= %r{ (?<code> @@ -133,10 +133,14 @@ module Gitlab [content, commands] end - def command_names + def command_names(limit_to_commands:) command_definitions.flat_map do |command| next if command.noop? + if limit_to_commands && (command.all_names & limit_to_commands).empty? + next + end + command.all_names end.compact end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index cc0817bdcd2..069cd1f802a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -62,6 +62,7 @@ module Gitlab clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed), clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed), clusters_applications_runner: count(::Clusters::Applications::Runner.installed), + clusters_applications_knative: count(::Clusters::Applications::Knative.installed), in_review_folder: count(::Environment.in_review_folder), groups: count(Group), issues: count(Issue), @@ -126,7 +127,6 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def services_usage types = { - JiraService: :projects_jira_active, SlackService: :projects_slack_notifications_active, SlackSlashCommandsService: :projects_slack_slash_active, PrometheusService: :projects_prometheus_active @@ -134,6 +134,23 @@ module Gitlab results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1)) types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 } + .merge(jira_usage) + end + + def jira_usage + # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999 + # so we can just check for subdomains of atlassian.net + services = count( + Service.unscoped.where(type: :JiraService, active: true) + .group("CASE WHEN properties LIKE '%.atlassian.net%' THEN 'cloud' ELSE 'server' END"), + fallback: Hash.new(-1) + ) + + { + projects_jira_server_active: services['server'] || 0, + projects_jira_cloud_active: services['cloud'] || 0, + projects_jira_active: services['server'] == -1 ? -1 : services.values.sum + } end def count(relation, fallback: -1) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 2c92458f777..9e59137a2c0 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -16,6 +16,11 @@ module Gitlab str.force_encoding(Encoding::UTF_8) end + # Append path to host, making sure there's one single / in between + def append_path(host, path) + "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}" + end + # A slugified version of the string, suitable for inclusion in URLs and # domain names. Rules: # |