diff options
Diffstat (limited to 'lib')
321 files changed, 1925 insertions, 502 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index c49c52213bf..8e259961828 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -143,6 +143,7 @@ module API mount ::API::Settings mount ::API::SidekiqMetrics mount ::API::Snippets + mount ::API::Submodules mount ::API::Subscriptions mount ::API::SystemHooks mount ::API::Tags diff --git a/lib/api/commits.rb b/lib/api/commits.rb index e59abd3e3d0..3b8f3fedccf 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -194,11 +194,47 @@ module API branch_name: params[:branch] } - result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute + result = ::Commits::CherryPickService + .new(user_project, current_user, commit_params) + .execute if result[:status] == :success - branch = find_branch!(params[:branch]) - present user_project.repository.commit(branch.dereferenced_target), with: Entities::Commit + present user_project.repository.commit(result[:result]), + with: Entities::Commit + else + render_api_error!(result[:message], 400) + end + end + + desc 'Revert a commit in a branch' do + detail 'This feature was introduced in GitLab 11.6' + success Entities::Commit + end + params do + requires :sha, type: String, desc: 'Commit SHA to revert' + requires :branch, type: String, desc: 'Target branch name', allow_blank: false + end + post ':id/repository/commits/:sha/revert', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do + authorize_push_to_branch!(params[:branch]) + + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + find_branch!(params[:branch]) + + commit_params = { + commit: commit, + start_branch: params[:branch], + branch_name: params[:branch] + } + + result = ::Commits::RevertService + .new(user_project, current_user, commit_params) + .execute + + if result[:status] == :success + present user_project.repository.commit(result[:result]), + with: Entities::Commit else render_api_error!(result[:message], 400) end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9f7be27b047..61d57c643f0 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -607,6 +607,22 @@ module API end class MergeRequestBasic < ProjectEntity + expose :merged_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.merged_by + end + + expose :merged_at do |merge_request, _options| + merge_request.metrics&.merged_at + end + + expose :closed_by, using: Entities::UserBasic do |merge_request, _options| + merge_request.metrics&.latest_closed_by + end + + expose :closed_at do |merge_request, _options| + merge_request.metrics&.latest_closed_at + end + expose :title_html, if: -> (_, options) { options[:render_html] } do |entity| MarkupHelper.markdown_field(entity, :title) end @@ -676,22 +692,6 @@ module API merge_request.merge_request_diff.real_size end - expose :merged_by, using: Entities::UserBasic do |merge_request, _options| - merge_request.metrics&.merged_by - end - - expose :merged_at do |merge_request, _options| - merge_request.metrics&.merged_at - end - - expose :closed_by, using: Entities::UserBasic do |merge_request, _options| - merge_request.metrics&.latest_closed_by - end - - expose :closed_at do |merge_request, _options| - merge_request.metrics&.latest_closed_at - end - expose :latest_build_started_at, if: -> (_, options) { build_available?(options) } do |merge_request, _options| merge_request.metrics&.latest_build_started_at end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 405fc30a2ed..491b5085bb8 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -8,6 +8,15 @@ module API helpers ::Gitlab::IssuableMetadata + # EE::API::Issues would override the following helpers + helpers do + params :issues_params_ee do + end + + params :issue_params_ee do + end + end + helpers do # rubocop: disable CodeReuse/ActiveRecord def find_issues(args = {}) @@ -19,7 +28,7 @@ module API args[:scope] = args[:scope].underscore if args[:scope] issues = IssuesFinder.new(current_user, args).execute - .preload(:assignees, :labels, :notes, :timelogs, :project, :author) + .preload(:assignees, :labels, :notes, :timelogs, :project, :author, :closed_by) issues.reorder(args[:order_by] => args[:sort]) end @@ -46,9 +55,11 @@ module API desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination + + use :issues_params_ee end - params :issue_params_ce do + params :issue_params do optional :description, type: String, desc: 'The description of an issue' optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue' optional :assignee_id, type: Integer, desc: '[Deprecated] The ID of a user to assign issue' @@ -57,10 +68,8 @@ module API optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" - end - params :issue_params do - use :issue_params_ce + use :issue_params_ee end end @@ -285,6 +294,30 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc 'List merge requests that are related to the issue' do + success Entities::MergeRequestBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/related_merge_requests' do + issue = find_project_issue(params[:issue_iid]) + + merge_request_iids = ::Issues::ReferencedMergeRequestsService.new(user_project, current_user) + .execute(issue) + .flatten + .map(&:iid) + + merge_requests = + if merge_request_iids.present? + MergeRequestsFinder.new(current_user, project_id: user_project.id, iids: merge_request_iids).execute + else + MergeRequest.none + end + + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + end + desc 'List merge requests closing issue' do success Entities::MergeRequestBasic end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index a617efaaa4c..16f07f16387 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -45,7 +45,7 @@ module API return merge_requests if args[:view] == 'simple' merge_requests - .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs) + .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs, metrics: [:latest_closed_by, :merged_by]) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/api/services.rb b/lib/api/services.rb index 0ae05ce08f1..1cb3b8a7277 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -298,6 +298,14 @@ module API desc: 'Title' } ], + 'discord' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'Discord webhook. e.g. https://discordapp.com/api/webhooks/…' + } + ], 'drone-ci' => [ { required: true, @@ -677,6 +685,7 @@ module API BuildkiteService, CampfireService, CustomIssueTrackerService, + DiscordService, DroneCiService, EmailsOnPushService, ExternalWikiService, diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index f1786c15f4f..1ae144ca9c1 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -14,7 +14,7 @@ module API end def public_snippets - SnippetsFinder.new(current_user, visibility: Snippet::PUBLIC).execute + SnippetsFinder.new(current_user, scope: :are_public).execute end end diff --git a/lib/api/submodules.rb b/lib/api/submodules.rb new file mode 100644 index 00000000000..72d7d994102 --- /dev/null +++ b/lib/api/submodules.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module API + class Submodules < Grape::API + before { authenticate! } + + helpers do + def commit_params(attrs) + { + submodule: attrs[:submodule], + commit_sha: attrs[:commit_sha], + branch_name: attrs[:branch], + commit_message: attrs[:commit_message] + } + end + end + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects, requirements: Files::FILE_ENDPOINT_REQUIREMENTS do + desc 'Update existing submodule reference in repository' do + success Entities::Commit + end + params do + requires :submodule, type: String, desc: 'Url encoded full path to submodule.' + requires :commit_sha, type: String, desc: 'Commit sha to update the submodule to.' + requires :branch, type: String, desc: 'Name of the branch to commit into.' + optional :commit_message, type: String, desc: 'Commit message. If no message is provided a default one will be set.' + end + put ":id/repository/submodules/:submodule", requirements: Files::FILE_ENDPOINT_REQUIREMENTS do + authorize! :push_code, user_project + + submodule_params = declared_params(include_missing: false) + + result = ::Submodules::UpdateService.new(user_project, current_user, commit_params(submodule_params)).execute + + if result[:status] == :success + commit_detail = user_project.repository.commit(result[:result]) + present commit_detail, with: Entities::CommitDetail + else + render_api_error!(result[:message], result[:http_status] || 400) + end + end + end + end +end diff --git a/lib/api/users.rb b/lib/api/users.rb index 47382b09207..2a56506f3a5 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -512,11 +512,9 @@ module API PersonalAccessTokensFinder.new({ user: user, impersonation: true }.merge(options)) end - # rubocop: disable CodeReuse/ActiveRecord def find_impersonation_token - finder.find_by(id: declared_params[:impersonation_token_id]) || not_found!('Impersonation Token') + finder.find_by_id(declared_params[:impersonation_token_id]) || not_found!('Impersonation Token') end - # rubocop: enable CodeReuse/ActiveRecord end before { authenticated_as_admin! } diff --git a/lib/banzai/filter/absolute_link_filter.rb b/lib/banzai/filter/absolute_link_filter.rb index 04ec568eee3..a9bdb004c4b 100644 --- a/lib/banzai/filter/absolute_link_filter.rb +++ b/lib/banzai/filter/absolute_link_filter.rb @@ -29,6 +29,7 @@ module Banzai end def absolute_link_attr(uri) + # Here we really want to expand relative path to absolute path URI.join(Gitlab.config.gitlab.url, uri).to_s end end diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index d7fe012883d..8e2358694d4 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -18,7 +18,7 @@ module Banzai issuables = extractor.extract([doc]) issuables.each do |node, issuable| - next if !can_read_cross_project? && issuable.project != project + next if !can_read_cross_project? && cross_reference?(issuable) if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable) node.content += " (#{issuable.state})" @@ -31,7 +31,14 @@ module Banzai private def issuable_reference?(text, issuable) - text == issuable.reference_link_text(project || group) + CGI.unescapeHTML(text) == issuable.reference_link_text(project || group) + end + + def cross_reference?(issuable) + return true if issuable.project != project + return true if issuable.respond_to?(:group) && issuable.group != group + + false end def can_read_cross_project? diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index 0a05d46db4c..341dbb74fe0 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -9,13 +9,11 @@ module Banzai # so we can avoid N+1 queries problem class IssuableExtractor - QUERY = %q( - descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")] - [@data-reference-type="issue" or @data-reference-type="merge_request"] - ).freeze - attr_reader :context + ISSUE_REFERENCE_TYPE = '@data-reference-type="issue"'.freeze + MERGE_REQUEST_REFERENCE_TYPE = '@data-reference-type="merge_request"'.freeze + # context - An instance of Banzai::RenderContext. def initialize(context) @context = context @@ -24,21 +22,38 @@ module Banzai # Returns Hash in the form { node => issuable_instance } def extract(documents) nodes = documents.flat_map do |document| - document.xpath(QUERY) + document.xpath(query) end - issue_parser = Banzai::ReferenceParser::IssueParser.new(context) + # The project or group for the issuable might be pending for deletion! + # Filter them out because we don't care about them. + issuables_for_nodes(nodes).select { |node, issuable| issuable.project || issuable.group } + end + + private - merge_request_parser = + def issuables_for_nodes(nodes) + parsers.each_with_object({}) do |parser, result| + result.merge!(parser.records_for_nodes(nodes)) + end + end + + def parsers + [ + Banzai::ReferenceParser::IssueParser.new(context), Banzai::ReferenceParser::MergeRequestParser.new(context) + ] + end - issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge( - merge_request_parser.records_for_nodes(nodes) + def query + %Q( + descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")] + [#{reference_types.join(' or ')}] ) + end - # The project for the issue/MR might be pending for deletion! - # Filter them out because we don't care about them. - issuables_for_nodes.select { |node, issuable| issuable.project } + def reference_types + [ISSUE_REFERENCE_TYPE, MERGE_REQUEST_REFERENCE_TYPE] end end end diff --git a/lib/bitbucket_server/client.rb b/lib/bitbucket_server/client.rb index 15e59f93141..83e8808db07 100644 --- a/lib/bitbucket_server/client.rb +++ b/lib/bitbucket_server/client.rb @@ -35,9 +35,9 @@ module BitbucketServer BitbucketServer::Representation::Repo.new(parsed_response) end - def repos + def repos(page_offset: 0, limit: nil) path = "/repos" - get_collection(path, :repo) + get_collection(path, :repo, page_offset: page_offset, limit: limit) end def create_branch(project_key, repo, branch_name, sha) @@ -61,8 +61,8 @@ module BitbucketServer private - def get_collection(path, type) - paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type) + def get_collection(path, type, page_offset: 0, limit: nil) + paginator = BitbucketServer::Paginator.new(connection, Addressable::URI.escape(path), type, page_offset: page_offset, limit: limit) BitbucketServer::Collection.new(paginator) rescue *SERVER_ERRORS => e raise ServerError, e diff --git a/lib/bitbucket_server/collection.rb b/lib/bitbucket_server/collection.rb index b50c5dde352..7e4b2277bbe 100644 --- a/lib/bitbucket_server/collection.rb +++ b/lib/bitbucket_server/collection.rb @@ -2,7 +2,13 @@ module BitbucketServer class Collection < Enumerator + attr_reader :paginator + + delegate :page_offset, :has_next_page?, to: :paginator + def initialize(paginator) + @paginator = paginator + super() do |yielder| loop do paginator.items.each { |item| yielder << item } @@ -12,6 +18,24 @@ module BitbucketServer lazy end + def current_page + return 1 if page_offset <= 1 + + [1, page_offset].max + end + + def prev_page + return nil unless current_page > 1 + + current_page - 1 + end + + def next_page + return nil unless has_next_page? + + current_page + 1 + end + def method_missing(method, *args) return super unless self.respond_to?(method) diff --git a/lib/bitbucket_server/connection.rb b/lib/bitbucket_server/connection.rb index 45a437844bd..7efcdcf8619 100644 --- a/lib/bitbucket_server/connection.rb +++ b/lib/bitbucket_server/connection.rb @@ -88,35 +88,19 @@ module BitbucketServer def build_url(path) return path if path.starts_with?(root_url) - url_join_paths(root_url, path) + Gitlab::Utils.append_path(root_url, path) end def root_url - url_join_paths(base_uri, "/rest/api/#{api_version}") + Gitlab::Utils.append_path(base_uri, "rest/api/#{api_version}") end def delete_url(resource, path) if resource == :branches - url_join_paths(base_uri, "/rest/branch-utils/#{api_version}#{path}") + Gitlab::Utils.append_path(base_uri, "rest/branch-utils/#{api_version}#{path}") else build_url(path) end end - - # URI.join is stupid in that slashes are important: - # - # # URI.join('http://example.com/subpath', 'hello') - # => http://example.com/hello - # - # We really want http://example.com/subpath/hello - # - def url_join_paths(*paths) - paths.map { |path| strip_slashes(path) }.join(SEPARATOR) - end - - def strip_slashes(path) - path = path[1..-1] if path.starts_with?(SEPARATOR) - path.chomp(SEPARATOR) - end end end diff --git a/lib/bitbucket_server/paginator.rb b/lib/bitbucket_server/paginator.rb index c351fb2f11f..aa5f84f44b3 100644 --- a/lib/bitbucket_server/paginator.rb +++ b/lib/bitbucket_server/paginator.rb @@ -4,34 +4,49 @@ module BitbucketServer class Paginator PAGE_LENGTH = 25 - def initialize(connection, url, type) + attr_reader :page_offset + + def initialize(connection, url, type, page_offset: 0, limit: nil) @connection = connection @type = type @url = url @page = nil + @page_offset = page_offset + @limit = limit || PAGE_LENGTH + @total = 0 end def items raise StopIteration unless has_next_page? + raise StopIteration if over_limit? @page = fetch_next_page + @total += @page.items.count @page.items end + def has_next_page? + page.nil? || page.next? + end + private - attr_reader :connection, :page, :url, :type + attr_reader :connection, :page, :url, :type, :limit - def has_next_page? - page.nil? || page.next? + def over_limit? + @limit.positive? && @total >= @limit end def next_offset - page.nil? ? 0 : page.next + page.nil? ? starting_offset : page.next + end + + def starting_offset + [0, page_offset - 1].max * limit end def fetch_next_page - parsed_response = connection.get(@url, start: next_offset, limit: PAGE_LENGTH) + parsed_response = connection.get(@url, start: next_offset, limit: @limit) Page.new(parsed_response, type) end end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index a340a276640..655278da711 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -153,7 +153,7 @@ module ExtractsPath private - # overriden in subclasses, do not remove + # overridden in subclasses, do not remove def get_id id = [params[:id] || params[:ref]] id << "/" + params[:path] unless params[:path].blank? diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 4a5f9d2839d..36fc8061d92 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -80,7 +80,7 @@ module Gitlab end # Get the first part of the email address (before @) - # In addtion in removes illegal characters + # In addition in removes illegal characters def generate_username(email) email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s end diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb new file mode 100644 index 00000000000..35bfc381180 --- /dev/null +++ b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true +# +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateClusterKubernetesNamespaceTable + include Gitlab::Database::MigrationHelpers + + BATCH_SIZE = 1_000 + + module Migratable + class KubernetesNamespace < ActiveRecord::Base + self.table_name = 'clusters_kubernetes_namespaces' + end + + class ClusterProject < ActiveRecord::Base + include EachBatch + + self.table_name = 'cluster_projects' + + belongs_to :project + + def self.with_no_kubernetes_namespace + where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id)) + end + + def namespace + slug = "#{project.path}-#{project.id}".downcase + slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') + end + + def service_account + "#{namespace}-service-account" + end + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + end + + def perform + cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index| + sql_values = sql_values_for(cluster_projects_batch) + + insert_into_cluster_kubernetes_namespace(sql_values) + end + end + + private + + def cluster_projects_with_no_kubernetes_namespace + Migratable::ClusterProject.with_no_kubernetes_namespace + end + + def sql_values_for(cluster_projects) + cluster_projects.map do |cluster_project| + values_for_cluster_project(cluster_project) + end + end + + def values_for_cluster_project(cluster_project) + { + cluster_project_id: cluster_project.id, + cluster_id: cluster_project.cluster_id, + project_id: cluster_project.project_id, + namespace: cluster_project.namespace, + service_account_name: cluster_project.service_account, + created_at: 'NOW()', + updated_at: 'NOW()' + } + end + + def insert_into_cluster_kubernetes_namespace(rows) + Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, + rows, + disable_quote: [:created_at, :updated_at]) + end + end + end +end diff --git a/lib/gitlab/background_migration/redact_links.rb b/lib/gitlab/background_migration/redact_links.rb index f5d3bcdd517..92256e59a6c 100644 --- a/lib/gitlab/background_migration/redact_links.rb +++ b/lib/gitlab/background_migration/redact_links.rb @@ -1,25 +1,14 @@ # frozen_string_literal: true # rubocop:disable Style/Documentation +require_relative 'redact_links/redactable' + module Gitlab module BackgroundMigration class RedactLinks - module Redactable - extend ActiveSupport::Concern - - def redact_field!(field) - self[field].gsub!(%r{/sent_notifications/\h{32}/unsubscribe}, '/sent_notifications/REDACTED/unsubscribe') - - if self.changed? - self.update_columns(field => self[field], - "#{field}_html" => nil) - end - end - end - class Note < ActiveRecord::Base include EachBatch - include Redactable + include ::Gitlab::BackgroundMigration::RedactLinks::Redactable self.table_name = 'notes' self.inheritance_column = :_type_disabled @@ -27,7 +16,7 @@ module Gitlab class Issue < ActiveRecord::Base include EachBatch - include Redactable + include ::Gitlab::BackgroundMigration::RedactLinks::Redactable self.table_name = 'issues' self.inheritance_column = :_type_disabled @@ -35,7 +24,7 @@ module Gitlab class MergeRequest < ActiveRecord::Base include EachBatch - include Redactable + include ::Gitlab::BackgroundMigration::RedactLinks::Redactable self.table_name = 'merge_requests' self.inheritance_column = :_type_disabled @@ -43,7 +32,7 @@ module Gitlab class Snippet < ActiveRecord::Base include EachBatch - include Redactable + include ::Gitlab::BackgroundMigration::RedactLinks::Redactable self.table_name = 'snippets' self.inheritance_column = :_type_disabled diff --git a/lib/gitlab/background_migration/redact_links/redactable.rb b/lib/gitlab/background_migration/redact_links/redactable.rb new file mode 100644 index 00000000000..baab34221f1 --- /dev/null +++ b/lib/gitlab/background_migration/redact_links/redactable.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RedactLinks + module Redactable + extend ActiveSupport::Concern + + def redact_field!(field) + self[field].gsub!(%r{/sent_notifications/\h{32}/unsubscribe}, '/sent_notifications/REDACTED/unsubscribe') + + if self.changed? + self.update_columns(field => self[field], + "#{field}_html" => nil) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb index 9941c2fe6d9..47579d46c1b 100644 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -67,7 +67,7 @@ module Gitlab .where('access_level >= ?', 20) confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id) - confidential_issues.each_batch(of: 100) do |batch| + confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch| batch.each do |issue| assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id) diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb index e5e8837221e..bc434b0cb64 100644 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb @@ -3,8 +3,8 @@ module Gitlab module BackgroundMigration - # Ensures services which previously recieved all notes events continue - # to recieve confidential ones. + # Ensures services which previously received all notes events continue + # to receive confidential ones. class SetConfidentialNoteEventsOnServices class Service < ActiveRecord::Base self.table_name = 'services' diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb index 171c8ef21b7..28d8d2c640b 100644 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb @@ -3,8 +3,8 @@ module Gitlab module BackgroundMigration - # Ensures hooks which previously recieved all notes events continue - # to recieve confidential ones. + # Ensures hooks which previously received all notes events continue + # to receive confidential ones. class SetConfidentialNoteEventsOnWebhooks class WebHook < ActiveRecord::Base self.table_name = 'web_hooks' diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 0d79594363e..f1a653a9d95 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -43,8 +43,7 @@ module Gitlab def highlighted_lines @blob.load_all_data! - @highlighted_lines ||= - Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines + @highlighted_lines ||= @blob.present.highlight.lines end def project diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index e780f8c646b..974b5ad6877 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # ANSI color library # # Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code @@ -265,7 +267,7 @@ module Gitlab def reset_state @offset = 0 @n_open_tags = 0 - @out = '' + @out = +'' reset end diff --git a/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb index ee3647f24fd..25a82086676 100644 --- a/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb +++ b/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb index fa6842cf36a..cf37d700991 100644 --- a/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb +++ b/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 551d4f4473e..08dac756cc1 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'zlib' require 'json' diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 85072a072d6..d0a80518ae8 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb index 9cd9b36c5f8..65cd935afaa 100644 --- a/lib/gitlab/ci/build/artifacts/path.rb +++ b/lib/gitlab/ci/build/artifacts/path.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb index 29a7a27c963..58adf6e506d 100644 --- a/lib/gitlab/ci/build/credentials/base.rb +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb index 2423aa8857d..fa805abb8bb 100644 --- a/lib/gitlab/ci/build/credentials/factory.rb +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb index 55eafcaed10..1c8588d9913 100644 --- a/lib/gitlab/ci/build/credentials/registry.rb +++ b/lib/gitlab/ci/build/credentials/registry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index c811f88f483..4dd932f61d4 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb index d10cc7802d4..43c46ad74af 100644 --- a/lib/gitlab/ci/build/policy.rb +++ b/lib/gitlab/ci/build/policy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index 782f6c4c0af..4c7dc947cd0 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 4aa5dc89f47..10934536536 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index f09ba42c074..ceb5210cfb5 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index 9d2a362b7d4..0698136166a 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index 0b1ebe4e048..d587c896712 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 7b7354bce16..a4f01468e8e 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Charts diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index fedaf18ef30..2fb3c4582e7 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci # diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index e80f9d2e452..ef5f25b42c0 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb index 3e87a09704e..3c2e1df9b83 100644 --- a/lib/gitlab/ci/config/entry/attributable.rb +++ b/lib/gitlab/ci/config/entry/attributable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb index f3357f85b99..b9639c83075 100644 --- a/lib/gitlab/ci/config/entry/boolean.rb +++ b/lib/gitlab/ci/config/entry/boolean.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index d7e09acbbf3..0a25057f482 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index 9f66f11be9b..d9658291ebe 100644 --- a/lib/gitlab/ci/config/entry/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index 697f622c45e..4aabf0cfa31 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb index 12a063059cb..690409ccf77 100644 --- a/lib/gitlab/ci/config/entry/coverage.rb +++ b/lib/gitlab/ci/config/entry/coverage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 0c1f9eb7cbf..07e9e1d3f67 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/factory.rb b/lib/gitlab/ci/config/entry/factory.rb index 6be8288748f..85c9c3511a4 100644 --- a/lib/gitlab/ci/config/entry/factory.rb +++ b/lib/gitlab/ci/config/entry/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index 04077fa7a61..eba203d9d06 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb index 6fc3aa385bc..dc0ede2a25f 100644 --- a/lib/gitlab/ci/config/entry/hidden.rb +++ b/lib/gitlab/ci/config/entry/hidden.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 2844be80a84..fc453b72fa5 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index f290ff3a565..c8cb3248fa7 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -12,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when start_in artifacts cache dependencies before_script after_script variables - environment coverage retry extends].freeze + environment coverage retry parallel extends].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -24,14 +26,13 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true - validates :retry, numericality: { only_integer: true, - greater_than_or_equal_to: 0, - less_than_or_equal_to: 2 } + validates :parallel, numericality: { only_integer: true, + greater_than_or_equal_to: 2, + less_than_or_equal_to: 50 } validates :when, inclusion: { in: %w[on_success on_failure always manual delayed], message: 'should be on_success, on_failure, ' \ 'always, manual or delayed' } - validates :dependencies, array_of_strings: true validates :extends, type: String end @@ -77,17 +78,21 @@ module Gitlab description: 'Artifacts configuration for this job.' entry :environment, Entry::Environment, - description: 'Environment configuration for this job.' + description: 'Environment configuration for this job.' entry :coverage, Entry::Coverage, - description: 'Coverage configuration for this job.' + description: 'Coverage configuration for this job.' + + entry :retry, Entry::Retry, + description: 'Retry configuration for this job.' helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage, :retry + :artifacts, :commands, :environment, :coverage, :retry, + :parallel attributes :script, :tags, :allow_failure, :when, :dependencies, - :retry, :extends, :start_in + :retry, :parallel, :extends, :start_in def compose!(deps = nil) super do @@ -155,7 +160,8 @@ module Gitlab environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, - retry: retry_defined? ? retry_value.to_i : nil, + retry: retry_defined? ? retry_value : nil, + parallel: parallel_defined? ? parallel_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored? } diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 96b6f2e5d6c..1535b108000 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb index f27ad0a7759..963b200c7bb 100644 --- a/lib/gitlab/ci/config/entry/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index a3d4432be82..4043629dea9 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index 26505c91be3..347089722e4 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/paths.rb b/lib/gitlab/ci/config/entry/paths.rb index 68dad161149..9580b5e2e7f 100644 --- a/lib/gitlab/ci/config/entry/paths.rb +++ b/lib/gitlab/ci/config/entry/paths.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index c92562f8c85..0535d7c1a1a 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/retry.rb b/lib/gitlab/ci/config/entry/retry.rb new file mode 100644 index 00000000000..e39cc5de229 --- /dev/null +++ b/lib/gitlab/ci/config/entry/retry.rb @@ -0,0 +1,90 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a retry config for a job. + # + class Retry < Simplifiable + strategy :SimpleRetry, if: -> (config) { config.is_a?(Integer) } + strategy :FullRetry, if: -> (config) { config.is_a?(Hash) } + + class SimpleRetry < Entry::Node + include Entry::Validatable + + validations do + validates :config, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } + end + + def value + { + max: config + } + end + + def location + 'retry' + end + end + + class FullRetry < Entry::Node + include Entry::Validatable + include Entry::Attributable + + ALLOWED_KEYS = %i[max when].freeze + attributes :max, :when + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :max, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } + + validates :when, array_of_strings_or_string: true + validates :when, + allowed_array_values: { in: FullRetry.possible_retry_when_values }, + if: -> (config) { config.when.is_a?(Array) } + validates :when, + inclusion: { in: FullRetry.possible_retry_when_values }, + if: -> (config) { config.when.is_a?(String) } + end + end + + def self.possible_retry_when_values + @possible_retry_when_values ||= ::Ci::Build.failure_reasons.keys.map(&:to_s) + ['always'] + end + + def value + super.tap do |config| + # make sure that `when` is an array, because we allow it to + # be passed as a String in config for simplicity + config[:when] = Array.wrap(config[:when]) if config[:when] + end + end + + def location + 'retry' + end + end + + class UnknownStrategy < Entry::Node + def errors + ["#{location} has to be either an integer or a hash"] + end + + def location + 'retry config' + end + end + + def self.default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/script.rb b/lib/gitlab/ci/config/entry/script.rb index 29ecd9995ca..f7d39e5cf55 100644 --- a/lib/gitlab/ci/config/entry/script.rb +++ b/lib/gitlab/ci/config/entry/script.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 3e2ebcff31a..47bf9205147 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb index 0066894e069..bdf7f80f382 100644 --- a/lib/gitlab/ci/config/entry/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb index 12764629686..9961bbfaa40 100644 --- a/lib/gitlab/ci/config/entry/simplifiable.rb +++ b/lib/gitlab/ci/config/entry/simplifiable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/stage.rb b/lib/gitlab/ci/config/entry/stage.rb index b7afaba1de8..65ab5953131 100644 --- a/lib/gitlab/ci/config/entry/stage.rb +++ b/lib/gitlab/ci/config/entry/stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/stages.rb b/lib/gitlab/ci/config/entry/stages.rb index ec187bd3732..ab184246d29 100644 --- a/lib/gitlab/ci/config/entry/stages.rb +++ b/lib/gitlab/ci/config/entry/stages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb index 1171ac10f22..77dcfa88170 100644 --- a/lib/gitlab/ci/config/entry/undefined.rb +++ b/lib/gitlab/ci/config/entry/undefined.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/unspecified.rb b/lib/gitlab/ci/config/entry/unspecified.rb index fbb2551e870..bab32489d2f 100644 --- a/lib/gitlab/ci/config/entry/unspecified.rb +++ b/lib/gitlab/ci/config/entry/unspecified.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb index e45787773a8..08a6593c980 100644 --- a/lib/gitlab/ci/config/entry/validatable.rb +++ b/lib/gitlab/ci/config/entry/validatable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 2df23a3edcd..33ffdd3a95d 100644 --- a/lib/gitlab/ci/config/entry/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index f6b4ba7843e..a1d552fb2e5 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config @@ -5,11 +7,11 @@ module Gitlab module Validators class AllowedKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - unknown_keys = record.config.try(:keys).to_a - options[:in] + unknown_keys = value.try(:keys).to_a - options[:in] if unknown_keys.any? - record.errors.add(:config, 'contains unknown keys: ' + - unknown_keys.join(', ')) + record.errors.add(attribute, "contains unknown keys: " + + unknown_keys.join(', ')) end end end @@ -22,6 +24,16 @@ module Gitlab end end + class AllowedArrayValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unkown_values = value - options[:in] + unless unkown_values.empty? + record.errors.add(attribute, "contains unknown values: " + + unkown_values.join(', ')) + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers @@ -66,6 +78,14 @@ module Gitlab end end + class HashOrIntegerValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) || value.is_a?(Integer) + record.errors.add(attribute, 'should be a hash or an integer') + end + end + end + class KeyValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 8acab605c91..6fd3cec2f5f 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb index 141d2714cb6..b4c491e84a6 100644 --- a/lib/gitlab/ci/config/loader.rb +++ b/lib/gitlab/ci/config/loader.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb new file mode 100644 index 00000000000..b7743bd2090 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + def initialize(jobs_config) + @jobs_config = jobs_config + end + + def normalize_jobs + extract_parallelized_jobs! + return @jobs_config if @parallelized_jobs.empty? + + parallelized_config = parallelize_jobs + parallelize_dependencies(parallelized_config) + end + + private + + def extract_parallelized_jobs! + @parallelized_jobs = {} + + @jobs_config.each do |job_name, config| + if config[:parallel] + @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + end + end + + @parallelized_jobs + end + + def parallelize_jobs + @jobs_config.each_with_object({}) do |(job_name, config), hash| + if @parallelized_jobs.key?(job_name) + @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) } + else + hash[job_name] = config + end + + hash + end + end + + def parallelize_dependencies(parallelized_config) + parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) + parallelized_config.each_with_object({}) do |(job_name, config), hash| + if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? + deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten + hash[job_name] = config.merge(dependencies: deps) + else + hash[job_name] = config + end + + hash + end + end + + def self.parallelize_job_names(name, total) + Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] } + end + end + end + end +end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 73f36735e35..b1db9084662 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class CronParser diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb index 0daddaa638c..58d55b1bd6f 100644 --- a/lib/gitlab/ci/mask_secret.rb +++ b/lib/gitlab/ci/mask_secret.rb @@ -1,9 +1,13 @@ +# frozen_string_literal: true + module Gitlab module Ci::MaskSecret class << self def mask!(value, token) return value unless value.present? && token.present? + # We assume 'value' must be mutable, given + # that frozen string is enabled. value.gsub!(token, 'x' * token.length) value end diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb index 3994a50772b..fbdb84c0522 100644 --- a/lib/gitlab/ci/model.rb +++ b/lib/gitlab/ci/model.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Model diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index efed19da21c..bab1c73e2f1 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index b5eb0cfa2f0..b445a872b3d 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index a53c80d34f7..05978804d92 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -1,4 +1,7 @@ -module Gitlab # rubocop:disable Naming/FileName +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module Gitlab module Ci module Pipeline module Chain diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 02493c7fe02..aa627bdb009 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline @@ -5,26 +7,11 @@ module Gitlab class Create < Chain::Base include Chain::Helpers - # rubocop: disable CodeReuse/ActiveRecord def perform! - ::Ci::Pipeline.transaction do - pipeline.save! - - ## - # Create environments before the pipeline starts. - # - pipeline.builds.each do |build| - if build.has_environment? - project.environments.find_or_create_by( - name: build.expanded_environment_name - ) - end - end - end + pipeline.save! rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") end - # rubocop: enable CodeReuse/ActiveRecord def break? !pipeline.persisted? diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index bf1380a1da9..6bb3a75291b 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index f34c11ca3c2..633d3cd4f6b 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index e24630656d3..99780409085 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index 32cbb7ca6af..b9707d2f8f5 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 13c6fedd831..ebd7e6e8289 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb index a3bd2a5a23a..28c38cc3d18 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb index 9699c24e5b6..d88851d8245 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb index 30701e1de1b..de24bbf688b 100644 --- a/lib/gitlab/ci/pipeline/duration.rb +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb index f57df7c5637..61d392121d8 100644 --- a/lib/gitlab/ci/pipeline/expression.rb +++ b/lib/gitlab/ci/pipeline/expression.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb index 047ab66e9b3..70c774416f6 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb index 3a2f0c6924e..668e85f5b9e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index 10957598f76..cd17bc4d78b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb index a2778716924..be7258c201a 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb index f640d0b5855..3ebceb92eb7 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 9b239c29ea4..d7e6dacf068 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 346c92dc51e..2db2bf011f1 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb index f2611d65faf..ef9ddb6cae9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 37643c8ef53..85c0899e4f6 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 4cacb1e62c9..f26542361a2 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index 90f94d0b763..ed184309ab4 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index b36f1e0f865..b03611f756e 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/token.rb b/lib/gitlab/ci/pipeline/expression/token.rb index 58211800b88..513d43f6fca 100644 --- a/lib/gitlab/ci/pipeline/expression/token.rb +++ b/lib/gitlab/ci/pipeline/expression/token.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb index db9706924bb..1fd3a61017f 100644 --- a/lib/gitlab/ci/pipeline/seed/base.rb +++ b/lib/gitlab/ci/pipeline/seed/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 6980b0b7aff..ef738a93bfe 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 2b58d9863a0..4775ff15581 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index b4d08ed257f..292e273a03a 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb index c87bdb4a8a2..7397ff35d46 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb index 726c6a11a81..11810bdc0a8 100644 --- a/lib/gitlab/ci/reports/test_reports_comparer.rb +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index b5f15397c0f..b0391160c15 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index 642aa593092..9cb7db5934c 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb index 6c9125647ad..45d9ba41e92 100644 --- a/lib/gitlab/ci/status/build/action.rb +++ b/lib/gitlab/ci/status/build/action.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 024047d4983..43fb5cdbbe6 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/canceled.rb b/lib/gitlab/ci/status/build/canceled.rb index c83e2734a73..0518b9e673d 100644 --- a/lib/gitlab/ci/status/build/canceled.rb +++ b/lib/gitlab/ci/status/build/canceled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index c1fc70ac266..6a75ec5c37f 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/created.rb b/lib/gitlab/ci/status/build/created.rb index 5be8e9de425..780fea23123 100644 --- a/lib/gitlab/ci/status/build/created.rb +++ b/lib/gitlab/ci/status/build/created.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb index 495227c2ffb..d74cfc1ee77 100644 --- a/lib/gitlab/ci/status/build/erased.rb +++ b/lib/gitlab/ci/status/build/erased.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 4a74d6d6ed1..6e4bfe23f2b 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 4babc23a495..d40454df737 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status @@ -12,7 +14,8 @@ module Gitlab missing_dependency_failure: 'missing dependency failure', runner_unsupported: 'unsupported runner', stale_schedule: 'stale schedule', - job_execution_timeout: 'job execution timeout' + job_execution_timeout: 'job execution timeout', + archived_failure: 'archived failure' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index ca0046fb1f7..d7570fdd3e2 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index 042da6392d3..d01b09f1398 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/pending.rb b/lib/gitlab/ci/status/build/pending.rb index 9dd9a27ad57..95f668295dd 100644 --- a/lib/gitlab/ci/status/build/pending.rb +++ b/lib/gitlab/ci/status/build/pending.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index a8b9ebf0803..c66b8ca5654 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/retried.rb b/lib/gitlab/ci/status/build/retried.rb index 6e190e4ee3c..b489dc68733 100644 --- a/lib/gitlab/ci/status/build/retried.rb +++ b/lib/gitlab/ci/status/build/retried.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 5aeb8e51480..eb6b3f21604 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index 62ad9083616..b3452eae189 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status @@ -7,7 +9,7 @@ module Gitlab { image: 'illustrations/illustrations_scheduled-job_countdown.svg', size: 'svg-394', - title: _("This is a delayed to run in ") + " #{execute_in}", + title: _("This is a delayed job to run in %{remainingTime}"), content: _("This job will automatically run after it's timer finishes. " \ "Often they are used for incremental roll-out deploys " \ "to production environments. When unscheduled it converts " \ @@ -16,21 +18,12 @@ module Gitlab end def status_tooltip - "delayed manual action (#{execute_in})" + "delayed manual action (%{remainingTime})" end def self.matches?(build, user) build.scheduled? && build.scheduled_at end - - private - - include TimeHelper - - def execute_in - remaining_seconds = [0, subject.scheduled_at - Time.now].max - duration_in_numbers(remaining_seconds) - end end end end diff --git a/lib/gitlab/ci/status/build/skipped.rb b/lib/gitlab/ci/status/build/skipped.rb index 3e678d0baee..4fe2f7b3114 100644 --- a/lib/gitlab/ci/status/build/skipped.rb +++ b/lib/gitlab/ci/status/build/skipped.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index dea838bfa39..a620e7ad126 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/unschedule.rb b/lib/gitlab/ci/status/build/unschedule.rb index e1b7b83428c..9110839cb55 100644 --- a/lib/gitlab/ci/status/build/unschedule.rb +++ b/lib/gitlab/ci/status/build/unschedule.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index e6195a60d4f..07f37732023 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index 9d6a2f51c11..ea773ee9944 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index 846f00b83dd..fface4bb97b 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb index 1e8101f8949..b72a28ed0b6 100644 --- a/lib/gitlab/ci/status/extended.rb +++ b/lib/gitlab/ci/status/extended.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb index 9307545b5b1..4169f5b3210 100644 --- a/lib/gitlab/ci/status/external/common.rb +++ b/lib/gitlab/ci/status/external/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/external/factory.rb b/lib/gitlab/ci/status/external/factory.rb index 07b15bd8d97..91fafb940a8 100644 --- a/lib/gitlab/ci/status/external/factory.rb +++ b/lib/gitlab/ci/status/external/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index 15836c699c7..3446644eff8 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index 27ce85bd3ed..770ed7d4d5a 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/group/common.rb b/lib/gitlab/ci/status/group/common.rb index cfd4329a923..0b5ea0712ca 100644 --- a/lib/gitlab/ci/status/group/common.rb +++ b/lib/gitlab/ci/status/group/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb index d118116cfc3..ee785856fdd 100644 --- a/lib/gitlab/ci/status/group/factory.rb +++ b/lib/gitlab/ci/status/group/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index fc387e2fd25..50c92add400 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index 6780780db32..cea7e6ed938 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index bf7e484ee9b..ed13a439be0 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb index 61bb07beb0f..7b34a2ea858 100644 --- a/lib/gitlab/ci/status/pipeline/common.rb +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/delayed.rb b/lib/gitlab/ci/status/pipeline/delayed.rb index 12736861c89..e61acdcd167 100644 --- a/lib/gitlab/ci/status/pipeline/delayed.rb +++ b/lib/gitlab/ci/status/pipeline/delayed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 0adf83fa197..5d1a8bbd924 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index ee13905e46d..ac7dd74cdce 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb index 3adcfa36af2..16ad1da89e3 100644 --- a/lib/gitlab/ci/status/scheduled.rb +++ b/lib/gitlab/ci/status/scheduled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index 0dbdc4de426..aaec1e1d201 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb index f60a7662075..f12daaa9676 100644 --- a/lib/gitlab/ci/status/stage/common.rb +++ b/lib/gitlab/ci/status/stage/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb index 4c37f084d07..58f4642510b 100644 --- a/lib/gitlab/ci/status/stage/factory.rb +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index 731013ec017..020f2c5b89f 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index 32b4cf43e48..6632cd9b143 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index db48b187e5e..149506ea498 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -27,7 +27,7 @@ # Continuous deployment to production is enabled by default. # If you want to deploy to staging first, set STAGING_ENABLED environment variable. # If you want to enable incremental rollout, either manual or time based, -# set INCREMENTAL_ROLLOUT_TYPE environment variable to "manual" or "timed". +# set INCREMENTAL_ROLLOUT_MODE environment variable to "manual" or "timed". # If you want to use canary deployments, set CANARY_ENABLED environment variable. # # If Auto DevOps fails to detect the proper buildpack, or if you want to @@ -49,7 +49,7 @@ variables: POSTGRES_ENABLED: "true" POSTGRES_DB: $CI_ENVIRONMENT_SLUG - KUBERNETES_VERSION: 1.8.6 + KUBERNETES_VERSION: 1.10.9 HELM_VERSION: 2.11.0 DOCKER_DRIVER: overlay2 @@ -116,12 +116,11 @@ code_quality: license_management: stage: test - image: docker:stable + image: + name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" + entrypoint: [""] allow_failure: true - services: - - docker:stable-dind script: - - setup_docker - license_management artifacts: paths: [gl-license-management-report.json] @@ -150,10 +149,10 @@ performance: only: refs: - branches - kubernetes: active except: variables: - $PERFORMANCE_DISABLED + - $KUBECONFIG == null sast: stage: test @@ -228,7 +227,6 @@ dast: only: refs: - branches - kubernetes: active variables: - $GITLAB_FEATURES =~ /\bdast\b/ except: @@ -236,6 +234,7 @@ dast: - master variables: - $DAST_DISABLED + - $KUBECONFIG == null review: stage: review @@ -257,12 +256,12 @@ review: only: refs: - branches - kubernetes: active except: refs: - master variables: - $REVIEW_DISABLED + - $KUBECONFIG == null stop_review: stage: cleanup @@ -280,12 +279,12 @@ stop_review: only: refs: - branches - kubernetes: active except: refs: - master variables: - $REVIEW_DISABLED + - $KUBECONFIG == null # Staging deploys are disabled by default since # continuous deployment to production is enabled by default @@ -309,9 +308,11 @@ staging: only: refs: - master - kubernetes: active variables: - $STAGING_ENABLED + except: + variables: + - $KUBECONFIG == null # Canaries are also disabled by default, but if you want them, # and know what the downsides are, you can enable this by setting @@ -334,9 +335,11 @@ canary: only: refs: - master - kubernetes: active variables: - $CANARY_ENABLED + except: + variables: + - $KUBECONFIG == null .production: &production_template stage: production @@ -362,13 +365,13 @@ production: only: refs: - master - kubernetes: active except: variables: - $STAGING_ENABLED - $CANARY_ENABLED - $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_MODE + - $KUBECONFIG == null production_manual: <<: *production_template @@ -377,7 +380,6 @@ production_manual: only: refs: - master - kubernetes: active variables: - $STAGING_ENABLED - $CANARY_ENABLED @@ -385,6 +387,7 @@ production_manual: variables: - $INCREMENTAL_ROLLOUT_ENABLED - $INCREMENTAL_ROLLOUT_MODE + - $KUBECONFIG == null # This job implements incremental rollout on for every push to `master`. @@ -414,13 +417,13 @@ production_manual: only: refs: - master - kubernetes: active variables: - $INCREMENTAL_ROLLOUT_MODE == "manual" - $INCREMENTAL_ROLLOUT_ENABLED except: variables: - $INCREMENTAL_ROLLOUT_MODE == "timed" + - $KUBECONFIG == null .timed_rollout_template: &timed_rollout_template <<: *rollout_template @@ -429,9 +432,11 @@ production_manual: only: refs: - master - kubernetes: active variables: - $INCREMENTAL_ROLLOUT_MODE == "timed" + except: + variables: + - $KUBECONFIG == null timed rollout 10%: <<: *timed_rollout_template @@ -525,11 +530,7 @@ rollout 100%: } function license_management() { - # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" - LICENSE_MANAGEMENT_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - docker run --volume "$PWD:/code" \ - "registry.gitlab.com/gitlab-org/security-products/license-management:$LICENSE_MANAGEMENT_VERSION" analyze /code + /run.sh analyze . } function sast() { @@ -823,7 +824,7 @@ rollout 100%: function initialize_tiller() { echo "Checking Tiller..." - export HELM_HOST=":44134" + export HELM_HOST="localhost:44134" tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & echo "Tiller is listening on ${HELM_HOST}" diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index d61ff239e13..492b3d03db2 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -15,7 +15,7 @@ # * Publishes the documentation for `master` branch. variables: - # This will supress any download for dependencies and plugins or upload messages which would clutter the console log. + # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 93e219a21f9..8eccd262db9 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Trace diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 2147f62a84a..e9b3199d56e 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + ## # This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) # source: https://gitlab.com/snippets/1685610 @@ -66,8 +68,8 @@ module Gitlab end end - def read(length = nil, outbuf = "") - out = "" + def read(length = nil, outbuf = nil) + out = [] length ||= size - tell @@ -83,17 +85,18 @@ module Gitlab length -= chunk_data.bytesize end + out = out.join + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality if outbuf - outbuf.slice!(0, outbuf.bytesize) - outbuf << out + outbuf.replace(out) end out end def readline - out = "" + out = [] until eof? data = chunk_slice_from_offset @@ -109,7 +112,7 @@ module Gitlab end end - out + out.join end def write(data) diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb index c09089d6475..f33f8cc56c1 100644 --- a/lib/gitlab/ci/trace/section_parser.rb +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Trace diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index a71040e5e56..bd40fdf59b1 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Trace @@ -129,8 +131,7 @@ module Gitlab debris = '' until (buf = read_backward(BUFFER_SIZE)).empty? - buf += debris - debris, *lines = buf.each_line.to_a + debris, *lines = (buf + debris).each_line.to_a lines.reverse_each do |line| yield(line.force_encoding(Encoding.default_external)) end diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index ad30b3f427c..a7b4e0348c2 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Variables diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 7da6d09d440..fdf852e8788 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Variables diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index a427aa30683..e6ec400e476 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class YamlProcessor @@ -50,6 +52,8 @@ module Gitlab after_script: job[:after_script], environment: job[:environment], retry: job[:retry], + parallel: job[:parallel], + instance: job[:instance], start_in: job[:start_in] }.compact } end @@ -102,7 +106,7 @@ module Gitlab ## # Jobs # - @jobs = @ci_config.jobs + @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs @jobs.each do |name, job| # logical validation for job diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 331c39f7d6b..4ed9a9a02ab 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -11,7 +11,11 @@ module Gitlab # Importantly RAM is for _all_workers (ie, the cluster), # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX worker_count = puma_options[:workers] || 1 - config.ram = worker_count * puma_per_worker_max_memory_mb + # The Puma Worker Killer checks the total RAM used by both the master + # and worker processes. Bump the limits to N+1 instead of N workers + # to account for this: + # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57 + config.ram = (worker_count + 1) * puma_per_worker_max_memory_mb config.frequency = 20 # seconds diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 30911b49b18..0ca99506311 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -1,8 +1,11 @@ +# frozen_string_literal: true + module Gitlab module Conflict class File include Gitlab::Routing include IconsHelper + include Gitlab::Utils::StrongMemoize CONTEXT_LINES = 3 @@ -30,11 +33,8 @@ module Gitlab end def highlight_lines! - their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") - our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") - - their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines - our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines + their_highlight = Gitlab::Highlight.highlight(their_path, their_lines, language: their_language).lines + our_highlight = Gitlab::Highlight.highlight(our_path, our_lines, language: our_language).lines lines.each do |line| line.rich_text = @@ -182,6 +182,34 @@ module Gitlab raw_line[:line_new], parent_file: self) end end + + def their_language + strong_memoize(:their_language) do + repository.gitattribute(their_path, 'gitlab-language') + end + end + + def our_language + strong_memoize(:our_language) do + if our_path == their_path + their_language + else + repository.gitattribute(our_path, 'gitlab-language') + end + end + end + + def their_lines + strong_memoize(:their_lines) do + lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") + end + end + + def our_lines + strong_memoize(:our_lines) do + lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") + end + end end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 65a65b67975..53406af2c4e 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Conflict class FileCollection diff --git a/lib/gitlab/cross_project_access/check_collection.rb b/lib/gitlab/cross_project_access/check_collection.rb index 88376232065..55527ba5e87 100644 --- a/lib/gitlab/cross_project_access/check_collection.rb +++ b/lib/gitlab/cross_project_access/check_collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class CrossProjectAccess class CheckCollection diff --git a/lib/gitlab/cross_project_access/check_info.rb b/lib/gitlab/cross_project_access/check_info.rb index e8a845c7f1e..2a9eacad680 100644 --- a/lib/gitlab/cross_project_access/check_info.rb +++ b/lib/gitlab/cross_project_access/check_info.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class CrossProjectAccess class CheckInfo diff --git a/lib/gitlab/cross_project_access/class_methods.rb b/lib/gitlab/cross_project_access/class_methods.rb index 90eac94800c..64ad30794d3 100644 --- a/lib/gitlab/cross_project_access/class_methods.rb +++ b/lib/gitlab/cross_project_access/class_methods.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class CrossProjectAccess module ClassMethods diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index e3e3767cc75..304d60996a6 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index 86d708be0d6..36231b187cd 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module BaseQuery diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 038d5a19bc4..e2d6a301734 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class BaseStage diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index 06357c9b377..591db3c35e6 100644 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class CodeEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb index 5f9dc9a4303..2e5f9ef5a40 100644 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class CodeStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb index 50e126cf00b..98a30a8fc97 100644 --- a/lib/gitlab/cycle_analytics/event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module EventFetcher diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index 1754f91dccb..30c6ead8968 100644 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class IssueEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb index 7b03811efb2..4eae2da512c 100644 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class IssueStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb index f5d08c0b658..3e0302d308d 100644 --- a/lib/gitlab/cycle_analytics/metrics_tables.rb +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module MetricsTables diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index 1e11e84a9cb..afefd09b614 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class Permissions diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 086203b9ccc..db8ac3becea 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class PlanEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb index 1a0afb56b4f..513e4575be0 100644 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class PlanStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb index 0fa2e87f673..6681cb42c90 100644 --- a/lib/gitlab/cycle_analytics/production_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class ProductionEventFetcher < IssueEventFetcher diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb index d0ca62e46e4..aff65b150fb 100644 --- a/lib/gitlab/cycle_analytics/production_helper.rb +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module ProductionHelper diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb index 0fa8a65cb99..6fd7214dce7 100644 --- a/lib/gitlab/cycle_analytics/production_stage.rb +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class ProductionStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index dada819a2a8..de100295281 100644 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class ReviewEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb index cfbbdc43fd9..294b656bc55 100644 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class ReviewStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb index 28e0455df59..1bd40a7aa18 100644 --- a/lib/gitlab/cycle_analytics/stage.rb +++ b/lib/gitlab/cycle_analytics/stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Stage diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index fc77bd86097..5198dd5b4eb 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class StageSummary diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb index 2f014153ca5..70ce82383b3 100644 --- a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class StagingEventFetcher < BaseEventFetcher diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb index d5684bb9201..dbc2414ff66 100644 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class StagingStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index a917ddccac7..709221c648e 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Summary diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index 550c1755a71..f0019b26fa2 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Summary diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 099d798aac6..3b56dc2a7bc 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Summary diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index 9bbf7a2685f..51695c86192 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics module Summary diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb index a2589c6601a..4d5ea5b7c34 100644 --- a/lib/gitlab/cycle_analytics/test_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class TestEventFetcher < StagingEventFetcher diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index 0e9d235ca79..c31b664148b 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class TestStage < BaseStage diff --git a/lib/gitlab/cycle_analytics/updater.rb b/lib/gitlab/cycle_analytics/updater.rb index 953268ebd46..c642809a792 100644 --- a/lib/gitlab/cycle_analytics/updater.rb +++ b/lib/gitlab/cycle_analytics/updater.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class Updater diff --git a/lib/gitlab/cycle_analytics/usage_data.rb b/lib/gitlab/cycle_analytics/usage_data.rb index 5122e3417ca..913ee373f54 100644 --- a/lib/gitlab/cycle_analytics/usage_data.rb +++ b/lib/gitlab/cycle_analytics/usage_data.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module CycleAnalytics class UsageData diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 0b71b31a476..3407380127e 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Build diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index f573368e572..65601dcdf31 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Note diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index f382992cb0a..76c8b4ec5c2 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Pipeline diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index b498f113859..9bf2f9291a8 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Push diff --git a/lib/gitlab/data_builder/repository.rb b/lib/gitlab/data_builder/repository.rb index c9c13ec6487..0e627fd623e 100644 --- a/lib/gitlab/data_builder/repository.rb +++ b/lib/gitlab/data_builder/repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module Repository diff --git a/lib/gitlab/data_builder/wiki_page.rb b/lib/gitlab/data_builder/wiki_page.rb index 226974b698c..9368446fa59 100644 --- a/lib/gitlab/data_builder/wiki_page.rb +++ b/lib/gitlab/data_builder/wiki_page.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DataBuilder module WikiPage diff --git a/lib/gitlab/database/arel_methods.rb b/lib/gitlab/database/arel_methods.rb index d7e3ce08b32..991e4152dcb 100644 --- a/lib/gitlab/database/arel_methods.rb +++ b/lib/gitlab/database/arel_methods.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module ArelMethods diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index 5f549ed2b3c..ea6529e2dc4 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # For large tables, PostgreSQL can take a long time to count rows due to MVCC. # We can optimize this by using the reltuples count as described in https://wiki.postgresql.org/wiki/Slow_Counting. module Gitlab diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb index 25e56998038..79d2caff151 100644 --- a/lib/gitlab/database/date_time.rb +++ b/lib/gitlab/database/date_time.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module DateTime diff --git a/lib/gitlab/database/grant.rb b/lib/gitlab/database/grant.rb index 7d334a79009..862ab96c887 100644 --- a/lib/gitlab/database/grant.rb +++ b/lib/gitlab/database/grant.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database # Model that can be used for querying permissions of a SQL user. diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index f64e3d53138..0da5119a3ed 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # https://www.periscopedata.com/blog/medians-in-sql.html module Gitlab module Database diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index a17f27a3147..134d1e7a724 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module MigrationHelpers @@ -879,7 +881,7 @@ module Gitlab columns(table).find { |column| column.name == name } end - # This will replace the first occurance of a string in a column with + # This will replace the first occurrence of a string in a column with # the replacement # On postgresql we can use `regexp_replace` for that. # On mysql we find the location of the pattern, and overwrite it diff --git a/lib/gitlab/database/multi_threaded_migration.rb b/lib/gitlab/database/multi_threaded_migration.rb index 7ae5a4c17c8..1d39a3d0b57 100644 --- a/lib/gitlab/database/multi_threaded_migration.rb +++ b/lib/gitlab/database/multi_threaded_migration.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module MultiThreadedMigration diff --git a/lib/gitlab/database/read_only_relation.rb b/lib/gitlab/database/read_only_relation.rb index 4571ad122ce..2362208e5dd 100644 --- a/lib/gitlab/database/read_only_relation.rb +++ b/lib/gitlab/database/read_only_relation.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database # Module that can be injected into a ActiveRecord::Relation to make it diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb index f333ff22300..2314246da55 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # This module can be included in migrations to make it easier to rename paths # of `Namespace` & `Project` models certain paths would become `reserved`. # diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index 26ae6966746..f1dc3ed74fe 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module RenameReservedPathsMigration diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb index 14de28a1d08..a5b42bbfdd9 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module RenameReservedPathsMigration diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb index 73971af6a74..6bbad707f0f 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module RenameReservedPathsMigration diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index 827aeb12a02..580be9fe267 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database module RenameReservedPathsMigration diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index b2d8ee81977..6516d6e648d 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Database BINARY_TYPE = diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index d2360583741..ac2efe598b4 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class BaseLinker diff --git a/lib/gitlab/dependency_linker/cartfile_linker.rb b/lib/gitlab/dependency_linker/cartfile_linker.rb index 4f69f2c4ab2..0e33f0956dd 100644 --- a/lib/gitlab/dependency_linker/cartfile_linker.rb +++ b/lib/gitlab/dependency_linker/cartfile_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class CartfileLinker < MethodLinker diff --git a/lib/gitlab/dependency_linker/cocoapods.rb b/lib/gitlab/dependency_linker/cocoapods.rb index 2fbde7da1b4..38eabe303de 100644 --- a/lib/gitlab/dependency_linker/cocoapods.rb +++ b/lib/gitlab/dependency_linker/cocoapods.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker module Cocoapods diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb index cfd4ec15125..22d2bead891 100644 --- a/lib/gitlab/dependency_linker/composer_json_linker.rb +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class ComposerJsonLinker < PackageJsonLinker diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index bfea836bcb2..8ab219c4962 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class GemfileLinker < MethodLinker diff --git a/lib/gitlab/dependency_linker/gemspec_linker.rb b/lib/gitlab/dependency_linker/gemspec_linker.rb index f1783ee2ab4..b924ea86d89 100644 --- a/lib/gitlab/dependency_linker/gemspec_linker.rb +++ b/lib/gitlab/dependency_linker/gemspec_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class GemspecLinker < MethodLinker diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb index fe091baee6d..d24c137793e 100644 --- a/lib/gitlab/dependency_linker/godeps_json_linker.rb +++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class GodepsJsonLinker < JsonLinker diff --git a/lib/gitlab/dependency_linker/json_linker.rb b/lib/gitlab/dependency_linker/json_linker.rb index a8ef25233d8..298d214df61 100644 --- a/lib/gitlab/dependency_linker/json_linker.rb +++ b/lib/gitlab/dependency_linker/json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class JsonLinker < BaseLinker diff --git a/lib/gitlab/dependency_linker/method_linker.rb b/lib/gitlab/dependency_linker/method_linker.rb index 0ffa2a83c93..d4d85bb3390 100644 --- a/lib/gitlab/dependency_linker/method_linker.rb +++ b/lib/gitlab/dependency_linker/method_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class MethodLinker < BaseLinker diff --git a/lib/gitlab/dependency_linker/package_json_linker.rb b/lib/gitlab/dependency_linker/package_json_linker.rb index 330c95f0880..578e25f806a 100644 --- a/lib/gitlab/dependency_linker/package_json_linker.rb +++ b/lib/gitlab/dependency_linker/package_json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class PackageJsonLinker < JsonLinker diff --git a/lib/gitlab/dependency_linker/podfile_linker.rb b/lib/gitlab/dependency_linker/podfile_linker.rb index 60ad166ea17..def9b04cca9 100644 --- a/lib/gitlab/dependency_linker/podfile_linker.rb +++ b/lib/gitlab/dependency_linker/podfile_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class PodfileLinker < GemfileLinker diff --git a/lib/gitlab/dependency_linker/podspec_json_linker.rb b/lib/gitlab/dependency_linker/podspec_json_linker.rb index d82237ed3f1..1a2493e7cc0 100644 --- a/lib/gitlab/dependency_linker/podspec_json_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_json_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class PodspecJsonLinker < JsonLinker diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb index 924e55e9820..6b1758c5a43 100644 --- a/lib/gitlab/dependency_linker/podspec_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class PodspecLinker < MethodLinker diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb index 9c9620bc36a..f630c13b760 100644 --- a/lib/gitlab/dependency_linker/requirements_txt_linker.rb +++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module DependencyLinker class RequirementsTxtLinker < BaseLinker diff --git a/lib/gitlab/diff/diff_refs.rb b/lib/gitlab/diff/diff_refs.rb index 81df47964be..d4823f60826 100644 --- a/lib/gitlab/diff/diff_refs.rb +++ b/lib/gitlab/diff/diff_refs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class DiffRefs diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index fb117baca9e..8ba44dff06f 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class File @@ -26,6 +28,7 @@ module Gitlab @repository = repository @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs + @unfolded = false # Ensure items are collected in the the batch new_blob_lazy @@ -135,6 +138,24 @@ module Gitlab Gitlab::Diff::Parser.new.parse(raw_diff.each_line, diff_file: self).to_a end + # Changes diff_lines according to the given position. That is, + # it checks whether the position requires blob lines into the diff + # in order to be presented. + def unfold_diff_lines(position) + return unless position + + unfolder = Gitlab::Diff::LinesUnfolder.new(self, position) + + if unfolder.unfold_required? + @diff_lines = unfolder.unfolded_diff_lines + @unfolded = true + end + end + + def unfolded? + @unfolded + end + def highlighted_diff_lines @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index b79ff771a2b..10df037a0dd 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module FileCollection @@ -17,7 +19,6 @@ module Gitlab @diffable = diffable @include_stats = diff_options.delete(:include_stats) - @diffs = diffable.raw_diffs(diff_options) @project = project @diff_options = diff_options @diff_refs = diff_refs @@ -25,8 +26,12 @@ module Gitlab @repository = project.repository end + def diffs + @diffs ||= diffable.raw_diffs(diff_options) + end + def diff_files - @diff_files ||= @diffs.decorate! { |diff| decorate_diff!(diff) } + @diff_files ||= diffs.decorate! { |diff| decorate_diff!(diff) } end def diff_file_with_old_path(old_path) diff --git a/lib/gitlab/diff/file_collection/commit.rb b/lib/gitlab/diff/file_collection/commit.rb index 4dc297ec036..7b1d6171e82 100644 --- a/lib/gitlab/diff/file_collection/commit.rb +++ b/lib/gitlab/diff/file_collection/commit.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module FileCollection diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb index 20d8f891cc3..586c5cf87af 100644 --- a/lib/gitlab/diff/file_collection/compare.rb +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module FileCollection diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 0dd073a3a8e..e29bf75f341 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module FileCollection diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb index 5e923b9e602..9704aed82c1 100644 --- a/lib/gitlab/diff/formatters/base_formatter.rb +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module Formatters diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb index ccd0d309972..5bc9f0c337f 100644 --- a/lib/gitlab/diff/formatters/image_formatter.rb +++ b/lib/gitlab/diff/formatters/image_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module Formatters diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb index 01c7e9f51ab..f6e247ef665 100644 --- a/lib/gitlab/diff/formatters/text_formatter.rb +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff module Formatters diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index a605ddb5c33..d2484217ab9 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class Highlight @@ -79,7 +81,7 @@ module Gitlab return [] unless blob blob.load_all_data! - Gitlab::Highlight.highlight(blob.path, blob.data, repository: repository).lines + blob.present.highlight.lines end end end diff --git a/lib/gitlab/diff/image_point.rb b/lib/gitlab/diff/image_point.rb index 1f157354ea4..a3ce032f8e2 100644 --- a/lib/gitlab/diff/image_point.rb +++ b/lib/gitlab/diff/image_point.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class ImagePoint diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 72d5ec547da..5815d1bae4a 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class InlineDiff @@ -71,7 +73,7 @@ module Gitlab def find_changed_line_pairs(lines) # Prefixes of all diff lines, indicating their types # For example: `" - + -+ ---+++ --+ -++"` - line_prefixes = lines.each_with_object("") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') + line_prefixes = lines.each_with_object(+"") { |line, s| s << (line[0] || ' ') }.gsub(/[^ +-]/, ' ') changed_line_pairs = [] line_prefixes.scan(LINE_PAIRS_PATTERN) do diff --git a/lib/gitlab/diff/inline_diff_markdown_marker.rb b/lib/gitlab/diff/inline_diff_markdown_marker.rb index c2a2eb15931..3c536c43a9e 100644 --- a/lib/gitlab/diff/inline_diff_markdown_marker.rb +++ b/lib/gitlab/diff/inline_diff_markdown_marker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class InlineDiffMarkdownMarker < Gitlab::StringRangeMarker diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 81e91ea0ab7..1bbde1ffd2a 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class InlineDiffMarker < Gitlab::StringRangeMarker diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 5b67cd46c48..f0c4977fc50 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,11 +1,13 @@ +# frozen_string_literal: true + module Gitlab module Diff class Line SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code, :type, :index, :old_pos, :new_pos + attr_reader :line_code, :type, :old_pos, :new_pos attr_writer :rich_text - attr_accessor :text + attr_accessor :text, :index def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text, @type, @index = text, type, index @@ -19,7 +21,14 @@ module Gitlab end def self.init_from_hash(hash) - new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos], line_code: hash[:line_code], rich_text: hash[:rich_text]) + new(hash[:text], + hash[:type], + hash[:index], + hash[:old_pos], + hash[:new_pos], + parent_file: hash[:parent_file], + line_code: hash[:line_code], + rich_text: hash[:rich_text]) end def to_hash diff --git a/lib/gitlab/diff/line_mapper.rb b/lib/gitlab/diff/line_mapper.rb index cf71d47df8e..fba7bff4781 100644 --- a/lib/gitlab/diff/line_mapper.rb +++ b/lib/gitlab/diff/line_mapper.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # When provided a diff for a specific file, maps old line numbers to new line # numbers and back, to find out where a specific line in a file was moved by the # changes. diff --git a/lib/gitlab/diff/lines_unfolder.rb b/lib/gitlab/diff/lines_unfolder.rb new file mode 100644 index 00000000000..9306b7e16a2 --- /dev/null +++ b/lib/gitlab/diff/lines_unfolder.rb @@ -0,0 +1,235 @@ +# frozen_string_literal: true + +# Given a position, calculates which Blob lines should be extracted, treated and +# injected in the current diff file lines in order to present a "unfolded" diff. +module Gitlab + module Diff + class LinesUnfolder + include Gitlab::Utils::StrongMemoize + + UNFOLD_CONTEXT_SIZE = 3 + + def initialize(diff_file, position) + @diff_file = diff_file + @blob = diff_file.old_blob + @position = position + @generate_top_match_line = true + @generate_bottom_match_line = true + + # These methods update `@generate_top_match_line` and + # `@generate_bottom_match_line`. + @from_blob_line = calculate_from_blob_line! + @to_blob_line = calculate_to_blob_line! + end + + # Returns merged diff lines with required blob lines with correct + # positions. + def unfolded_diff_lines + strong_memoize(:unfolded_diff_lines) do + next unless unfold_required? + + merged_diff_with_blob_lines + end + end + + # Returns the extracted lines from the old blob which should be merged + # with the current diff lines. + def blob_lines + strong_memoize(:blob_lines) do + # Blob lines, unlike diffs, doesn't start with an empty space for + # unchanged line, so the parsing and highlighting step can get fuzzy + # without the following change. + line_prefix = ' ' + blob_as_diff_lines = @blob.data.each_line.map { |line| "#{line_prefix}#{line}" } + + lines = Gitlab::Diff::Parser.new.parse(blob_as_diff_lines, diff_file: @diff_file).to_a + + from = from_blob_line - 1 + to = to_blob_line - 1 + + lines[from..to] + end + end + + def unfold_required? + strong_memoize(:unfold_required) do + next false unless @diff_file.text? + next false unless @position.unchanged? + next false if @diff_file.new_file? || @diff_file.deleted_file? + next false unless @position.old_line + # Invalid position (MR import scenario) + next false if @position.old_line > @blob.lines.size + next false if @diff_file.diff_lines.empty? + next false if @diff_file.line_for_position(@position) + next false unless unfold_line + + true + end + end + + private + + attr_reader :from_blob_line, :to_blob_line + + def merged_diff_with_blob_lines + lines = @diff_file.diff_lines + match_line = unfold_line + insert_index = bottom? ? -1 : match_line.index + + lines -= [match_line] unless bottom? + + lines.insert(insert_index, *blob_lines_with_matches) + + # The inserted blob lines have invalid indexes, so we need + # to reindex them. + reindex(lines) + + lines + end + + # Returns 'unchanged' blob lines with recalculated `old_pos` and + # `new_pos` and the recalculated new match line (needed if we for instance + # we unfolded once, but there are still folded lines). + def blob_lines_with_matches + old_pos = from_blob_line + new_pos = from_blob_line + offset + + new_blob_lines = [] + + new_blob_lines.push(top_blob_match_line) if top_blob_match_line + + blob_lines.each do |line| + new_blob_lines << Gitlab::Diff::Line.new(line.text, line.type, nil, old_pos, new_pos, + parent_file: @diff_file) + + old_pos += 1 + new_pos += 1 + end + + new_blob_lines.push(bottom_blob_match_line) if bottom_blob_match_line + + new_blob_lines + end + + def reindex(lines) + lines.each_with_index { |line, i| line.index = i } + end + + def top_blob_match_line + strong_memoize(:top_blob_match_line) do + next unless @generate_top_match_line + + old_pos = from_blob_line + new_pos = from_blob_line + offset + + build_match_line(old_pos, new_pos) + end + end + + def bottom_blob_match_line + strong_memoize(:bottom_blob_match_line) do + # The bottom line match addition is already handled on + # Diff::File#diff_lines_for_serializer + next if bottom? + next unless @generate_bottom_match_line + + position = line_after_unfold_position.old_pos + + old_pos = position + new_pos = position + offset + + build_match_line(old_pos, new_pos) + end + end + + def build_match_line(old_pos, new_pos) + blob_lines_length = blob_lines.length + old_line_ref = [old_pos, blob_lines_length].join(',') + new_line_ref = [new_pos, blob_lines_length].join(',') + new_match_line_str = "@@ -#{old_line_ref}+#{new_line_ref} @@" + + Gitlab::Diff::Line.new(new_match_line_str, 'match', nil, old_pos, new_pos) + end + + # Returns the first line position that should be extracted + # from `blob_lines`. + def calculate_from_blob_line! + return unless unfold_required? + + from = comment_position - UNFOLD_CONTEXT_SIZE + + # There's no line before the match if it's in the top-most + # position. + prev_line_number = line_before_unfold_position&.old_pos || 0 + + if from <= prev_line_number + 1 + @generate_top_match_line = false + from = prev_line_number + 1 + end + + from + end + + # Returns the last line position that should be extracted + # from `blob_lines`. + def calculate_to_blob_line! + return unless unfold_required? + + to = comment_position + UNFOLD_CONTEXT_SIZE + + return to if bottom? + + next_line_number = line_after_unfold_position.old_pos + + if to >= next_line_number - 1 + @generate_bottom_match_line = false + to = next_line_number - 1 + end + + to + end + + def offset + unfold_line.new_pos - unfold_line.old_pos + end + + def line_before_unfold_position + return unless index = unfold_line&.index + + @diff_file.diff_lines[index - 1] if index > 0 + end + + def line_after_unfold_position + return unless index = unfold_line&.index + + @diff_file.diff_lines[index + 1] if index >= 0 + end + + def bottom? + strong_memoize(:bottom) do + @position.old_line > last_line.old_pos + end + end + + # Returns the line which needed to be expanded in order to send a comment + # in `@position`. + def unfold_line + strong_memoize(:unfold_line) do + next last_line if bottom? + + @diff_file.diff_lines.find do |line| + line.old_pos > comment_position && line.type == 'match' + end + end + end + + def comment_position + @position.old_line + end + + def last_line + @diff_file.diff_lines.last + end + end + end +end diff --git a/lib/gitlab/diff/parallel_diff.rb b/lib/gitlab/diff/parallel_diff.rb index 0cb26fa45c8..77b65fea726 100644 --- a/lib/gitlab/diff/parallel_diff.rb +++ b/lib/gitlab/diff/parallel_diff.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class ParallelDiff diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 7ae7ed286ed..4a47e4b80b6 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Diff class Parser diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index f967494199e..e8f98f52111 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Defines a specific location, identified by paths line numbers and image coordinates, # within a specific diff, identified by start, head and base commit ids. module Gitlab @@ -101,6 +103,10 @@ module Gitlab @diff_refs ||= DiffRefs.new(base_sha: base_sha, start_sha: start_sha, head_sha: head_sha) end + def unfolded_diff?(repository) + diff_file(repository)&.unfolded? + end + def diff_file(repository) return @diff_file if defined?(@diff_file) @@ -134,7 +140,13 @@ module Gitlab return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) - comparison.diffs(diff_options).diff_files.first + file = comparison.diffs(diff_options).diff_files.first + + # We need to unfold diff lines according to the position in order + # to correctly calculate the line code and trace position changes. + file&.unfold_diff_lines(self) + + file end def get_formatter_class(type) diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index b68a1636814..af3df820422 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Finds the diff position in the new diff that corresponds to the same location # specified by the provided position in the old diff. module Gitlab @@ -24,7 +26,7 @@ module Gitlab # head of `feature` was commit B, resulting in the original diff A->B. # Since creation, `master` was updated to C. # Now `feature` is being updated to D, and the newly generated MR diff is C->D. - # It is possible that C and D are direct decendants of A and B respectively, + # It is possible that C and D are direct descendants of A and B respectively, # but this isn't necessarily the case as rebases and merges come into play. # # Suppose we have a diff note on the original diff A->B. Now that the MR diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb index 543e62794c5..ec38bd769a3 100644 --- a/lib/gitlab/downtime_check/message.rb +++ b/lib/gitlab/downtime_check/message.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab class DowntimeCheck class Message @@ -18,13 +20,13 @@ module Gitlab def to_s label = offline ? OFFLINE : ONLINE - message = "[#{label}]: #{path}" + message = ["[#{label}]: #{path}"] if reason? - message += ":\n\n#{reason}\n\n" + message << ":\n\n#{reason}\n\n" end - message + message.join end def reason? diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index 83440ae227d..a826519b2dd 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email class AttachmentUploader diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index e68ae60ff98..5772727e855 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -44,10 +44,26 @@ module Gitlab @project ||= Project.find_by_full_path(project_path) end + def metrics_params + super.merge(includes_patches: patch_attachments.any?) + end + private + def build_merge_request + MergeRequests::BuildService.new(project, author, merge_request_params).execute + end + def create_merge_request - merge_request = MergeRequests::BuildService.new(project, author, merge_request_params).execute + merge_request = build_merge_request + + if patch_attachments.any? + apply_patches_to_source_branch(start_branch: merge_request.target_branch) + remove_patch_attachments + # Rebuild the merge request as the source branch might just have + # been created, so we should re-validate. + merge_request = build_merge_request + end if merge_request.errors.any? merge_request @@ -59,12 +75,42 @@ module Gitlab def merge_request_params params = { source_project_id: project.id, - source_branch: mail.subject, + source_branch: source_branch, target_project_id: project.id } params[:description] = message if message.present? params end + + def apply_patches_to_source_branch(start_branch:) + patches = patch_attachments.map { |patch| patch.body.decoded } + + result = Commits::CommitPatchService + .new(project, author, branch_name: source_branch, patches: patches, start_branch: start_branch) + .execute + + if result[:status] != :success + message = "Could not apply patches to #{source_branch}:\n#{result[:message]}" + raise InvalidAttachment, message + end + end + + def remove_patch_attachments + patch_attachments.each { |patch| mail.parts.delete(patch) } + # reset the message, so it needs to be reporocessed when the attachments + # have been modified + @message = nil + end + + def patch_attachments + @patches ||= mail.attachments + .select { |attachment| attachment.filename.ends_with?('.patch') } + .sort_by(&:filename) + end + + def source_branch + @source_branch ||= mail.subject + end end end end diff --git a/lib/gitlab/email/hook/additional_headers_interceptor.rb b/lib/gitlab/email/hook/additional_headers_interceptor.rb index 064cb5e659a..aa2ef76069b 100644 --- a/lib/gitlab/email/hook/additional_headers_interceptor.rb +++ b/lib/gitlab/email/hook/additional_headers_interceptor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Hook diff --git a/lib/gitlab/email/hook/delivery_metrics_observer.rb b/lib/gitlab/email/hook/delivery_metrics_observer.rb index 1c2985f6045..c7af485fcc5 100644 --- a/lib/gitlab/email/hook/delivery_metrics_observer.rb +++ b/lib/gitlab/email/hook/delivery_metrics_observer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Hook diff --git a/lib/gitlab/email/hook/disable_email_interceptor.rb b/lib/gitlab/email/hook/disable_email_interceptor.rb index 7bb8b53f0c8..6b6b1d85109 100644 --- a/lib/gitlab/email/hook/disable_email_interceptor.rb +++ b/lib/gitlab/email/hook/disable_email_interceptor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Hook diff --git a/lib/gitlab/email/hook/email_template_interceptor.rb b/lib/gitlab/email/hook/email_template_interceptor.rb index be0c4dd862e..13f8db2051d 100644 --- a/lib/gitlab/email/hook/email_template_interceptor.rb +++ b/lib/gitlab/email/hook/email_template_interceptor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Hook diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb index 50559a48973..77f299bcade 100644 --- a/lib/gitlab/email/html_parser.rb +++ b/lib/gitlab/email/html_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email class HTMLParser diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index cd9d3a6483f..ec412e7a8b1 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Email module Message @@ -116,7 +118,7 @@ module Gitlab end def subject - subject_text = '[Git]' + subject_text = ['[Git]'] subject_text << "[#{project.full_path}]" subject_text << "[#{ref_name}]" if @action == :push subject_text << ' ' @@ -134,6 +136,8 @@ module Gitlab subject_action[0] = subject_action[0].capitalize subject_text << "#{subject_action} #{ref_type} #{ref_name}" end + + subject_text.join end end end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index d8c594ad0e7..d28f6b301fa 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver @@ -18,6 +20,7 @@ module Gitlab InvalidIssueError = Class.new(InvalidRecordError) InvalidMergeRequestError = Class.new(InvalidRecordError) UnknownIncomingEmail = Class.new(ProcessingError) + InvalidAttachment = Class.new(ProcessingError) class Receiver def initialize(raw) diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index ae6b84607d6..2743f011ca6 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # Inspired in great part by Discourse's Email::Receiver module Gitlab module Email diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index d5d35dbd97f..0341f930b9c 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module EtagCaching class Middleware diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 75167a6b088..08e30214b46 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module EtagCaching class Router diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 21172ff8d93..2395e7be026 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module EtagCaching class Store diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 4d89ee5a669..d6338b09e3d 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -8,7 +8,7 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: %r{\Areadme[^/]*\z}i, + readme: %r{\A(readme|index)[^/]*\z}i, changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, license: %r{\A((un)?licen[sc]e|copying)(\.[^/]+)?\z}i, contributing: %r{\Acontributing[^/]*\z}i, diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 455814a9159..641446b52a5 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -31,19 +31,19 @@ module Gitlab class ReferenceRewriter RewriteError = Class.new(StandardError) - def initialize(text, source_project, current_user) + def initialize(text, source_parent, current_user) @text = text - @source_project = source_project + @source_parent = source_parent @current_user = current_user @original_html = markdown(text) @pattern = Gitlab::ReferenceExtractor.references_pattern end - def rewrite(target_project) + def rewrite(target_parent) return @text unless needs_rewrite? @text.gsub(@pattern) do |reference| - unfold_reference(reference, Regexp.last_match, target_project) + unfold_reference(reference, Regexp.last_match, target_parent) end end @@ -53,14 +53,14 @@ module Gitlab private - def unfold_reference(reference, match, target_project) + def unfold_reference(reference, match, target_parent) before = @text[0...match.begin(0)] after = @text[match.end(0)..-1] referable = find_referable(reference) return reference unless referable - cross_reference = build_cross_reference(referable, target_project) + cross_reference = build_cross_reference(referable, target_parent) return reference if reference == cross_reference if cross_reference.nil? @@ -72,17 +72,17 @@ module Gitlab end def find_referable(reference) - extractor = Gitlab::ReferenceExtractor.new(@source_project, + extractor = Gitlab::ReferenceExtractor.new(@source_parent, @current_user) extractor.analyze(reference) extractor.all.first end - def build_cross_reference(referable, target_project) + def build_cross_reference(referable, target_parent) if referable.respond_to?(:project) - referable.to_reference(target_project) + referable.to_reference(target_parent) else - referable.to_reference(@source_project, target_project: target_project) + referable.to_reference(@source_parent, target_project: target_parent) end end @@ -91,7 +91,7 @@ module Gitlab end def markdown(text) - Banzai.render(text, project: @source_project, no_original_data: true) + Banzai.render(text, project: @source_parent, no_original_data: true) end end end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index f7e66697da3..b767c8a278d 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -16,14 +16,15 @@ module Gitlab @pattern = FileUploader::MARKDOWN_PATTERN end - def rewrite(target_project) + def rewrite(target_parent) return @text unless needs_rewrite? @text.gsub(@pattern) do |markdown| file = find_file(@source_project, $~[:secret], $~[:file]) break markdown unless file.try(:exists?) - moved = FileUploader.copy_to(file, target_project) + klass = target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader + moved = klass.copy_to(file, target_parent) moved.markdown_link end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 13b0bb930f4..0bd1d3420a2 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -5,6 +5,7 @@ module Gitlab class Blob include Gitlab::BlobHelper include Gitlab::EncodingHelper + extend Gitlab::Git::WrapsGitalyErrors # This number is the maximum amount of data that we want to display to # the user. We load as much as we can for encoding detection and LFS @@ -75,7 +76,7 @@ module Gitlab # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set def batch_lfs_pointers(repository, blob_ids) - repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a) end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 74cdabfed9d..2820491b65d 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -3,6 +3,7 @@ module Gitlab module Git class Commit include Gitlab::EncodingHelper + extend Gitlab::Git::WrapsGitalyErrors attr_accessor :raw_commit, :head @@ -59,7 +60,7 @@ module Gitlab # This saves us an RPC round trip. return nil if commit_id.include?(':') - commit = repo.wrapped_gitaly_errors do + commit = wrapped_gitaly_errors do repo.gitaly_commit_client.find_commit(commit_id) end @@ -100,7 +101,7 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - repo.wrapped_gitaly_errors do + wrapped_gitaly_errors do repo.gitaly_commit_client.between(base, head) end end @@ -125,7 +126,7 @@ module Gitlab # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) def find_all(repo, options = {}) - repo.wrapped_gitaly_errors do + wrapped_gitaly_errors do Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end end @@ -142,7 +143,7 @@ module Gitlab # relation to each other. The last 10 commits for a branch for example, # should go through .where def batch_by_oid(repo, oids) - repo.wrapped_gitaly_errors do + wrapped_gitaly_errors do repo.gitaly_commit_client.list_commits_by_oid(oids) end end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index ae6f554bc06..83a9fd5f81a 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -3,6 +3,8 @@ module Gitlab module Git class CommitStats + include Gitlab::Git::WrapsGitalyErrors + attr_reader :id, :additions, :deletions, :total # Instantiate a CommitStats object @@ -14,7 +16,7 @@ module Gitlab @deletions = 0 @total = 0 - repo.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_stats(repo, commit) end end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 6dc792c16b8..307f1b8cb66 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -2,6 +2,8 @@ module Gitlab module Git module Conflict class Resolver + include Gitlab::Git::WrapsGitalyErrors + ConflictSideMissing = Class.new(StandardError) ResolutionError = Class.new(StandardError) @@ -12,7 +14,7 @@ module Gitlab end def conflicts - @conflicts ||= @target_repository.wrapped_gitaly_errors do + @conflicts ||= wrapped_gitaly_errors do gitaly_conflicts_client(@target_repository).list_conflict_files.to_a end rescue GRPC::FailedPrecondition => e @@ -22,7 +24,7 @@ module Gitlab end def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) - source_repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) end end diff --git a/lib/gitlab/git/patches/collection.rb b/lib/gitlab/git/patches/collection.rb new file mode 100644 index 00000000000..ad6b5d32abc --- /dev/null +++ b/lib/gitlab/git/patches/collection.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class Collection + MAX_PATCH_SIZE = 2.megabytes + + def initialize(one_or_more_patches) + @patches = Array(one_or_more_patches).map do |patch_content| + Gitlab::Git::Patches::Patch.new(patch_content) + end + end + + def content + @patches.map(&:content).join("\n") + end + + def valid_size? + size < MAX_PATCH_SIZE + end + + # rubocop: disable CodeReuse/ActiveRecord + # `@patches` is not an `ActiveRecord` relation, but an `Enumerable` + # We're using sum from `ActiveSupport` + def size + @size ||= @patches.sum(&:size) + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/git/patches/commit_patches.rb b/lib/gitlab/git/patches/commit_patches.rb new file mode 100644 index 00000000000..c62994432d3 --- /dev/null +++ b/lib/gitlab/git/patches/commit_patches.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class CommitPatches + include Gitlab::Git::WrapsGitalyErrors + + def initialize(user, repository, branch, patch_collection) + @user, @repository, @branch, @patches = user, repository, branch, patch_collection + end + + def commit + repository.with_cache_hooks do + wrapped_gitaly_errors do + operation_service.user_commit_patches(user, branch, patches.content) + end + end + end + + private + + attr_reader :user, :repository, :branch, :patches + + def operation_service + repository.raw.gitaly_operation_client + end + end + end + end +end diff --git a/lib/gitlab/git/patches/patch.rb b/lib/gitlab/git/patches/patch.rb new file mode 100644 index 00000000000..fe6ae1b5b00 --- /dev/null +++ b/lib/gitlab/git/patches/patch.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class Patch + attr_reader :content + + def initialize(content) + @content = content + end + + def size + content.bytesize + end + end + end + end +end diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index e4743b4db0a..7f9520de5ce 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -1,13 +1,15 @@ module Gitlab module Git class RemoteMirror + include Gitlab::Git::WrapsGitalyErrors + def initialize(repository, ref_name) @repository = repository @ref_name = ref_name end def update(only_branches_matching: []) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 9df04372cc2..aaaf794668d 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -6,6 +6,7 @@ module Gitlab module Git class Repository include Gitlab::Git::RepositoryMirroring + include Gitlab::Git::WrapsGitalyErrors include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize @@ -418,13 +419,17 @@ module Gitlab end def diff_stats(left_id, right_id) + if [left_id, right_id].any? { |ref| ref.blank? || Gitlab::Git.blank_ref?(ref) } + return empty_diff_stats + end + stats = wrapped_gitaly_errors do gitaly_commit_client.diff_stats(left_id, right_id) end Gitlab::Git::DiffStatsCollection.new(stats) rescue CommandError, TypeError - Gitlab::Git::DiffStatsCollection.new([]) + empty_diff_stats end # Returns a RefName for a given SHA @@ -570,6 +575,20 @@ module Gitlab end end + def update_submodule(user:, submodule:, commit_sha:, message:, branch:) + args = { + user: user, + submodule: submodule, + commit_sha: commit_sha, + branch: branch, + message: message + } + + wrapped_gitaly_errors do + gitaly_operation_client.user_update_submodule(args) + end + end + # Delete the specified branch from the repository def delete_branch(branch_name) wrapped_gitaly_errors do @@ -719,6 +738,26 @@ module Gitlab end end + # Fetch remote for repository + # + # remote - remote name + # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication + # forced - should we use --force flag? + # no_tags - should we use --no-tags flag? + # prune - should we use --prune flag? + def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + wrapped_gitaly_errors do + gitaly_repository_client.fetch_remote( + remote, + ssh_auth: ssh_auth, + forced: forced, + no_tags: no_tags, + prune: prune, + timeout: GITLAB_PROJECTS_TIMEOUT + ) + end + end + def blob_at(sha, path) Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) end @@ -845,23 +884,9 @@ module Gitlab end def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) - rescue GRPC::NotFound => e - raise NoRepository.new(e) - rescue GRPC::InvalidArgument => e - raise ArgumentError.new(e) - rescue GRPC::BadStatus => e - raise CommandError.new(e) - end - - def wrapped_gitaly_errors(&block) - yield block - rescue GRPC::NotFound => e - raise NoRepository.new(e) - rescue GRPC::InvalidArgument => e - raise ArgumentError.new(e) - rescue GRPC::BadStatus => e - raise CommandError.new(e) + wrapped_gitaly_errors do + Gitlab::GitalyClient.migrate(method, status: status, &block) + end end def clean_stale_repository_files @@ -941,6 +966,10 @@ module Gitlab private + def empty_diff_stats + Gitlab::Git::DiffStatsCollection.new([]) + end + def uncached_has_local_branches? wrapped_gitaly_errors do gitaly_repository_client.has_local_branches? diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index e0867aeb5a7..b5b701699f0 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -2,6 +2,7 @@ module Gitlab module Git class Tree include Gitlab::EncodingHelper + extend Gitlab::Git::WrapsGitalyErrors attr_accessor :id, :root_id, :name, :path, :flat_path, :type, :mode, :commit_id, :submodule_url @@ -15,7 +16,7 @@ module Gitlab def where(repository, sha, path = nil, recursive = false) path = nil if path == '' || path == '/' - repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 7fe56979d5c..02c643d0da0 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -1,6 +1,8 @@ module Gitlab module Git class Wiki + include Gitlab::Git::WrapsGitalyErrors + DuplicatePageError = Class.new(StandardError) OperationError = Class.new(StandardError) @@ -65,37 +67,37 @@ module Gitlab end def write_page(name, format, content, commit_details) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_write_page(name, format, content, commit_details) end end def delete_page(page_path, commit_details) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_delete_page(page_path, commit_details) end end def update_page(page_path, title, format, content, commit_details) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_update_page(page_path, title, format, content, commit_details) end end def pages(limit: 0) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_get_all_pages(limit: limit) end end def page(title:, version: nil, dir: nil) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_find_page(title: title, version: version, dir: dir) end end def file(name, version) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_find_file(name, version) end end @@ -105,7 +107,7 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - versions = @repository.wrapped_gitaly_errors do + versions = wrapped_gitaly_errors do gitaly_wiki_client.page_versions(page_path, options) end @@ -127,7 +129,7 @@ module Gitlab def page_formatted_data(title:, dir: nil, version: nil) version = version&.id - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) end end diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb new file mode 100644 index 00000000000..4b161f7e6ce --- /dev/null +++ b/lib/gitlab/git/wraps_gitaly_errors.rb @@ -0,0 +1,15 @@ +module Gitlab + module Git + module WrapsGitalyErrors + def wrapped_gitaly_errors(&block) + yield block + rescue GRPC::NotFound => e + raise Gitlab::Git::Repository::NoRepository.new(e) + rescue GRPC::InvalidArgument => e + raise ArgumentError.new(e) + rescue GRPC::BadStatus => e + raise Gitlab::Git::CommandError.new(e) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 0f148614b20..4c78b790ce5 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -230,6 +230,32 @@ module Gitlab response.squash_sha end + def user_update_submodule(user:, submodule:, commit_sha:, branch:, message:) + request = Gitaly::UserUpdateSubmoduleRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + commit_sha: commit_sha, + branch: encode_binary(branch), + submodule: encode_binary(submodule), + commit_message: encode_binary(message) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_update_submodule, + request + ) + + if response.pre_receive_error.present? + raise Gitlab::Git::PreReceiveError, response.pre_receive_error + elsif response.commit_error.present? + raise Gitlab::Git::CommitError, response.commit_error + else + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + end + def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, start_branch_name, start_repository) @@ -273,6 +299,29 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end + def user_commit_patches(user, branch_name, patches) + header = Gitaly::UserApplyPatchRequest::Header.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + target_branch: encode_binary(branch_name) + ) + reader = binary_stringio(patches) + + chunks = Enumerator.new do |chunk| + chunk.yield Gitaly::UserApplyPatchRequest.new(header: header) + + until reader.eof? + patch_chunk = reader.read(MAX_MSG_SIZE) + + chunk.yield(Gitaly::UserApplyPatchRequest.new(patches: patch_chunk)) + end + end + + response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks) + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + private def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 2956ed4b911..d7b36946b65 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -349,7 +349,7 @@ module Gitlab f.write(message.data) end end - # If the file is empty means that we recieved an empty stream, we delete the file + # If the file is empty means that we received an empty stream, we delete the file FileUtils.rm(save_path) if File.zero?(save_path) end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 860c39feb64..15137140639 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -8,7 +8,10 @@ module Gitlab def add_gon_variables gon.api_version = 'v4' - gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s + gon.default_avatar_url = + Gitlab::Utils.append_path( + Gitlab.config.gitlab.url, + ActionController::Base.helpers.image_path('no_avatar.png')) gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 83095acc528..a4e60bbd828 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -4,22 +4,25 @@ module Gitlab class Highlight TIMEOUT_BACKGROUND = 30.seconds TIMEOUT_FOREGROUND = 3.seconds + MAXIMUM_TEXT_HIGHLIGHT_SIZE = 1.megabyte - def self.highlight(blob_name, blob_content, repository: nil, plain: false) - new(blob_name, blob_content, repository: repository) + def self.highlight(blob_name, blob_content, language: nil, plain: false) + new(blob_name, blob_content, language: language) .highlight(blob_content, continue: false, plain: plain) end attr_reader :blob_name - def initialize(blob_name, blob_content, repository: nil) + def initialize(blob_name, blob_content, language: nil) @formatter = Rouge::Formatters::HTMLGitlab - @repository = repository + @language = language @blob_name = blob_name @blob_content = blob_content end def highlight(text, continue: true, plain: false) + plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE + highlighted_text = highlight_text(text, continue: continue, plain: plain) highlighted_text = link_dependencies(text, highlighted_text) if blob_name highlighted_text @@ -36,11 +39,9 @@ module Gitlab private def custom_language - language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language') - - return nil unless language_name + return nil unless @language - Rouge::Lexer.find_fancy(language_name) + Rouge::Lexer.find_fancy(@language) end def highlight_text(text, continue: true, plain: false) diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb index 97dc1a987c4..9215067d973 100644 --- a/lib/gitlab/import/merge_request_helpers.rb +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -22,7 +22,7 @@ module Gitlab # additional work that is strictly necessary. merge_request_id = insert_and_return_id(attributes, project.merge_requests) - merge_request = project.merge_requests.find(merge_request_id) + merge_request = project.merge_requests.reload.find(merge_request_id) # We use .insert_and_return_id which effectively disables all callbacks. # Trigger iid logic here to make sure we track internal id values consistently. diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2bed470514b..9790818ecaf 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -92,6 +92,7 @@ excluded_attributes: - :path - :namespace_id - :creator_id + - :pool_repository_id - :import_url - :import_status - :avatar diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 3d693d23c99..99581eb0416 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -154,7 +154,7 @@ module Gitlab Project.transaction do process_sub_relation(relation, relation_item) - # For every subrelation that hangs from Project, save the associated records alltogether + # For every subrelation that hangs from Project, save the associated records altogether # This effectively batches all records per subrelation item, only keeping those in memory # We have to keep in mind that more batch granularity << Memory, but >> Slowness if save diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 4a1bdf34c3e..1cd4f9e17b7 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -2,6 +2,7 @@ module Gitlab module Kubernetes module Helm HELM_VERSION = '2.7.2'.freeze + KUBECTL_VERSION = '1.11.0'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze SERVICE_ACCOUNT = 'tiller'.freeze CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb index e21bc531444..7c026ac9e68 100644 --- a/lib/gitlab/kubernetes/helm/api.rb +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -54,7 +54,11 @@ module Gitlab def create_config_map(command) command.config_map_resource.tap do |config_map_resource| - kubeclient.create_config_map(config_map_resource) + if config_map_exists?(config_map_resource) + kubeclient.update_config_map(config_map_resource) + else + kubeclient.create_config_map(config_map_resource) + end end end @@ -88,23 +92,21 @@ module Gitlab end end - def service_account_exists?(resource) - resource_exists? do - kubeclient.get_service_account(resource.metadata.name, resource.metadata.namespace) - end + def config_map_exists?(resource) + kubeclient.get_config_map(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false end - def cluster_role_binding_exists?(resource) - resource_exists? do - kubeclient.get_cluster_role_binding(resource.metadata.name) - end + def service_account_exists?(resource) + kubeclient.get_service_account(resource.metadata.name, resource.metadata.namespace) + rescue ::Kubeclient::ResourceNotFoundError + false end - def resource_exists? - yield - rescue ::Kubeclient::HttpError => e - raise e unless e.error_code == 404 - + def cluster_role_binding_exists?(resource) + kubeclient.get_cluster_role_binding(resource.metadata.name) + rescue ::Kubeclient::ResourceNotFoundError false end end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index 6752f2cff43..f2ce24fefa1 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -10,13 +10,7 @@ module Gitlab def generate_script <<~HEREDOC - set -eo pipefail - ALPINE_VERSION=$(cat /etc/alpine-release | cut -d '.' -f 1,2) - echo http://mirror.clarkson.edu/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories - echo http://mirror1.hs-esslingen.de/pub/Mirrors/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories - apk add -U wget ca-certificates openssl >/dev/null - wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null - mv /tmp/linux-amd64/helm /usr/bin/ + set -xeo pipefail HEREDOC end diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb index c7046a9ea75..6691080deca 100644 --- a/lib/gitlab/kubernetes/helm/init_command.rb +++ b/lib/gitlab/kubernetes/helm/init_command.rb @@ -45,7 +45,7 @@ module Gitlab def init_helm_command command = %w[helm init] + init_command_flags - command.shelljoin + " >/dev/null\n" + command.shelljoin end def init_command_flags diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 1be7924d6ac..ff1c1657b98 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -4,22 +4,27 @@ module Gitlab class InstallCommand include BaseCommand - attr_reader :name, :files, :chart, :version, :repository + attr_reader :name, :files, :chart, :version, :repository, :preinstall, :postinstall - def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil) + def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil, preinstall: nil, postinstall: nil) @name = name @chart = chart @version = version @rbac = rbac @files = files @repository = repository + @preinstall = preinstall + @postinstall = postinstall end def generate_script super + [ init_command, repository_command, - script_command + repository_update_command, + preinstall_command, + install_command, + postinstall_command ].compact.join("\n") end @@ -30,17 +35,29 @@ module Gitlab private def init_command - 'helm init --client-only >/dev/null' + 'helm init --client-only' end def repository_command ['helm', 'repo', 'add', name, repository].shelljoin if repository end - def script_command + def repository_update_command + 'helm repo update' if repository + end + + def install_command command = ['helm', 'install', chart] + install_command_flags - command.shelljoin + " >/dev/null\n" + command.shelljoin + end + + def preinstall_command + preinstall.join("\n") if preinstall + end + + def postinstall_command + postinstall.join("\n") if postinstall end def install_command_flags diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 95192b11c0d..e9c621d96f0 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -25,7 +25,7 @@ module Gitlab def container_specification { name: 'helm', - image: 'alpine:3.6', + image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{Gitlab::Kubernetes::Helm::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}", env: generate_pod_env(command), command: %w(/bin/sh), args: %w(-c $(COMMAND_SCRIPT)) diff --git a/lib/gitlab/kubernetes/helm/upgrade_command.rb b/lib/gitlab/kubernetes/helm/upgrade_command.rb index 74188046739..b36315f7a82 100644 --- a/lib/gitlab/kubernetes/helm/upgrade_command.rb +++ b/lib/gitlab/kubernetes/helm/upgrade_command.rb @@ -36,7 +36,7 @@ module Gitlab private def init_command - 'helm init --client-only >/dev/null' + 'helm init --client-only' end def repository_command @@ -50,7 +50,7 @@ module Gitlab " --namespace #{::Gitlab::Kubernetes::Helm::NAMESPACE}" \ " -f /data/helm/#{name}/config/values.yaml" - "helm upgrade #{name} #{chart}#{upgrade_flags} >/dev/null\n" + "helm upgrade #{name} #{chart}#{upgrade_flags}" end def optional_version_flag diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index f266177bec1..b947f6b551e 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -16,7 +16,8 @@ module Gitlab SUPPORTED_API_GROUPS = { core: { group: 'api', version: 'v1' }, rbac: { group: 'apis/rbac.authorization.k8s.io', version: 'v1' }, - extensions: { group: 'apis/extensions', version: 'v1beta1' } + extensions: { group: 'apis/extensions', version: 'v1beta1' }, + knative: { group: 'apis/serving.knative.dev', version: 'v1alpha1' } }.freeze SUPPORTED_API_GROUPS.each do |name, params| diff --git a/lib/gitlab/kubernetes/namespace.rb b/lib/gitlab/kubernetes/namespace.rb index e6ff6160ab9..783c8a24741 100644 --- a/lib/gitlab/kubernetes/namespace.rb +++ b/lib/gitlab/kubernetes/namespace.rb @@ -10,9 +10,7 @@ module Gitlab def exists? @client.get_namespace(name) - rescue ::Kubeclient::HttpError => ke - raise ke unless ke.error_code == 404 - + rescue ::Kubeclient::ResourceNotFoundError false end diff --git a/lib/gitlab/kubernetes/role_binding.rb b/lib/gitlab/kubernetes/role_binding.rb index 4f3ee040bf2..cb0cb42d007 100644 --- a/lib/gitlab/kubernetes/role_binding.rb +++ b/lib/gitlab/kubernetes/role_binding.rb @@ -3,9 +3,8 @@ module Gitlab module Kubernetes class RoleBinding - attr_reader :role_name, :namespace, :service_account_name - - def initialize(role_name:, namespace:, service_account_name:) + def initialize(name:, role_name:, namespace:, service_account_name:) + @name = name @role_name = role_name @namespace = namespace @service_account_name = service_account_name @@ -21,14 +20,16 @@ module Gitlab private + attr_reader :name, :role_name, :namespace, :service_account_name + def metadata - { name: "gitlab-#{namespace}", namespace: namespace } + { name: name, namespace: namespace } end def role_ref { apiGroup: 'rbac.authorization.k8s.io', - kind: 'Role', + kind: 'ClusterRole', name: role_name } end diff --git a/lib/gitlab/manifest_import/manifest.rb b/lib/gitlab/manifest_import/manifest.rb index 4d6034fb956..b69b9ac4b64 100644 --- a/lib/gitlab/manifest_import/manifest.rb +++ b/lib/gitlab/manifest_import/manifest.rb @@ -63,7 +63,7 @@ module Gitlab end def repository_url(name) - URI.join(remote, name).to_s + Gitlab::Utils.append_path(remote, name) end def remote diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index 142b7d1a472..d419fa66e57 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -4,10 +4,11 @@ module Gitlab module MarkupHelper extend self - MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze - ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze - OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze + MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown].freeze + ASCIIDOC_EXTENSIONS = %w[adoc ad asciidoc].freeze + OTHER_EXTENSIONS = %w[textile rdoc org creole wiki mediawiki rst].freeze EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS + PLAIN_FILENAMES = %w[readme index].freeze # Public: Determines if a given filename is compatible with GitHub::Markup. # @@ -43,7 +44,7 @@ module Gitlab # # Returns boolean def plain?(filename) - extension(filename) == 'txt' || filename.casecmp('readme').zero? + extension(filename) == 'txt' || plain_filename?(filename) end def previewable?(filename) @@ -55,5 +56,9 @@ module Gitlab def extension(filename) File.extname(filename).downcase.delete('.') end + + def plain_filename?(filename) + PLAIN_FILENAMES.include?(filename.downcase) + end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 1fd8f147b44..6943567fb6d 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -38,7 +38,7 @@ module Gitlab def go_body(path) config = Gitlab.config - project_url = URI.join(config.gitlab.url, path) + project_url = Gitlab::Utils.append_path(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) repository_url = if Gitlab::CurrentSettings.enabled_git_access_protocol == 'ssh' diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 44025650de0..fa68dead80b 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -236,7 +236,7 @@ module Gitlab def single_line_regexp(regex) # Turns a multiline extended regexp into a single line one, - # beacuse `rake routes` breaks on multiline regexes. + # because `rake routes` breaks on multiline regexes. Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze end end diff --git a/lib/gitlab/private_commit_email.rb b/lib/gitlab/private_commit_email.rb new file mode 100644 index 00000000000..bade2248ccd --- /dev/null +++ b/lib/gitlab/private_commit_email.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module PrivateCommitEmail + TOKEN = "_private".freeze + + class << self + def regex + hostname_regexp = Regexp.escape(Gitlab::CurrentSettings.current_application_settings.commit_email_hostname) + + /\A(?<id>([0-9]+))\-([^@]+)@#{hostname_regexp}\z/ + end + + def user_id_for_email(email) + match = email&.match(regex) + return unless match + + match[:id].to_i + end + + def for_user(user) + hostname = Gitlab::CurrentSettings.current_application_settings.commit_email_hostname + + "#{user.id}-#{user.username}@#{hostname}" + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 3a202d915e3..04df881bf03 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -82,7 +82,7 @@ module Gitlab ref: ref, startline: startline, data: data.join, - project_id: project ? project.id : nil + project: project ) end diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/proxy_http_connection_adapter.rb index 82213098672..a64cb47e77e 100644 --- a/lib/gitlab/proxy_http_connection_adapter.rb +++ b/lib/gitlab/proxy_http_connection_adapter.rb @@ -4,7 +4,7 @@ # of the global setting allow_local_requests_from_hooks_and_services this adapter # will allow/block connection to internal IPs and/or urls. # -# This functionality can be overriden by providing the setting the option +# This functionality can be overridden by providing the setting the option # allow_local_requests = true in the request. For example: # Gitlab::HTTP.get('http://www.gitlab.com', allow_local_requests: true) # diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index 96415271316..c682eb22890 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -2,13 +2,14 @@ module Gitlab module QuickActions class CommandDefinition attr_accessor :name, :aliases, :description, :explanation, :params, - :condition_block, :parse_params_block, :action_block + :condition_block, :parse_params_block, :action_block, :warning def initialize(name, attributes = {}) @name = name @aliases = attributes[:aliases] || [] @description = attributes[:description] || '' + @warning = attributes[:warning] || '' @explanation = attributes[:explanation] || '' @params = attributes[:params] || [] @condition_block = attributes[:condition_block] @@ -33,11 +34,13 @@ module Gitlab def explain(context, arg) return unless available?(context) - if explanation.respond_to?(:call) - execute_block(explanation, context, arg) - else - explanation - end + message = if explanation.respond_to?(:call) + execute_block(explanation, context, arg) + else + explanation + end + + warning.empty? ? message : "#{message} (#{warning})" end def execute(context, arg) @@ -61,6 +64,7 @@ module Gitlab name: name, aliases: aliases, description: desc, + warning: warning, params: prms } end diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index d82dccd0db5..192c7ec2ff5 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -31,6 +31,10 @@ module Gitlab @description = block_given? ? block : text end + def warning(message = '') + @warning = message + end + # Allows to define params for the next quick action. # These params are shown in the autocomplete menu. # @@ -133,6 +137,7 @@ module Gitlab name, aliases: aliases, description: @description, + warning: @warning, explanation: @explanation, params: @params, condition_block: @condition_block, @@ -150,6 +155,7 @@ module Gitlab @explanation = nil @params = nil @condition_block = nil + @warning = nil @parse_params_block = nil end end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 30c6806b68e..59f8dd889aa 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -29,7 +29,7 @@ module Gitlab # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" # ``` - def extract_commands(content) + def extract_commands(content, only: nil) return [content, []] unless content content = content.dup @@ -37,7 +37,7 @@ module Gitlab commands = [] content.delete!("\r") - content.gsub!(commands_regex) do + content.gsub!(commands_regex(only: only)) do if $~[:cmd] commands << [$~[:cmd].downcase, $~[:arg]].reject(&:blank?) '' @@ -60,8 +60,8 @@ module Gitlab # It looks something like: # # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ - def commands_regex - names = command_names.map(&:to_s) + def commands_regex(only:) + names = command_names(limit_to_commands: only).map(&:to_s) @commands_regex ||= %r{ (?<code> @@ -133,10 +133,14 @@ module Gitlab [content, commands] end - def command_names + def command_names(limit_to_commands:) command_definitions.flat_map do |command| next if command.noop? + if limit_to_commands && (command.all_names & limit_to_commands).empty? + next + end + command.all_names end.compact end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 5ce3eda2ccb..458737f31eb 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -4,8 +4,10 @@ module Gitlab class SearchResults class FoundBlob include EncodingHelper + include Presentable + include BlobLanguageFromGitAttributes - attr_reader :id, :filename, :basename, :ref, :startline, :data, :project_id + attr_reader :id, :filename, :basename, :ref, :startline, :data, :project def initialize(opts = {}) @id = opts.fetch(:id, nil) @@ -15,6 +17,11 @@ module Gitlab @startline = opts.fetch(:startline, nil) @data = encode_utf8(opts.fetch(:data, nil)) @per_page = opts.fetch(:per_page, 20) + @project = opts.fetch(:project, nil) + # Some caller does not have project object (e.g. elastic search), + # yet they can trigger many calls in one go, + # causing duplicated queries. + # Allow those to just pass project_id instead. @project_id = opts.fetch(:project_id, nil) end @@ -22,8 +29,12 @@ module Gitlab filename end - def no_highlighting? - false + def project_id + @project_id || @project&.id + end + + def present + super(presenter_class: BlobPresenter) end end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 24e3866128b..8079c5882c4 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -7,7 +7,7 @@ module Gitlab end def self.context(current_user = nil) - return unless self.enabled? + return unless enabled? Raven.tags_context(locale: I18n.locale) @@ -29,14 +29,22 @@ module Gitlab # # Provide an issue URL for follow up. def self.track_exception(exception, issue_url: nil, extra: {}) + track_acceptable_exception(exception, issue_url: issue_url, extra: extra) + + raise exception if should_raise? + end + + # This should be used when you do not want to raise an exception in + # development and test. If you need development and test to behave + # just the same as production you can use this instead of + # track_exception. + def self.track_acceptable_exception(exception, issue_url: nil, extra: {}) if enabled? extra[:issue_url] = issue_url if issue_url context # Make sure we've set everything we know in the context Raven.capture_exception(exception, extra: extra) end - - raise exception if should_raise? end def self.program_context diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 16c1edb2f11..c6a6fb9b5ce 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -108,23 +108,6 @@ module Gitlab success end - # Fetch remote for repository - # - # repository - an instance of Git::Repository - # remote - remote name - # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication - # forced - should we use --force flag? - # no_tags - should we use --no-tags flag? - # - # Ex. - # fetch_remote(my_repo, "upstream") - # - def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - wrapped_gitaly_errors do - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) - end - end - # Move repository reroutes to mv_directory which is an alias for # mv_namespace. Given the underlying implementation is a move action, # indescriminate of what the folders might be. diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 25f965e843d..6396b828dc7 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -3,7 +3,7 @@ module Gitlab class IssueNew < IssueCommand def self.match(text) # we can not match \n with the dot by passing the m modifier as than - # the title and description are not seperated + # the title and description are not separated /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text) end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index cc0817bdcd2..069cd1f802a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -62,6 +62,7 @@ module Gitlab clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed), clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed), clusters_applications_runner: count(::Clusters::Applications::Runner.installed), + clusters_applications_knative: count(::Clusters::Applications::Knative.installed), in_review_folder: count(::Environment.in_review_folder), groups: count(Group), issues: count(Issue), @@ -126,7 +127,6 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def services_usage types = { - JiraService: :projects_jira_active, SlackService: :projects_slack_notifications_active, SlackSlashCommandsService: :projects_slack_slash_active, PrometheusService: :projects_prometheus_active @@ -134,6 +134,23 @@ module Gitlab results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1)) types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 } + .merge(jira_usage) + end + + def jira_usage + # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999 + # so we can just check for subdomains of atlassian.net + services = count( + Service.unscoped.where(type: :JiraService, active: true) + .group("CASE WHEN properties LIKE '%.atlassian.net%' THEN 'cloud' ELSE 'server' END"), + fallback: Hash.new(-1) + ) + + { + projects_jira_server_active: services['server'] || 0, + projects_jira_cloud_active: services['cloud'] || 0, + projects_jira_active: services['server'] == -1 ? -1 : services.values.sum + } end def count(relation, fallback: -1) diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb index bd0d24e4369..874599688bb 100644 --- a/lib/gitlab/user_extractor.rb +++ b/lib/gitlab/user_extractor.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # This class extracts all users found in a piece of text by the username or the -# email adress +# email address module Gitlab class UserExtractor @@ -14,13 +14,11 @@ module Gitlab @text = text end - # rubocop: disable CodeReuse/ActiveRecord def users return User.none unless @text.present? @users ||= User.from_union(union_relations) end - # rubocop: enable CodeReuse/ActiveRecord def usernames matches[:usernames] diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 2c92458f777..9e59137a2c0 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -16,6 +16,11 @@ module Gitlab str.force_encoding(Encoding::UTF_8) end + # Append path to host, making sure there's one single / in between + def append_path(host, path) + "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}" + end + # A slugified version of the string, suitable for inclusion in URLs and # domain names. Rules: # diff --git a/lib/google_api/auth.rb b/lib/google_api/auth.rb index e724e58e9ca..56f056fd869 100644 --- a/lib/google_api/auth.rb +++ b/lib/google_api/auth.rb @@ -16,7 +16,7 @@ module GoogleApi client.auth_code.authorize_url( redirect_uri: redirect_uri, scope: scope, - state: state # This is used for arbitary redirection + state: state # This is used for arbitrary redirection ) end diff --git a/lib/json_web_token/hmac_token.rb b/lib/json_web_token/hmac_token.rb new file mode 100644 index 00000000000..ceb1b9c913f --- /dev/null +++ b/lib/json_web_token/hmac_token.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'jwt' + +module JSONWebToken + class HMACToken < Token + IAT_LEEWAY = 60 + JWT_ALGORITHM = 'HS256' + + def initialize(secret) + super() + + @secret = secret + end + + def self.decode(token, secret, leeway: IAT_LEEWAY, verify_iat: true) + JWT.decode(token, secret, true, leeway: leeway, verify_iat: verify_iat, algorithm: JWT_ALGORITHM) + end + + def encoded + JWT.encode(payload, secret, JWT_ALGORITHM) + end + + private + + attr_reader :secret + end +end diff --git a/lib/json_web_token/token.rb b/lib/json_web_token/token.rb index ce5d6f248d0..c59beef02c9 100644 --- a/lib/json_web_token/token.rb +++ b/lib/json_web_token/token.rb @@ -1,17 +1,22 @@ # frozen_string_literal: true +require 'securerandom' + module JSONWebToken class Token attr_accessor :issuer, :subject, :audience, :id attr_accessor :issued_at, :not_before, :expire_time + DEFAULT_NOT_BEFORE_TIME = 5 + DEFAULT_EXPIRE_TIME = 60 + def initialize @id = SecureRandom.uuid @issued_at = Time.now # we give a few seconds for time shift - @not_before = issued_at - 5.seconds + @not_before = issued_at - DEFAULT_NOT_BEFORE_TIME # default 60 seconds should be more than enough for this authentication token - @expire_time = issued_at + 1.minute + @expire_time = issued_at + DEFAULT_EXPIRE_TIME @custom_payload = {} end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index e5b5f3548e4..a2c3e32948f 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,6 +1,7 @@ namespace :gitlab do desc 'GitLab | Check the configuration of GitLab and its environment' task check: %w{gitlab:gitlab_shell:check + gitlab:gitaly:check gitlab:sidekiq:check gitlab:incoming_email:check gitlab:ldap:check @@ -44,13 +45,6 @@ namespace :gitlab do start_checking "GitLab Shell" check_gitlab_shell - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - check_repo_base_exists - check_repo_base_is_not_symlink - check_repo_base_user_and_group - check_repo_base_permissions - check_repos_hooks_directory_is_link - end check_gitlab_shell_self_test finished_checking "GitLab Shell" @@ -59,152 +53,6 @@ namespace :gitlab do # Checks ######################## - def check_repo_base_exists - puts "Repo base directory exists?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - if File.exist?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - - def check_repo_base_is_not_symlink - puts "Repo storage directories are symlinks?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - unless File.exist?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - break - end - - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun - end - end - end - - def check_repo_base_permissions - puts "Repo paths access is drwxrws---?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - unless File.exist?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - break - end - - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - - def check_repo_base_user_and_group - gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user - puts "Repo paths owned by #{gitlab_shell_ssh_user}:root, or #{gitlab_shell_ssh_user}:#{Gitlab.config.gitlab_shell.owner_group}?" - - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_base_path = repository_storage.legacy_disk_path - print "#{name}... " - - unless File.exist?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - break - end - - user_id = uid_for(gitlab_shell_ssh_user) - root_group_id = gid_for('root') - group_ids = [root_group_id, gid_for(Gitlab.config.gitlab_shell.owner_group)] - if File.stat(repo_base_path).uid == user_id && group_ids.include?(File.stat(repo_base_path).gid) - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{user_id}. Groupd id for root: #{root_group_id}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:root #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - - def check_repos_hooks_directory_is_link - print "hooks directories in repos are links: ... " - - gitlab_shell_hooks_path = Gitlab.config.gitlab_shell.hooks_path - - unless Project.count > 0 - puts "can't check, you have no projects".color(:magenta) - return - end - - puts "" - - Project.find_each(batch_size: 100) do |project| - print sanitized_message(project) - project_hook_directory = File.join(project.repository.path_to_repo, "hooks") - - if project.empty_repo? - puts "repository is empty".color(:magenta) - elsif File.directory?(project_hook_directory) && File.directory?(gitlab_shell_hooks_path) && - (File.realpath(project_hook_directory) == File.realpath(gitlab_shell_hooks_path)) - puts 'ok'.color(:green) - else - puts "wrong or missing hooks".color(:red) - try_fixing_it( - sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"), - 'Check the hooks_path in config/gitlab.yml', - 'Check your gitlab-shell installation' - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun - end - end - end - def check_gitlab_shell_self_test gitlab_shell_repo_base = gitlab_shell_path check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) @@ -247,6 +95,26 @@ namespace :gitlab do end end + namespace :gitaly do + desc 'GitLab | Check the health of Gitaly' + task check: :gitlab_environment do + warn_user_is_not_gitlab + start_checking 'Gitaly' + + Gitlab::HealthChecks::GitalyCheck.readiness.each do |result| + print "#{result.labels[:shard]} ... " + + if result.success + puts 'OK'.color(:green) + else + puts "FAIL: #{result.message}".color(:red) + end + end + + finished_checking 'Gitaly' + end + end + namespace :sidekiq do desc "GitLab | Check the configuration of Sidekiq" task check: :gitlab_environment do |