diff options
| author | Chris Baumbauer <cab@cabnetworks.net> | 2018-11-02 08:39:25 -0700 |
|---|---|---|
| committer | Chris Baumbauer <cab@cabnetworks.net> | 2018-11-02 08:39:25 -0700 |
| commit | dc078c241765cfea5f49409407b82db7296c132d (patch) | |
| tree | deeabb4b302f896b60ffe8e27163a4dddf625c4b /lib | |
| parent | 28a9bbceb7ab5ffe2305db02052d663b1e68ab8e (diff) | |
| parent | a80ee886e4b4a41ace5fb796bf920c4e395bf4dd (diff) | |
| download | gitlab-ce-dc078c241765cfea5f49409407b82db7296c132d.tar.gz | |
Merge branch 'master' into triggermesh-phase1-knative
Diffstat (limited to 'lib')
158 files changed, 1027 insertions, 274 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 18c30723d73..9f7be27b047 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -160,13 +160,27 @@ module API # (fixed in https://github.com/rails/rails/pull/25976). project.tags.map(&:name).sort end + expose :ssh_url_to_repo, :http_url_to_repo, :web_url, :readme_url + + expose :license_url, if: :license do |project| + license = project.repository.license_blob + + if license + Gitlab::Routing.url_helpers.project_blob_url(project, File.join(project.default_branch, license.path)) + end + end + + expose :license, with: 'API::Entities::LicenseBasic', if: :license do |project| + project.repository.license + end + expose :avatar_url do |project, options| project.avatar_url(only_path: false) end + expose :star_count, :forks_count expose :last_activity_at - expose :namespace, using: 'API::Entities::NamespaceBasic' expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes @@ -1208,11 +1222,14 @@ module API expose :deployable, using: Entities::Job end - class License < Grape::Entity + class LicenseBasic < Grape::Entity expose :key, :name, :nickname - expose :popular?, as: :popular expose :url, as: :html_url expose(:source_url) { |license| license.meta['source'] } + end + + class License < LicenseBasic + expose :popular?, as: :popular expose(:description) { |license| license.meta['description'] } expose(:conditions) { |license| license.meta['conditions'] } expose(:permissions) { |license| license.meta['permissions'] } diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb index 23b1cd1ad45..1058f4e8a5e 100644 --- a/lib/api/helpers/custom_validators.rb +++ b/lib/api/helpers/custom_validators.rb @@ -10,8 +10,21 @@ module API raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) end end + + class IntegerNoneAny < Grape::Validations::Base + def validate_param!(attr_name, params) + value = params[attr_name] + + return if value.is_a?(Integer) || + [IssuableFinder::FILTER_NONE, IssuableFinder::FILTER_ANY].include?(value.to_s.downcase) + + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], + message: "should be an integer, 'None' or 'Any'" + end + end end end end Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) +Grape::Validations.register_validator(:integer_none_any, ::API::Helpers::CustomValidators::IntegerNoneAny) diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 4dd6b19e353..ae40b5f7557 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -65,6 +65,8 @@ module API result rescue Gitlab::GitAccess::UnauthorizedError => e break response_with_status(code: 401, success: false, message: e.message) + rescue Gitlab::GitAccess::TimeoutError => e + break response_with_status(code: 503, success: false, message: e.message) rescue Gitlab::GitAccess::NotFoundError => e break response_with_status(code: 404, success: false, message: e.message) end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 25d78053c88..7909f9c7a00 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -8,6 +8,15 @@ module API helpers ::Gitlab::IssuableMetadata + # EE::API::Issues would override the following helpers + helpers do + params :issues_params_ee do + end + + params :issue_params_ee do + end + end + helpers do # rubocop: disable CodeReuse/ActiveRecord def find_issues(args = {}) @@ -40,14 +49,17 @@ module API optional :updated_after, type: DateTime, desc: 'Return issues updated after the specified time' optional :updated_before, type: DateTime, desc: 'Return issues updated before the specified time' optional :author_id, type: Integer, desc: 'Return issues which are authored by the user with the given ID' - optional :assignee_id, type: Integer, desc: 'Return issues which are assigned to the user with the given ID' + optional :assignee_id, types: [Integer, String], integer_none_any: true, + desc: 'Return issues which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], desc: 'Return issues for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' use :pagination + + use :issues_params_ee end - params :issue_params_ce do + params :issue_params do optional :description, type: String, desc: 'The description of an issue' optional :assignee_ids, type: Array[Integer], desc: 'The array of user IDs to assign issue' optional :assignee_id, type: Integer, desc: '[Deprecated] The ID of a user to assign issue' @@ -56,10 +68,8 @@ module API optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' optional :discussion_locked, type: Boolean, desc: " Boolean parameter indicating if the issue's discussion is locked" - end - params :issue_params do - use :issue_params_ce + use :issue_params_ee end end @@ -284,6 +294,30 @@ module API end # rubocop: enable CodeReuse/ActiveRecord + desc 'List merge requests that are related to the issue' do + success Entities::MergeRequestBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/related_merge_requests' do + issue = find_project_issue(params[:issue_iid]) + + merge_request_iids = ::Issues::ReferencedMergeRequestsService.new(user_project, current_user) + .execute(issue) + .flatten + .map(&:iid) + + merge_requests = + if merge_request_iids.present? + MergeRequestsFinder.new(current_user, project_id: user_project.id, iids: merge_request_iids).execute + else + MergeRequest.none + end + + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + end + desc 'List merge requests closing issue' do success Entities::MergeRequestBasic end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 440d94ae186..a617efaaa4c 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -89,7 +89,8 @@ module API optional :updated_before, type: DateTime, desc: 'Return merge requests updated before the specified time' optional :view, type: String, values: %w[simple], desc: 'If simple, returns the `iid`, URL, title, description, and basic state of merge request' optional :author_id, type: Integer, desc: 'Return merge requests which are authored by the user with the given ID' - optional :assignee_id, type: Integer, desc: 'Return merge requests which are assigned to the user with the given ID' + optional :assignee_id, types: [Integer, String], integer_none_any: true, + desc: 'Return merge requests which are assigned to the user with the given ID' optional :scope, type: String, values: %w[created-by-me assigned-to-me created_by_me assigned_to_me all], desc: 'Return merge requests for the given scope: `created_by_me`, `assigned_to_me` or `all`' optional :my_reaction_emoji, type: String, desc: 'Return issues reacted by the authenticated user by the given emoji' diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ae2d327e45b..0a914f9012e 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -114,7 +114,8 @@ module API options = options.reverse_merge( with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, statistics: params[:statistics], - current_user: current_user + current_user: current_user, + license: false ) options[:with] = Entities::BasicProjectDetails if params[:simple] @@ -230,13 +231,17 @@ module API params do use :statistics_params use :with_custom_attributes + + optional :license, type: Boolean, default: false, + desc: 'Include project license data' end get ":id" do options = { with: current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails, current_user: current_user, user_can_admin_project: can?(current_user, :admin_project, user_project), - statistics: params[:statistics] + statistics: params[:statistics], + license: params[:license] } project, options = with_custom_attributes(user_project, options) diff --git a/lib/api/runner.rb b/lib/api/runner.rb index d8768a54986..2f15f3a7d76 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -142,8 +142,7 @@ module API requires :id, type: Integer, desc: %q(Job's ID) optional :trace, type: String, desc: %q(Job's full trace) optional :state, type: String, desc: %q(Job's status: success, failed) - optional :failure_reason, type: String, values: CommitStatus.failure_reasons.keys, - desc: %q(Job's failure_reason) + optional :failure_reason, type: String, desc: %q(Job's failure_reason) end put '/:id' do job = authenticate_job! diff --git a/lib/api/validations/types/safe_file.rb b/lib/api/validations/types/safe_file.rb new file mode 100644 index 00000000000..53b5790bfa2 --- /dev/null +++ b/lib/api/validations/types/safe_file.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# This module overrides the Grape type validator defined in +# https://github.com/ruby-grape/grape/blob/master/lib/grape/validations/types/file.rb +module API + module Validations + module Types + class SafeFile < ::Grape::Validations::Types::File + def value_coerced?(value) + super && value[:tempfile].is_a?(Tempfile) + end + end + end + end +end diff --git a/lib/api/wikis.rb b/lib/api/wikis.rb index 6e1d4eb335f..24746f4efc6 100644 --- a/lib/api/wikis.rb +++ b/lib/api/wikis.rb @@ -6,7 +6,7 @@ module API def commit_params(attrs) { file_name: attrs[:file][:filename], - file_content: File.read(attrs[:file][:tempfile]), + file_content: attrs[:file][:tempfile].read, branch_name: attrs[:branch] } end @@ -100,7 +100,7 @@ module API success Entities::WikiAttachment end params do - requires :file, type: File, desc: 'The attachment file to be uploaded' + requires :file, type: ::API::Validations::Types::SafeFile, desc: 'The attachment file to be uploaded' optional :branch, type: String, desc: 'The name of the branch' end post ":id/wikis/attachments", requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index d7fe012883d..8e2358694d4 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -18,7 +18,7 @@ module Banzai issuables = extractor.extract([doc]) issuables.each do |node, issuable| - next if !can_read_cross_project? && issuable.project != project + next if !can_read_cross_project? && cross_reference?(issuable) if VISIBLE_STATES.include?(issuable.state) && issuable_reference?(node.inner_html, issuable) node.content += " (#{issuable.state})" @@ -31,7 +31,14 @@ module Banzai private def issuable_reference?(text, issuable) - text == issuable.reference_link_text(project || group) + CGI.unescapeHTML(text) == issuable.reference_link_text(project || group) + end + + def cross_reference?(issuable) + return true if issuable.project != project + return true if issuable.respond_to?(:group) && issuable.group != group + + false end def can_read_cross_project? diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index 0a05d46db4c..341dbb74fe0 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -9,13 +9,11 @@ module Banzai # so we can avoid N+1 queries problem class IssuableExtractor - QUERY = %q( - descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")] - [@data-reference-type="issue" or @data-reference-type="merge_request"] - ).freeze - attr_reader :context + ISSUE_REFERENCE_TYPE = '@data-reference-type="issue"'.freeze + MERGE_REQUEST_REFERENCE_TYPE = '@data-reference-type="merge_request"'.freeze + # context - An instance of Banzai::RenderContext. def initialize(context) @context = context @@ -24,21 +22,38 @@ module Banzai # Returns Hash in the form { node => issuable_instance } def extract(documents) nodes = documents.flat_map do |document| - document.xpath(QUERY) + document.xpath(query) end - issue_parser = Banzai::ReferenceParser::IssueParser.new(context) + # The project or group for the issuable might be pending for deletion! + # Filter them out because we don't care about them. + issuables_for_nodes(nodes).select { |node, issuable| issuable.project || issuable.group } + end + + private - merge_request_parser = + def issuables_for_nodes(nodes) + parsers.each_with_object({}) do |parser, result| + result.merge!(parser.records_for_nodes(nodes)) + end + end + + def parsers + [ + Banzai::ReferenceParser::IssueParser.new(context), Banzai::ReferenceParser::MergeRequestParser.new(context) + ] + end - issuables_for_nodes = issue_parser.records_for_nodes(nodes).merge( - merge_request_parser.records_for_nodes(nodes) + def query + %Q( + descendant-or-self::a[contains(concat(" ", @class, " "), " gfm ")] + [#{reference_types.join(' or ')}] ) + end - # The project for the issue/MR might be pending for deletion! - # Filter them out because we don't care about them. - issuables_for_nodes.select { |node, issuable| issuable.project } + def reference_types + [ISSUE_REFERENCE_TYPE, MERGE_REQUEST_REFERENCE_TYPE] end end end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index a340a276640..655278da711 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -153,7 +153,7 @@ module ExtractsPath private - # overriden in subclasses, do not remove + # overridden in subclasses, do not remove def get_id id = [params[:id] || params[:ref]] id << "/" + params[:path] unless params[:path].blank? diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index d2029a141e7..6eb5f9e2300 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -151,17 +151,15 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord def personal_access_token_check(password) return unless password.present? - token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) + token = PersonalAccessTokensFinder.new(state: 'active').find_by_token(password) if token && valid_scoped_token?(token, available_scopes) Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end - # rubocop: enable CodeReuse/ActiveRecord def valid_oauth_token?(token) token && token.accessible? && valid_scoped_token?(token, [:api]) diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 4a5f9d2839d..36fc8061d92 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -80,7 +80,7 @@ module Gitlab end # Get the first part of the email address (before @) - # In addtion in removes illegal characters + # In addition in removes illegal characters def generate_username(email) email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 5df6db6f366..c304adc64db 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -73,7 +73,6 @@ module Gitlab end end - # rubocop: disable CodeReuse/ActiveRecord def find_personal_access_token token = current_request.params[PRIVATE_TOKEN_PARAM].presence || @@ -82,9 +81,8 @@ module Gitlab return unless token # Expiration, revocation and scopes are verified in `validate_access_token!` - PersonalAccessToken.find_by(token: token) || raise(UnauthorizedError) + PersonalAccessToken.find_by_token(token) || raise(UnauthorizedError) end - # rubocop: enable CodeReuse/ActiveRecord def find_oauth_access_token token = Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) diff --git a/lib/gitlab/background_migration/digest_column.rb b/lib/gitlab/background_migration/digest_column.rb new file mode 100644 index 00000000000..22a3bb8f8f3 --- /dev/null +++ b/lib/gitlab/background_migration/digest_column.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# rubocop:disable Style/Documentation +module Gitlab + module BackgroundMigration + class DigestColumn + class PersonalAccessToken < ActiveRecord::Base + self.table_name = 'personal_access_tokens' + end + + def perform(model, attribute_from, attribute_to, start_id, stop_id) + model = model.constantize if model.is_a?(String) + + model.transaction do + relation = model.where(id: start_id..stop_id).where.not(attribute_from => nil).lock + + relation.each do |instance| + instance.update_columns(attribute_to => Gitlab::CryptoHelper.sha256(instance.read_attribute(attribute_from)), + attribute_from => nil) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/redact_links.rb b/lib/gitlab/background_migration/redact_links.rb new file mode 100644 index 00000000000..f5d3bcdd517 --- /dev/null +++ b/lib/gitlab/background_migration/redact_links.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class RedactLinks + module Redactable + extend ActiveSupport::Concern + + def redact_field!(field) + self[field].gsub!(%r{/sent_notifications/\h{32}/unsubscribe}, '/sent_notifications/REDACTED/unsubscribe') + + if self.changed? + self.update_columns(field => self[field], + "#{field}_html" => nil) + end + end + end + + class Note < ActiveRecord::Base + include EachBatch + include Redactable + + self.table_name = 'notes' + self.inheritance_column = :_type_disabled + end + + class Issue < ActiveRecord::Base + include EachBatch + include Redactable + + self.table_name = 'issues' + self.inheritance_column = :_type_disabled + end + + class MergeRequest < ActiveRecord::Base + include EachBatch + include Redactable + + self.table_name = 'merge_requests' + self.inheritance_column = :_type_disabled + end + + class Snippet < ActiveRecord::Base + include EachBatch + include Redactable + + self.table_name = 'snippets' + self.inheritance_column = :_type_disabled + end + + def perform(model_name, field, start_id, stop_id) + link_pattern = "%/sent_notifications/" + ("_" * 32) + "/unsubscribe%" + model = "Gitlab::BackgroundMigration::RedactLinks::#{model_name}".constantize + + model.where("#{field} like ?", link_pattern).where(id: start_id..stop_id).each do |resource| + resource.redact_field!(field) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb index e5e8837221e..bc434b0cb64 100644 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb @@ -3,8 +3,8 @@ module Gitlab module BackgroundMigration - # Ensures services which previously recieved all notes events continue - # to recieve confidential ones. + # Ensures services which previously received all notes events continue + # to receive confidential ones. class SetConfidentialNoteEventsOnServices class Service < ActiveRecord::Base self.table_name = 'services' diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb index 171c8ef21b7..28d8d2c640b 100644 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb @@ -3,8 +3,8 @@ module Gitlab module BackgroundMigration - # Ensures hooks which previously recieved all notes events continue - # to recieve confidential ones. + # Ensures hooks which previously received all notes events continue + # to receive confidential ones. class SetConfidentialNoteEventsOnWebhooks class WebHook < ActiveRecord::Base self.table_name = 'web_hooks' diff --git a/lib/gitlab/blame.rb b/lib/gitlab/blame.rb index 0d79594363e..f1a653a9d95 100644 --- a/lib/gitlab/blame.rb +++ b/lib/gitlab/blame.rb @@ -43,8 +43,7 @@ module Gitlab def highlighted_lines @blob.load_all_data! - @highlighted_lines ||= - Gitlab::Highlight.highlight(@blob.path, @blob.data, repository: repository).lines + @highlighted_lines ||= @blob.present.highlight.lines end def project diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index b369b9e7600..dfbb83f7bb9 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -42,7 +42,7 @@ module Gitlab end def self.cache_key_for_project(project) - "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:projects/#{project.id}/pipeline_status" + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:projects/#{project.id}/pipeline_status/#{project.commit&.sha}" end def self.update_for_pipeline(pipeline) @@ -84,9 +84,7 @@ module Gitlab def load_from_project return unless commit - self.sha = commit.sha - self.status = commit.status - self.ref = project.default_branch + self.sha, self.status, self.ref = commit.sha, commit.status, project.default_branch end # We only cache the status for the HEAD commit of a project @@ -104,6 +102,8 @@ module Gitlab def load_from_cache Gitlab::Redis::Cache.with do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) + + self.status = nil if self.status.empty? end end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 49e7f7e1fd7..074afe9c412 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -18,11 +18,24 @@ module Gitlab lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' }.freeze - attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name + LOG_MESSAGES = { + push_checks: "Checking if you are allowed to push...", + delete_default_branch_check: "Checking if default branch is being deleted...", + protected_branch_checks: "Checking if you are force pushing to a protected branch...", + protected_branch_push_checks: "Checking if you are allowed to push to the protected branch...", + protected_branch_deletion_checks: "Checking if you are allowed to delete the protected branch...", + tag_checks: "Checking if you are allowed to change existing tags...", + protected_tag_checks: "Checking if you are creating, updating or deleting a protected tag...", + lfs_objects_exist_check: "Scanning repository for blobs stored in LFS and verifying their files have been uploaded to GitLab...", + commits_check_file_paths_validation: "Validating commits' file paths...", + commits_check: "Validating commit contents..." + }.freeze + + attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name, :logger def initialize( change, user_access:, project:, skip_authorization: false, - skip_lfs_integrity_check: false, protocol: + skip_lfs_integrity_check: false, protocol:, logger: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @@ -32,6 +45,9 @@ module Gitlab @skip_authorization = skip_authorization @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol + + @logger = logger + @logger.append_message("Running checks for ref: #{@branch_name || @tag_name}") end def exec(skip_commits_check: false) @@ -49,26 +65,32 @@ module Gitlab protected def push_checks - unless can_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + logger.log_timed(LOG_MESSAGES[__method__]) do + unless can_push? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code] + end end end def branch_checks return unless branch_name - if deletion? && branch_name == project.default_branch - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do + if deletion? && branch_name == project.default_branch + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch] + end end protected_branch_checks end def protected_branch_checks - return unless ProtectedBranch.protected?(project, branch_name) + logger.log_timed(LOG_MESSAGES[__method__]) do + return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks - if forced_push? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + if forced_push? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch] + end end if deletion? @@ -79,23 +101,27 @@ module Gitlab end def protected_branch_deletion_checks - unless user_access.can_delete_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] - end + logger.log_timed(LOG_MESSAGES[__method__]) do + unless user_access.can_delete_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch] + end - unless updated_from_web? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + unless updated_from_web? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch] + end end end def protected_branch_push_checks - if matching_merge_request? - unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] - end - else - unless user_access.can_push_to_branch?(branch_name) - raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + logger.log_timed(LOG_MESSAGES[__method__]) do + if matching_merge_request? + unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch] + end + else + unless user_access.can_push_to_branch?(branch_name) + raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message + end end end end @@ -103,21 +129,25 @@ module Gitlab def tag_checks return unless tag_name - if tag_exists? && user_access.cannot_do_action?(:admin_project) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + logger.log_timed(LOG_MESSAGES[__method__]) do + if tag_exists? && user_access.cannot_do_action?(:admin_project) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags] + end end protected_tag_checks end def protected_tag_checks - return unless ProtectedTag.protected?(project, tag_name) + logger.log_timed(LOG_MESSAGES[__method__]) do + return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? - raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update? + raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion? - unless user_access.can_create_tag?(tag_name) - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + unless user_access.can_create_tag?(tag_name) + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag] + end end end @@ -125,14 +155,20 @@ module Gitlab return if deletion? || newrev.nil? return unless should_run_commit_validations? - # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - commits.each do |commit| - commit_check.validate(commit, validations_for_commit(commit)) + logger.log_timed(LOG_MESSAGES[__method__]) do + # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + commits.each do |commit| + logger.check_timeout_reached + + commit_check.validate(commit, validations_for_commit(commit)) + end end end - commit_check.validate_file_paths + logger.log_timed(LOG_MESSAGES[:commits_check_file_paths_validation]) do + commit_check.validate_file_paths + end end # Method overwritten in EE to inject custom validations @@ -194,10 +230,12 @@ module Gitlab end def lfs_objects_exist_check - lfs_check = Checks::LfsIntegrity.new(project, newrev) + logger.log_timed(LOG_MESSAGES[__method__]) do + lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left) - if lfs_check.objects_missing? - raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] + if lfs_check.objects_missing? + raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] + end end end diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index fa3dc1808df..1652d5a30a4 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -3,9 +3,10 @@ module Gitlab module Checks class LfsIntegrity - def initialize(project, newrev) + def initialize(project, newrev, time_left) @project = project @newrev = newrev + @time_left = time_left end # rubocop: disable CodeReuse/ActiveRecord @@ -13,7 +14,7 @@ module Gitlab return false unless @newrev && @project.lfs_enabled? new_lfs_pointers = Gitlab::Git::LfsChanges.new(@project.repository, @newrev) - .new_pointers(object_limit: ::Gitlab::Git::Repository::REV_LIST_COMMIT_LIMIT) + .new_pointers(object_limit: ::Gitlab::Git::Repository::REV_LIST_COMMIT_LIMIT, dynamic_timeout: @time_left) return false unless new_lfs_pointers.present? diff --git a/lib/gitlab/checks/timed_logger.rb b/lib/gitlab/checks/timed_logger.rb new file mode 100644 index 00000000000..f365e0a43f6 --- /dev/null +++ b/lib/gitlab/checks/timed_logger.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class TimedLogger + TimeoutError = Class.new(StandardError) + + attr_reader :start_time, :header, :log, :timeout + + def initialize(start_time: Time.now, log: [], header: "", timeout:) + @start_time = start_time + @timeout = timeout + @header = header + @log = log + end + + # Adds trace of method being tracked with + # the correspondent time it took to run it. + # We make use of the start default argument + # on unit tests related to this method + # + def log_timed(log_message, start = Time.now) + check_timeout_reached + + timed = true + + yield + + append_message(log_message + time_suffix_message(start: start)) + rescue GRPC::DeadlineExceeded, TimeoutError + args = { cancelled: true } + args[:start] = start if timed + + append_message(log_message + time_suffix_message(args)) + + raise TimeoutError + end + + def check_timeout_reached + return unless time_expired? + + raise TimeoutError + end + + def time_left + (start_time + timeout.seconds) - Time.now + end + + def full_message + header + log.join("\n") + end + + # We always want to append in-place on the log + def append_message(message) + log << message + end + + private + + def time_expired? + time_left <= 0 + end + + def time_suffix_message(cancelled: false, start: nil) + return " (#{elapsed_time(start)}ms)" unless cancelled + + if start + " (cancelled after #{elapsed_time(start)}ms)" + else + " (cancelled)" + end + end + + def elapsed_time(start) + to_ms(Time.now - start) + end + + def to_ms(elapsed) + (elapsed.to_f * 1000).round(2) + end + end + end +end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index e780f8c646b..974b5ad6877 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + # ANSI color library # # Implementation per http://en.wikipedia.org/wiki/ANSI_escape_code @@ -265,7 +267,7 @@ module Gitlab def reset_state @offset = 0 @n_open_tags = 0 - @out = '' + @out = +'' reset end diff --git a/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb index ee3647f24fd..25a82086676 100644 --- a/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb +++ b/lib/gitlab/ci/build/artifacts/adapters/gzip_stream.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb index fa6842cf36a..cf37d700991 100644 --- a/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb +++ b/lib/gitlab/ci/build/artifacts/adapters/raw_stream.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 551d4f4473e..08dac756cc1 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'zlib' require 'json' diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 85072a072d6..d0a80518ae8 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/artifacts/path.rb b/lib/gitlab/ci/build/artifacts/path.rb index 9cd9b36c5f8..65cd935afaa 100644 --- a/lib/gitlab/ci/build/artifacts/path.rb +++ b/lib/gitlab/ci/build/artifacts/path.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb index 29a7a27c963..58adf6e506d 100644 --- a/lib/gitlab/ci/build/credentials/base.rb +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb index 2423aa8857d..fa805abb8bb 100644 --- a/lib/gitlab/ci/build/credentials/factory.rb +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb index 55eafcaed10..1c8588d9913 100644 --- a/lib/gitlab/ci/build/credentials/registry.rb +++ b/lib/gitlab/ci/build/credentials/registry.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index c811f88f483..4dd932f61d4 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb index d10cc7802d4..43c46ad74af 100644 --- a/lib/gitlab/ci/build/policy.rb +++ b/lib/gitlab/ci/build/policy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index 782f6c4c0af..4c7dc947cd0 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 4aa5dc89f47..10934536536 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index f09ba42c074..ceb5210cfb5 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb index 9d2a362b7d4..0698136166a 100644 --- a/lib/gitlab/ci/build/policy/variables.rb +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index 0b1ebe4e048..d587c896712 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Build diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 7b7354bce16..a4f01468e8e 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Charts diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index fedaf18ef30..2fb3c4582e7 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci # diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index e80f9d2e452..ef5f25b42c0 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb index 3e87a09704e..3c2e1df9b83 100644 --- a/lib/gitlab/ci/config/entry/attributable.rb +++ b/lib/gitlab/ci/config/entry/attributable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb index f3357f85b99..b9639c83075 100644 --- a/lib/gitlab/ci/config/entry/boolean.rb +++ b/lib/gitlab/ci/config/entry/boolean.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index d7e09acbbf3..0a25057f482 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index 9f66f11be9b..d9658291ebe 100644 --- a/lib/gitlab/ci/config/entry/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index 697f622c45e..4aabf0cfa31 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb index 12a063059cb..690409ccf77 100644 --- a/lib/gitlab/ci/config/entry/coverage.rb +++ b/lib/gitlab/ci/config/entry/coverage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 0c1f9eb7cbf..07e9e1d3f67 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/factory.rb b/lib/gitlab/ci/config/entry/factory.rb index 6be8288748f..85c9c3511a4 100644 --- a/lib/gitlab/ci/config/entry/factory.rb +++ b/lib/gitlab/ci/config/entry/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/global.rb b/lib/gitlab/ci/config/entry/global.rb index 04077fa7a61..eba203d9d06 100644 --- a/lib/gitlab/ci/config/entry/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb index 6fc3aa385bc..dc0ede2a25f 100644 --- a/lib/gitlab/ci/config/entry/hidden.rb +++ b/lib/gitlab/ci/config/entry/hidden.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 2844be80a84..fc453b72fa5 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index f290ff3a565..e4610faa327 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 96b6f2e5d6c..1535b108000 100644 --- a/lib/gitlab/ci/config/entry/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb index f27ad0a7759..963b200c7bb 100644 --- a/lib/gitlab/ci/config/entry/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index a3d4432be82..4043629dea9 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index 26505c91be3..347089722e4 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/paths.rb b/lib/gitlab/ci/config/entry/paths.rb index 68dad161149..9580b5e2e7f 100644 --- a/lib/gitlab/ci/config/entry/paths.rb +++ b/lib/gitlab/ci/config/entry/paths.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index c92562f8c85..0535d7c1a1a 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/script.rb b/lib/gitlab/ci/config/entry/script.rb index 29ecd9995ca..f7d39e5cf55 100644 --- a/lib/gitlab/ci/config/entry/script.rb +++ b/lib/gitlab/ci/config/entry/script.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 3e2ebcff31a..47bf9205147 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb index 0066894e069..bdf7f80f382 100644 --- a/lib/gitlab/ci/config/entry/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/simplifiable.rb b/lib/gitlab/ci/config/entry/simplifiable.rb index 12764629686..9961bbfaa40 100644 --- a/lib/gitlab/ci/config/entry/simplifiable.rb +++ b/lib/gitlab/ci/config/entry/simplifiable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/stage.rb b/lib/gitlab/ci/config/entry/stage.rb index b7afaba1de8..65ab5953131 100644 --- a/lib/gitlab/ci/config/entry/stage.rb +++ b/lib/gitlab/ci/config/entry/stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/stages.rb b/lib/gitlab/ci/config/entry/stages.rb index ec187bd3732..ab184246d29 100644 --- a/lib/gitlab/ci/config/entry/stages.rb +++ b/lib/gitlab/ci/config/entry/stages.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb index 1171ac10f22..77dcfa88170 100644 --- a/lib/gitlab/ci/config/entry/undefined.rb +++ b/lib/gitlab/ci/config/entry/undefined.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/unspecified.rb b/lib/gitlab/ci/config/entry/unspecified.rb index fbb2551e870..bab32489d2f 100644 --- a/lib/gitlab/ci/config/entry/unspecified.rb +++ b/lib/gitlab/ci/config/entry/unspecified.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb index e45787773a8..08a6593c980 100644 --- a/lib/gitlab/ci/config/entry/validatable.rb +++ b/lib/gitlab/ci/config/entry/validatable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 2df23a3edcd..33ffdd3a95d 100644 --- a/lib/gitlab/ci/config/entry/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index f6b4ba7843e..805d26ca8d8 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 8acab605c91..6fd3cec2f5f 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb index 141d2714cb6..b4c491e84a6 100644 --- a/lib/gitlab/ci/config/loader.rb +++ b/lib/gitlab/ci/config/loader.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Config diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 73f36735e35..b1db9084662 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class CronParser diff --git a/lib/gitlab/ci/mask_secret.rb b/lib/gitlab/ci/mask_secret.rb index 0daddaa638c..58d55b1bd6f 100644 --- a/lib/gitlab/ci/mask_secret.rb +++ b/lib/gitlab/ci/mask_secret.rb @@ -1,9 +1,13 @@ +# frozen_string_literal: true + module Gitlab module Ci::MaskSecret class << self def mask!(value, token) return value unless value.present? && token.present? + # We assume 'value' must be mutable, given + # that frozen string is enabled. value.gsub!(token, 'x' * token.length) value end diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb index 3994a50772b..fbdb84c0522 100644 --- a/lib/gitlab/ci/model.rb +++ b/lib/gitlab/ci/model.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Model diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 5d7d9a751d8..ed5a79d9b9b 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -14,10 +14,10 @@ module Gitlab test_case = create_test_case(test_case) test_suite.add_test_case(test_case) end - rescue REXML::ParseException => e - raise JunitParserError, "XML parsing failed: #{e.message}" - rescue => e - raise JunitParserError, "JUnit parsing failed: #{e.message}" + rescue REXML::ParseException + raise JunitParserError, "XML parsing failed" + rescue + raise JunitParserError, "JUnit parsing failed" end private diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb index efed19da21c..bab1c73e2f1 100644 --- a/lib/gitlab/ci/pipeline/chain/base.rb +++ b/lib/gitlab/ci/pipeline/chain/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index b5eb0cfa2f0..b445a872b3d 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index a53c80d34f7..05978804d92 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -1,4 +1,7 @@ -module Gitlab # rubocop:disable Naming/FileName +# rubocop:disable Naming/FileName +# frozen_string_literal: true + +module Gitlab module Ci module Pipeline module Chain diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 02493c7fe02..c882241ef6a 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index bf1380a1da9..6bb3a75291b 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index f34c11ca3c2..633d3cd4f6b 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index e24630656d3..99780409085 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index 32cbb7ca6af..b9707d2f8f5 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 13c6fedd831..ebd7e6e8289 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/validate/config.rb b/lib/gitlab/ci/pipeline/chain/validate/config.rb index a3bd2a5a23a..28c38cc3d18 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/config.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/config.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/chain/validate/repository.rb b/lib/gitlab/ci/pipeline/chain/validate/repository.rb index 9699c24e5b6..d88851d8245 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/repository.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/repository.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb index 30701e1de1b..de24bbf688b 100644 --- a/lib/gitlab/ci/pipeline/duration.rb +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb index f57df7c5637..61d392121d8 100644 --- a/lib/gitlab/ci/pipeline/expression.rb +++ b/lib/gitlab/ci/pipeline/expression.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb index 047ab66e9b3..70c774416f6 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb index 3a2f0c6924e..668e85f5b9e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index 10957598f76..cd17bc4d78b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb index a2778716924..be7258c201a 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb index f640d0b5855..3ebceb92eb7 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 9b239c29ea4..d7e6dacf068 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 346c92dc51e..2db2bf011f1 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb index f2611d65faf..ef9ddb6cae9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 37643c8ef53..85c0899e4f6 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 4cacb1e62c9..f26542361a2 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index 90f94d0b763..ed184309ab4 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index b36f1e0f865..b03611f756e 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/expression/token.rb b/lib/gitlab/ci/pipeline/expression/token.rb index 58211800b88..513d43f6fca 100644 --- a/lib/gitlab/ci/pipeline/expression/token.rb +++ b/lib/gitlab/ci/pipeline/expression/token.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/seed/base.rb b/lib/gitlab/ci/pipeline/seed/base.rb index db9706924bb..1fd3a61017f 100644 --- a/lib/gitlab/ci/pipeline/seed/base.rb +++ b/lib/gitlab/ci/pipeline/seed/base.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 6980b0b7aff..ef738a93bfe 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 2b58d9863a0..4775ff15581 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Pipeline diff --git a/lib/gitlab/ci/reports/test_case.rb b/lib/gitlab/ci/reports/test_case.rb index b4d08ed257f..292e273a03a 100644 --- a/lib/gitlab/ci/reports/test_case.rb +++ b/lib/gitlab/ci/reports/test_case.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_reports.rb index c87bdb4a8a2..7397ff35d46 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_reports.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb index 726c6a11a81..11810bdc0a8 100644 --- a/lib/gitlab/ci/reports/test_reports_comparer.rb +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index b5f15397c0f..b0391160c15 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index 642aa593092..9cb7db5934c 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Reports diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 50b0d044265..4babc23a495 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -11,7 +11,8 @@ module Gitlab runner_system_failure: 'runner system failure', missing_dependency_failure: 'missing dependency failure', runner_unsupported: 'unsupported runner', - stale_schedule: 'stale schedule' + stale_schedule: 'stale schedule', + job_execution_timeout: 'job execution timeout' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml index bf7831b937c..6e138639b71 100644 --- a/lib/gitlab/ci/templates/Android.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -1,51 +1,45 @@ -# Read more about this script on this blog post https://about.gitlab.com/2016/11/30/setting-up-gitlab-ci-for-android-projects/, by Greyson Parrelli +# Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny image: openjdk:8-jdk variables: ANDROID_COMPILE_SDK: "28" - ANDROID_BUILD_TOOLS: "28.0.3" - ANDROID_SDK_TOOLS: "26.1.1" + ANDROID_BUILD_TOOLS: "28.0.2" + ANDROID_SDK_TOOLS: "4333796" before_script: -- apt-get --quiet update --yes -- apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 -- wget --quiet --output-document=android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip -- unzip android-sdk.zip -d android-sdk-linux -- echo y | android-sdk-linux/tools/bin/sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}" > /dev/null -- echo y | android-sdk-linux/tools/bin/sdkmanager platform-tools > /dev/null -- echo y | android-sdk-linux/tools/bin/sdkmanager "build-tools;${ANDROID_BUILD_TOOLS}" > /dev/null -- echo y | android-sdk-linux/tools/bin/sdkmanager "extras;google;google_play_services" > /dev/null -- echo y | android-sdk-linux/tools/bin/sdkmanager "extras;google;m2repository" > /dev/null -- export ANDROID_HOME=$PWD/android-sdk-linux -- export PATH=$PATH:$PWD/android-sdk-linux/platform-tools/ -- yes | android-sdk-linux/tools/bin/sdkmanager --licenses & -- chmod +x ./gradlew + - apt-get --quiet update --yes + - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 + - wget --quiet --output-document=android-sdk.zip https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_TOOLS}.zip + - unzip -d android-sdk-linux android-sdk.zip + - echo y | android-sdk-linux/tools/bin/sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}" >/dev/null + - echo y | android-sdk-linux/tools/bin/sdkmanager "platform-tools" >/dev/null + - echo y | android-sdk-linux/tools/bin/sdkmanager "build-tools;${ANDROID_BUILD_TOOLS}" >/dev/null + - export ANDROID_HOME=$PWD/android-sdk-linux + - export PATH=$PATH:$PWD/android-sdk-linux/platform-tools/ + - chmod +x ./gradlew + # temporarily disable checking for EPIPE error and use yes to accept all licenses + - set +o pipefail + - yes | android-sdk-linux/tools/bin/sdkmanager --licenses + - set -o pipefail stages: -- build -- test + - build + - test -build: +lintDebug: stage: build script: - - ./gradlew assembleDebug + - ./gradlew -Pci --console=plain :app:lintDebug -PbuildDir=lint + +assembleDebug: + stage: build + script: + - ./gradlew assembleDebug artifacts: paths: - app/build/outputs/ -unitTests: - stage: test - script: - - ./gradlew test - -functionalTests: +debugTests: stage: test script: - - wget --quiet --output-document=android-wait-for-emulator https://raw.githubusercontent.com/travis-ci/travis-cookbooks/0f497eb71291b52a703143c5cd63a217c8766dc9/community-cookbooks/android-sdk/files/default/android-wait-for-emulator - - chmod +x android-wait-for-emulator - - echo y | android-sdk-linux/tools/android --silent update sdk --no-ui --all --filter sys-img-x86-google_apis-${ANDROID_COMPILE_SDK} - - echo no | android-sdk-linux/tools/android create avd -n test -t android-${ANDROID_COMPILE_SDK} --abi google_apis/x86 - - android-sdk-linux/tools/emulator64-x86 -avd test -no-window -no-audio & - - ./android-wait-for-emulator - - adb shell input keyevent 82 - - ./gradlew cAT + - ./gradlew -Pci --console=plain :app:testDebug diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 6fa59e41d20..db48b187e5e 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -210,7 +210,7 @@ container_scanning: refs: - branches variables: - - $GITLAB_FEATURES =~ /\bsast_container\b/ + - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ except: variables: - $CONTAINER_SCANNING_DISABLED diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index d61ff239e13..492b3d03db2 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -15,7 +15,7 @@ # * Publishes the documentation for `master` branch. variables: - # This will supress any download for dependencies and plugins or upload messages which would clutter the console log. + # This will suppress any download for dependencies and plugins or upload messages which would clutter the console log. # `showDateTime` will show the passed time in milliseconds. You need to specify `--batch-mode` to make this work. MAVEN_OPTS: "-Dhttps.protocols=TLSv1.2 -Dmaven.repo.local=$CI_PROJECT_DIR/.m2/repository -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Dorg.slf4j.simpleLogger.showDateTime=true -Djava.awt.headless=true" # As of Maven 3.3.0 instead of this you may define these options in `.mvn/maven.config` so the same config is used diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 93e219a21f9..8eccd262db9 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Trace diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index a427aa30683..39a1b52e531 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class YamlProcessor diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb new file mode 100644 index 00000000000..b05dca409d1 --- /dev/null +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + # + # LifecycleEvents lets Rails initializers register application startup hooks + # that are sensitive to forking. For example, to defer the creation of + # watchdog threads. This lets us abstract away the Unix process + # lifecycles of Unicorn, Sidekiq, Puma, Puma Cluster, etc. + # + # We have three lifecycle events. + # + # - before_fork (only in forking processes) + # - worker_start + # - before_master_restart (only in forking processes) + # + # Blocks will be executed in the order in which they are registered. + # + class LifecycleEvents + class << self + # + # Hook registration methods (called from initializers) + # + def on_worker_start(&block) + if in_clustered_environment? + # Defer block execution + (@worker_start_hooks ||= []) << block + else + yield + end + end + + def on_before_fork(&block) + return unless in_clustered_environment? + + # Defer block execution + (@before_fork_hooks ||= []) << block + end + + def on_master_restart(&block) + return unless in_clustered_environment? + + # Defer block execution + (@master_restart_hooks ||= []) << block + end + + # + # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) + # + def do_worker_start + @worker_start_hooks&.each do |block| + block.call + end + end + + def do_before_fork + @before_fork_hooks&.each do |block| + block.call + end + end + + def do_master_restart + @master_restart_hooks && @master_restart_hooks.each do |block| + block.call + end + end + + # Puma doesn't use singletons (which is good) but + # this means we need to pass through whether the + # puma server is running in single mode or cluster mode + def set_puma_options(options) + @puma_options = options + end + + private + + def in_clustered_environment? + # Sidekiq doesn't fork + return false if Sidekiq.server? + + # Unicorn always forks + return true if defined?(::Unicorn) + + # Puma sometimes forks + return true if in_clustered_puma? + + # Default assumption is that we don't fork + false + end + + def in_clustered_puma? + return false unless defined?(::Puma) + + @puma_options && @puma_options[:workers] && @puma_options[:workers] > 0 + end + end + end + end +end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb new file mode 100644 index 00000000000..331c39f7d6b --- /dev/null +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Cluster + class PumaWorkerKillerInitializer + def self.start(puma_options, puma_per_worker_max_memory_mb: 650) + require 'puma_worker_killer' + + PumaWorkerKiller.config do |config| + # Note! ram is expressed in megabytes (whereas GITLAB_UNICORN_MEMORY_MAX is in bytes) + # Importantly RAM is for _all_workers (ie, the cluster), + # not each worker as is the case with GITLAB_UNICORN_MEMORY_MAX + worker_count = puma_options[:workers] || 1 + config.ram = worker_count * puma_per_worker_max_memory_mb + + config.frequency = 20 # seconds + + # We just want to limit to a fixed maximum, unrelated to the total amount + # of available RAM. + config.percent_usage = 0.98 + + # Ideally we'll never hit the maximum amount of memory. If so the worker + # is restarted already, thus periodically restarting workers shouldn't be + # needed. + config.rolling_restart_frequency = false + end + + PumaWorkerKiller.start + end + end + end +end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 30911b49b18..501c2111530 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -3,6 +3,7 @@ module Gitlab class File include Gitlab::Routing include IconsHelper + include Gitlab::Utils::StrongMemoize CONTEXT_LINES = 3 @@ -30,11 +31,8 @@ module Gitlab end def highlight_lines! - their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") - our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") - - their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines - our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines + their_highlight = Gitlab::Highlight.highlight(their_path, their_lines, language: their_language).lines + our_highlight = Gitlab::Highlight.highlight(our_path, our_lines, language: our_language).lines lines.each do |line| line.rich_text = @@ -182,6 +180,34 @@ module Gitlab raw_line[:line_new], parent_file: self) end end + + def their_language + strong_memoize(:their_language) do + repository.gitattribute(their_path, 'gitlab-language') + end + end + + def our_language + strong_memoize(:our_language) do + if our_path == their_path + their_language + else + repository.gitattribute(our_path, 'gitlab-language') + end + end + end + + def their_lines + strong_memoize(:their_lines) do + lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") + end + end + + def our_lines + strong_memoize(:our_lines) do + lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") + end + end end end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index c819bffdfd6..5ed6427072a 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -73,7 +73,7 @@ module Gitlab # re-running the contributed projects query in each union is expensive, so # use IN(project_ids...) instead. It's the intersection of two users so # the list will be (relatively) short - @contributed_project_ids ||= projects.uniq.pluck(:id) + @contributed_project_ids ||= projects.distinct.pluck(:id) authed_projects = Project.where(id: @contributed_project_ids) .with_feature_available_for_user(feature, current_user) .reorder(nil) diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb new file mode 100644 index 00000000000..68d0b5d8f8a --- /dev/null +++ b/lib/gitlab/crypto_helper.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module CryptoHelper + extend self + + AES256_GCM_OPTIONS = { + algorithm: 'aes-256-gcm', + key: Settings.attr_encrypted_db_key_base_truncated, + iv: Settings.attr_encrypted_db_key_base_truncated[0..11] + }.freeze + + def sha256(value) + salt = Settings.attr_encrypted_db_key_base_truncated + ::Digest::SHA256.base64digest("#{value}#{salt}") + end + + def aes256_gcm_encrypt(value) + encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value)) + Base64.encode64(encrypted_token) + end + + def aes256_gcm_decrypt(value) + return unless value + + encrypted_token = Base64.decode64(value) + Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token)) + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index a17f27a3147..f98d6dbd46f 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -879,7 +879,7 @@ module Gitlab columns(table).find { |column| column.name == name } end - # This will replace the first occurance of a string in a column with + # This will replace the first occurrence of a string in a column with # the replacement # On postgresql we can use `regexp_replace` for that. # On mysql we find the location of the pattern, and overwrite it diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index a605ddb5c33..1d833183ec3 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -79,7 +79,7 @@ module Gitlab return [] unless blob blob.load_all_data! - Gitlab::Highlight.highlight(blob.path, blob.data, repository: repository).lines + blob.present.highlight.lines end end end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index b68a1636814..8457e0c4cb6 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -24,7 +24,7 @@ module Gitlab # head of `feature` was commit B, resulting in the original diff A->B. # Since creation, `master` was updated to C. # Now `feature` is being updated to D, and the newly generated MR diff is C->D. - # It is possible that C and D are direct decendants of A and B respectively, + # It is possible that C and D are direct descendants of A and B respectively, # but this isn't necessarily the case as rebases and merges come into play. # # Suppose we have a diff note on the original diff A->B. Now that the MR diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 13b0bb930f4..0bd1d3420a2 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -5,6 +5,7 @@ module Gitlab class Blob include Gitlab::BlobHelper include Gitlab::EncodingHelper + extend Gitlab::Git::WrapsGitalyErrors # This number is the maximum amount of data that we want to display to # the user. We load as much as we can for encoding detection and LFS @@ -75,7 +76,7 @@ module Gitlab # Returns array of Gitlab::Git::Blob # Does not guarantee blob data will be set def batch_lfs_pointers(repository, blob_ids) - repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do repository.gitaly_blob_client.batch_lfs_pointers(blob_ids.to_a) end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 74cdabfed9d..2820491b65d 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -3,6 +3,7 @@ module Gitlab module Git class Commit include Gitlab::EncodingHelper + extend Gitlab::Git::WrapsGitalyErrors attr_accessor :raw_commit, :head @@ -59,7 +60,7 @@ module Gitlab # This saves us an RPC round trip. return nil if commit_id.include?(':') - commit = repo.wrapped_gitaly_errors do + commit = wrapped_gitaly_errors do repo.gitaly_commit_client.find_commit(commit_id) end @@ -100,7 +101,7 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - repo.wrapped_gitaly_errors do + wrapped_gitaly_errors do repo.gitaly_commit_client.between(base, head) end end @@ -125,7 +126,7 @@ module Gitlab # are documented here: # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) def find_all(repo, options = {}) - repo.wrapped_gitaly_errors do + wrapped_gitaly_errors do Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end end @@ -142,7 +143,7 @@ module Gitlab # relation to each other. The last 10 commits for a branch for example, # should go through .where def batch_by_oid(repo, oids) - repo.wrapped_gitaly_errors do + wrapped_gitaly_errors do repo.gitaly_commit_client.list_commits_by_oid(oids) end end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index ae6f554bc06..83a9fd5f81a 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -3,6 +3,8 @@ module Gitlab module Git class CommitStats + include Gitlab::Git::WrapsGitalyErrors + attr_reader :id, :additions, :deletions, :total # Instantiate a CommitStats object @@ -14,7 +16,7 @@ module Gitlab @deletions = 0 @total = 0 - repo.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_stats(repo, commit) end end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 6dc792c16b8..307f1b8cb66 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -2,6 +2,8 @@ module Gitlab module Git module Conflict class Resolver + include Gitlab::Git::WrapsGitalyErrors + ConflictSideMissing = Class.new(StandardError) ResolutionError = Class.new(StandardError) @@ -12,7 +14,7 @@ module Gitlab end def conflicts - @conflicts ||= @target_repository.wrapped_gitaly_errors do + @conflicts ||= wrapped_gitaly_errors do gitaly_conflicts_client(@target_repository).list_conflict_files.to_a end rescue GRPC::FailedPrecondition => e @@ -22,7 +24,7 @@ module Gitlab end def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) - source_repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) end end diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index f0fab1e76a3..d7148165408 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -6,8 +6,8 @@ module Gitlab @newrev = newrev end - def new_pointers(object_limit: nil, not_in: nil) - @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) + def new_pointers(object_limit: nil, not_in: nil, dynamic_timeout: nil) + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in, dynamic_timeout) end def all_pointers diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index e4743b4db0a..7f9520de5ce 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -1,13 +1,15 @@ module Gitlab module Git class RemoteMirror + include Gitlab::Git::WrapsGitalyErrors + def initialize(repository, ref_name) @repository = repository @ref_name = ref_name end def update(only_branches_matching: []) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 9df04372cc2..fcc92341c40 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -6,6 +6,7 @@ module Gitlab module Git class Repository include Gitlab::Git::RepositoryMirroring + include Gitlab::Git::WrapsGitalyErrors include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize @@ -845,23 +846,9 @@ module Gitlab end def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) - Gitlab::GitalyClient.migrate(method, status: status, &block) - rescue GRPC::NotFound => e - raise NoRepository.new(e) - rescue GRPC::InvalidArgument => e - raise ArgumentError.new(e) - rescue GRPC::BadStatus => e - raise CommandError.new(e) - end - - def wrapped_gitaly_errors(&block) - yield block - rescue GRPC::NotFound => e - raise NoRepository.new(e) - rescue GRPC::InvalidArgument => e - raise ArgumentError.new(e) - rescue GRPC::BadStatus => e - raise CommandError.new(e) + wrapped_gitaly_errors do + Gitlab::GitalyClient.migrate(method, status: status, &block) + end end def clean_stale_repository_files diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index e0867aeb5a7..b5b701699f0 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -2,6 +2,7 @@ module Gitlab module Git class Tree include Gitlab::EncodingHelper + extend Gitlab::Git::WrapsGitalyErrors attr_accessor :id, :root_id, :name, :path, :flat_path, :type, :mode, :commit_id, :submodule_url @@ -15,7 +16,7 @@ module Gitlab def where(repository, sha, path = nil, recursive = false) path = nil if path == '' || path == '/' - repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 7fe56979d5c..02c643d0da0 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -1,6 +1,8 @@ module Gitlab module Git class Wiki + include Gitlab::Git::WrapsGitalyErrors + DuplicatePageError = Class.new(StandardError) OperationError = Class.new(StandardError) @@ -65,37 +67,37 @@ module Gitlab end def write_page(name, format, content, commit_details) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_write_page(name, format, content, commit_details) end end def delete_page(page_path, commit_details) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_delete_page(page_path, commit_details) end end def update_page(page_path, title, format, content, commit_details) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_update_page(page_path, title, format, content, commit_details) end end def pages(limit: 0) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_get_all_pages(limit: limit) end end def page(title:, version: nil, dir: nil) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_find_page(title: title, version: version, dir: dir) end end def file(name, version) - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_find_file(name, version) end end @@ -105,7 +107,7 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - versions = @repository.wrapped_gitaly_errors do + versions = wrapped_gitaly_errors do gitaly_wiki_client.page_versions(page_path, options) end @@ -127,7 +129,7 @@ module Gitlab def page_formatted_data(title:, dir: nil, version: nil) version = version&.id - @repository.wrapped_gitaly_errors do + wrapped_gitaly_errors do gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) end end diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb new file mode 100644 index 00000000000..4b161f7e6ce --- /dev/null +++ b/lib/gitlab/git/wraps_gitaly_errors.rb @@ -0,0 +1,15 @@ +module Gitlab + module Git + module WrapsGitalyErrors + def wrapped_gitaly_errors(&block) + yield block + rescue GRPC::NotFound => e + raise Gitlab::Git::Repository::NoRepository.new(e) + rescue GRPC::InvalidArgument => e + raise ArgumentError.new(e) + rescue GRPC::BadStatus => e + raise Gitlab::Git::CommandError.new(e) + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 827c04ae035..802fa65dd63 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -9,6 +9,7 @@ module Gitlab UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) ProjectCreationError = Class.new(StandardError) + TimeoutError = Class.new(StandardError) ProjectMovedError = Class.new(NotFoundError) ERROR_MESSAGES = { @@ -26,11 +27,18 @@ module Gitlab cannot_push_to_read_only: "You can't push code to a read-only GitLab instance." }.freeze + INTERNAL_TIMEOUT = 50.seconds.freeze + LOG_HEADER = <<~MESSAGE + Push operation timed out + + Timing information for debugging purposes: + MESSAGE + DOWNLOAD_COMMANDS = %w{git-upload-pack git-upload-archive}.freeze PUSH_COMMANDS = %w{git-receive-pack}.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type, :changes, :logger def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @@ -44,6 +52,7 @@ module Gitlab end def check(cmd, changes) + @logger = Checks::TimedLogger.new(timeout: INTERNAL_TIMEOUT, header: LOG_HEADER) @changes = changes check_protocol! @@ -269,14 +278,19 @@ module Gitlab end def check_single_change_access(change, skip_lfs_integrity_check: false) - Checks::ChangeAccess.new( + change_access = Checks::ChangeAccess.new( change, user_access: user_access, project: project, skip_authorization: deploy_key?, skip_lfs_integrity_check: skip_lfs_integrity_check, - protocol: protocol - ).exec + protocol: protocol, + logger: logger + ) + + change_access.exec + rescue Checks::TimedLogger::TimeoutError + raise TimeoutError, logger.full_message end def deploy_key diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index e731e654f3c..cf2329e489d 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -11,8 +11,8 @@ module Gitlab @changes = deserialize_changes(changes) end - def identify(revision) - super(identifier, project, revision) + def identify + super(identifier) end def changes_refs diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 1840bf45154..086ce31e678 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -72,7 +72,7 @@ module Gitlab GitalyClient::BlobsStitcher.new(response) end - def get_new_lfs_pointers(revision, limit, not_in) + def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil) request = Gitaly::GetNewLFSPointersRequest.new( repository: @gitaly_repo, revision: encode_binary(revision), @@ -85,7 +85,20 @@ module Gitlab request.not_in_refs += not_in end - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request, timeout: GitalyClient.medium_timeout) + timeout = + if dynamic_timeout + [dynamic_timeout, GitalyClient.medium_timeout].min + else + GitalyClient.medium_timeout + end + + response = GitalyClient.call( + @gitaly_repo.storage_name, + :blob_service, + :get_new_lfs_pointers, + request, + timeout: timeout + ) map_lfs_pointers(response) end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 2956ed4b911..d7b36946b65 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -349,7 +349,7 @@ module Gitlab f.write(message.data) end end - # If the file is empty means that we recieved an empty stream, we delete the file + # If the file is empty means that we received an empty stream, we delete the file FileUtils.rm(save_path) if File.zero?(save_path) end diff --git a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb index 0014ce2689b..41004408dec 100644 --- a/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb +++ b/lib/gitlab/grape_logging/formatters/lograge_with_timestamp.rb @@ -6,7 +6,7 @@ module Gitlab def call(severity, datetime, _, data) time = data.delete :time - data[:params] = utf8_encode_values(data[:params]) if data.has_key?(:params) + data[:params] = process_params(data) attributes = { time: datetime.utc.iso8601(3), @@ -20,6 +20,14 @@ module Gitlab private + def process_params(data) + return [] unless data.has_key?(:params) + + data[:params] + .each_pair + .map { |k, v| { key: k, value: utf8_encode_values(v) } } + end + def utf8_encode_values(data) case data when Hash diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 83095acc528..a4e60bbd828 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -4,22 +4,25 @@ module Gitlab class Highlight TIMEOUT_BACKGROUND = 30.seconds TIMEOUT_FOREGROUND = 3.seconds + MAXIMUM_TEXT_HIGHLIGHT_SIZE = 1.megabyte - def self.highlight(blob_name, blob_content, repository: nil, plain: false) - new(blob_name, blob_content, repository: repository) + def self.highlight(blob_name, blob_content, language: nil, plain: false) + new(blob_name, blob_content, language: language) .highlight(blob_content, continue: false, plain: plain) end attr_reader :blob_name - def initialize(blob_name, blob_content, repository: nil) + def initialize(blob_name, blob_content, language: nil) @formatter = Rouge::Formatters::HTMLGitlab - @repository = repository + @language = language @blob_name = blob_name @blob_content = blob_content end def highlight(text, continue: true, plain: false) + plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE + highlighted_text = highlight_text(text, continue: continue, plain: plain) highlighted_text = link_dependencies(text, highlighted_text) if blob_name highlighted_text @@ -36,11 +39,9 @@ module Gitlab private def custom_language - language_name = @repository && @repository.gitattribute(@blob_name, 'gitlab-language') - - return nil unless language_name + return nil unless @language - Rouge::Lexer.find_fancy(language_name) + Rouge::Lexer.find_fancy(@language) end def highlight_text(text, continue: true, plain: false) diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 28a9fe29465..d5f94ad04f1 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -1,13 +1,11 @@ # frozen_string_literal: true # Detect user based on identifier like -# key-13 or user-36 or last commit +# key-13 or user-36 module Gitlab module Identifier - def identify(identifier, project = nil, newrev = nil) - if identifier.blank? - identify_using_commit(project, newrev) - elsif identifier =~ /\Auser-\d+\Z/ + def identify(identifier) + if identifier =~ /\Auser-\d+\Z/ # git push over http identify_using_user(identifier) elsif identifier =~ /\Akey-\d+\Z/ @@ -16,19 +14,6 @@ module Gitlab end end - # Tries to identify a user based on a commit SHA. - def identify_using_commit(project, ref) - return if project.nil? && ref.nil? - - commit = project.commit(ref) - - return if !commit || !commit.author_email - - identify_with_cache(:email, commit.author_email) do - commit.author - end - end - # Tries to identify a user based on a user identifier (e.g. "user-123"). # rubocop: disable CodeReuse/ActiveRecord def identify_using_user(identifier) diff --git a/lib/gitlab/import/merge_request_helpers.rb b/lib/gitlab/import/merge_request_helpers.rb index 97dc1a987c4..9215067d973 100644 --- a/lib/gitlab/import/merge_request_helpers.rb +++ b/lib/gitlab/import/merge_request_helpers.rb @@ -22,7 +22,7 @@ module Gitlab # additional work that is strictly necessary. merge_request_id = insert_and_return_id(attributes, project.merge_requests) - merge_request = project.merge_requests.find(merge_request_id) + merge_request = project.merge_requests.reload.find(merge_request_id) # We use .insert_and_return_id which effectively disables all callbacks. # Trigger iid logic here to make sure we track internal id values consistently. diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 3d693d23c99..99581eb0416 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -154,7 +154,7 @@ module Gitlab Project.transaction do process_sub_relation(relation, relation_item) - # For every subrelation that hangs from Project, save the associated records alltogether + # For every subrelation that hangs from Project, save the associated records altogether # This effectively batches all records per subrelation item, only keeping those in memory # We have to keep in mind that more batch granularity << Memory, but >> Slowness if save diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index e88a15b8acd..f266177bec1 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -13,11 +13,21 @@ module Gitlab class KubeClient include Gitlab::Utils::StrongMemoize - SUPPORTED_API_GROUPS = [ - 'api', - 'apis/rbac.authorization.k8s.io', - 'apis/extensions' - ].freeze + SUPPORTED_API_GROUPS = { + core: { group: 'api', version: 'v1' }, + rbac: { group: 'apis/rbac.authorization.k8s.io', version: 'v1' }, + extensions: { group: 'apis/extensions', version: 'v1beta1' } + }.freeze + + SUPPORTED_API_GROUPS.each do |name, params| + client_method_name = "#{name}_client".to_sym + + define_method(client_method_name) do + strong_memoize(client_method_name) do + build_kubeclient(params[:group], params[:version]) + end + end + end # Core API methods delegates to the core api group client delegate :get_pods, @@ -62,48 +72,21 @@ module Gitlab :watch_pod_log, to: :core_client - def initialize(api_prefix, api_groups = ['api'], api_version = 'v1', **kubeclient_options) - raise ArgumentError unless check_api_groups_supported?(api_groups) + attr_reader :api_prefix, :kubeclient_options + def initialize(api_prefix, **kubeclient_options) @api_prefix = api_prefix - @api_groups = api_groups - @api_version = api_version @kubeclient_options = kubeclient_options end - def discover! - clients.each(&:discover) - end - - def clients - hashed_clients.values - end - - def core_client - hashed_clients['api'] - end - - def rbac_client - hashed_clients['apis/rbac.authorization.k8s.io'] - end - - def extensions_client - hashed_clients['apis/extensions'] - end - - def hashed_clients - strong_memoize(:hashed_clients) do - @api_groups.map do |api_group| - api_url = join_api_url(@api_prefix, api_group) - [api_group, ::Kubeclient::Client.new(api_url, @api_version, **@kubeclient_options)] - end.to_h - end - end - private - def check_api_groups_supported?(api_groups) - api_groups.all? {|api_group| SUPPORTED_API_GROUPS.include?(api_group) } + def build_kubeclient(api_group, api_version) + ::Kubeclient::Client.new( + join_api_url(api_prefix, api_group), + api_version, + **kubeclient_options + ) end def join_api_url(api_prefix, api_path) diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb new file mode 100644 index 00000000000..b00244a6e04 --- /dev/null +++ b/lib/gitlab/patch/draw_route.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +# We're patching `ActionDispatch::Routing::Mapper` in +# config/initializers/routing_draw.rb +module Gitlab + module Patch + module DrawRoute + RoutesNotFound = Class.new(StandardError) + + def draw(routes_name) + drawn_any = draw_ce(routes_name) | draw_ee(routes_name) + + drawn_any || raise(RoutesNotFound.new("Cannot find #{routes_name}")) + end + + def draw_ce(routes_name) + draw_route(route_path("config/routes/#{routes_name}.rb")) + end + + def draw_ee(_) + true + end + + def route_path(routes_name) + Rails.root.join(routes_name) + end + + def draw_route(path) + if File.exist?(path) + instance_eval(File.read(path)) + true + else + false + end + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 44025650de0..fa68dead80b 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -236,7 +236,7 @@ module Gitlab def single_line_regexp(regex) # Turns a multiline extended regexp into a single line one, - # beacuse `rake routes` breaks on multiline regexes. + # because `rake routes` breaks on multiline regexes. Regexp.new(regex.source.gsub(/\(\?#.+?\)/, '').gsub(/\s*/, ''), regex.options ^ Regexp::EXTENDED).freeze end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 3a202d915e3..04df881bf03 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -82,7 +82,7 @@ module Gitlab ref: ref, startline: startline, data: data.join, - project_id: project ? project.id : nil + project: project ) end diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/proxy_http_connection_adapter.rb index 82213098672..a64cb47e77e 100644 --- a/lib/gitlab/proxy_http_connection_adapter.rb +++ b/lib/gitlab/proxy_http_connection_adapter.rb @@ -4,7 +4,7 @@ # of the global setting allow_local_requests_from_hooks_and_services this adapter # will allow/block connection to internal IPs and/or urls. # -# This functionality can be overriden by providing the setting the option +# This functionality can be overridden by providing the setting the option # allow_local_requests = true in the request. For example: # Gitlab::HTTP.get('http://www.gitlab.com', allow_local_requests: true) # diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 5ce3eda2ccb..458737f31eb 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -4,8 +4,10 @@ module Gitlab class SearchResults class FoundBlob include EncodingHelper + include Presentable + include BlobLanguageFromGitAttributes - attr_reader :id, :filename, :basename, :ref, :startline, :data, :project_id + attr_reader :id, :filename, :basename, :ref, :startline, :data, :project def initialize(opts = {}) @id = opts.fetch(:id, nil) @@ -15,6 +17,11 @@ module Gitlab @startline = opts.fetch(:startline, nil) @data = encode_utf8(opts.fetch(:data, nil)) @per_page = opts.fetch(:per_page, 20) + @project = opts.fetch(:project, nil) + # Some caller does not have project object (e.g. elastic search), + # yet they can trigger many calls in one go, + # causing duplicated queries. + # Allow those to just pass project_id instead. @project_id = opts.fetch(:project_id, nil) end @@ -22,8 +29,12 @@ module Gitlab filename end - def no_highlighting? - false + def project_id + @project_id || @project&.id + end + + def present + super(presenter_class: BlobPresenter) end end diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 25f965e843d..6396b828dc7 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -3,7 +3,7 @@ module Gitlab class IssueNew < IssueCommand def self.match(text) # we can not match \n with the dot by passing the m modifier as than - # the title and description are not seperated + # the title and description are not separated /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text) end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 7735b736689..86efe8ad114 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -32,6 +32,7 @@ module Gitlab end validate_localhost!(addrs_info) unless allow_localhost + validate_loopback!(addrs_info) unless allow_localhost validate_local_network!(addrs_info) unless allow_local_network validate_link_local!(addrs_info) unless allow_local_network @@ -86,6 +87,12 @@ module Gitlab raise BlockedUrlError, "Requests to localhost are not allowed" end + def validate_loopback!(addrs_info) + return unless addrs_info.any? { |addr| addr.ipv4_loopback? || addr.ipv6_loopback? } + + raise BlockedUrlError, "Requests to loopback addresses are not allowed" + end + def validate_local_network!(addrs_info) return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } diff --git a/lib/gitlab/user_extractor.rb b/lib/gitlab/user_extractor.rb index bd0d24e4369..874599688bb 100644 --- a/lib/gitlab/user_extractor.rb +++ b/lib/gitlab/user_extractor.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # This class extracts all users found in a piece of text by the username or the -# email adress +# email address module Gitlab class UserExtractor @@ -14,13 +14,11 @@ module Gitlab @text = text end - # rubocop: disable CodeReuse/ActiveRecord def users return User.none unless @text.present? @users ||= User.from_union(union_relations) end - # rubocop: enable CodeReuse/ActiveRecord def usernames matches[:usernames] diff --git a/lib/google_api/auth.rb b/lib/google_api/auth.rb index e724e58e9ca..56f056fd869 100644 --- a/lib/google_api/auth.rb +++ b/lib/google_api/auth.rb @@ -16,7 +16,7 @@ module GoogleApi client.auth_code.authorize_url( redirect_uri: redirect_uri, scope: scope, - state: state # This is used for arbitary redirection + state: state # This is used for arbitrary redirection ) end diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake index c66a2a263dc..0459de27c96 100644 --- a/lib/tasks/gitlab/ldap.rake +++ b/lib/tasks/gitlab/ldap.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :ldap do desc 'GitLab | LDAP | Rename provider' - task :rename_provider, [:old_provider, :new_provider] => :environment do |_, args| + task :rename_provider, [:old_provider, :new_provider] => :gitlab_environment do |_, args| old_provider = args[:old_provider] || prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue)) new_provider = args[:new_provider] || diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake index ad2d034b0b4..786efd14b1a 100644 --- a/lib/tasks/haml-lint.rake +++ b/lib/tasks/haml-lint.rake @@ -2,5 +2,16 @@ unless Rails.env.production? require 'haml_lint/rake_task' require 'haml_lint/inline_javascript' + # Workaround for warnings from parser/current + # Keep it even if it no longer emits any warnings, + # because we'll still see warnings in console/server anyway, + # and we don't need to break static-analysis for this. + task :haml_lint do + require 'parser' + def Parser.warn(*args) + puts(*args) # static-analysis ignores stdout if status is 0 + end + end + HamlLint::RakeTask.new end diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake index 81829668de8..eec024f9bbb 100644 --- a/lib/tasks/tokens.rake +++ b/lib/tasks/tokens.rake @@ -1,4 +1,7 @@ require_relative '../../app/models/concerns/token_authenticatable.rb' +require_relative '../../app/models/concerns/token_authenticatable_strategies/base.rb' +require_relative '../../app/models/concerns/token_authenticatable_strategies/insecure.rb' +require_relative '../../app/models/concerns/token_authenticatable_strategies/digest.rb' namespace :tokens do desc "Reset all GitLab incoming email tokens" @@ -26,13 +29,6 @@ class TmpUser < ActiveRecord::Base self.table_name = 'users' - def reset_incoming_email_token! - write_new_token(:incoming_email_token) - save!(validate: false) - end - - def reset_feed_token! - write_new_token(:feed_token) - save!(validate: false) - end + add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) } + add_authentication_token_field :feed_token end |