diff options
Diffstat (limited to 'lib/gitlab')
80 files changed, 1285 insertions, 209 deletions
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 6af763faf10..2a44e11efb6 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -5,7 +5,7 @@ module Gitlab REGISTRY_SCOPES = [:read_registry].freeze # Scopes used for GitLab API access - API_SCOPES = [:api, :read_user, :sudo].freeze + API_SCOPES = [:api, :read_user, :sudo, :read_repository].freeze # Scopes used for OpenID Connect OPENID_SCOPES = [:openid].freeze @@ -26,6 +26,7 @@ module Gitlab lfs_token_check(login, password, project) || oauth_access_token_check(login, password) || personal_access_token_check(password) || + deploy_token_check(login, password) || user_with_password_for_git(login, password) || Gitlab::Auth::Result.new @@ -163,7 +164,8 @@ module Gitlab def abilities_for_scopes(scopes) abilities_by_scope = { api: full_authentication_abilities, - read_registry: [:read_container_image] + read_registry: [:read_container_image], + read_repository: [:download_code] } scopes.flat_map do |scope| @@ -171,6 +173,22 @@ module Gitlab end.uniq end + def deploy_token_check(login, password) + return unless password.present? + + token = + DeployToken.active.find_by(token: password) + + return unless token && login + return if login != token.username + + scopes = abilities_for_scopes(token.scopes) + + if valid_scoped_token?(token, available_scopes) + Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) + end + end + def lfs_token_check(login, password, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/ldap/access.rb b/lib/gitlab/auth/ldap/access.rb index 77c0ddc2d48..34286900e72 100644 --- a/lib/gitlab/auth/ldap/access.rb +++ b/lib/gitlab/auth/ldap/access.rb @@ -52,6 +52,8 @@ module Gitlab block_user(user, 'does not exist anymore') false end + rescue LDAPConnectionError + false end def adapter diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index caf2d18c668..82ff1e77e5c 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -2,6 +2,9 @@ module Gitlab module Auth module LDAP class Adapter + SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze + MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze + attr_reader :provider, :ldap def self.open(provider, &block) @@ -16,7 +19,7 @@ module Gitlab def initialize(provider, ldap = nil) @provider = provider - @ldap = ldap || Net::LDAP.new(config.adapter_options) + @ldap = ldap || renew_connection_adapter end def config @@ -47,8 +50,10 @@ module Gitlab end def ldap_search(*args) + retries ||= 0 + # Net::LDAP's `time` argument doesn't work. Use Ruby `Timeout` instead. - Timeout.timeout(config.timeout) do + Timeout.timeout(timeout_time(retries)) do results = ldap.search(*args) if results.nil? @@ -63,16 +68,26 @@ module Gitlab results end end - rescue Net::LDAP::Error => error - Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") - [] - rescue Timeout::Error - Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") - [] + rescue Net::LDAP::Error, Timeout::Error => error + retries += 1 + error_message = connection_error_message(error) + + Rails.logger.warn(error_message) + + if retries < MAX_SEARCH_RETRIES + renew_connection_adapter + retry + else + raise LDAPConnectionError, error_message + end end private + def timeout_time(retry_number) + SEARCH_RETRY_FACTOR[retry_number] * config.timeout + end + def user_options(fields, value, limit) options = { attributes: Gitlab::Auth::LDAP::Person.ldap_attributes(config), @@ -104,6 +119,18 @@ module Gitlab filter end end + + def connection_error_message(exception) + if exception.is_a?(Timeout::Error) + "LDAP search timed out after #{config.timeout} seconds" + else + "LDAP search raised exception #{exception.class}: #{exception.message}" + end + end + + def renew_connection_adapter + @ldap = Net::LDAP.new(config.adapter_options) + end end end end diff --git a/lib/gitlab/auth/ldap/ldap_connection_error.rb b/lib/gitlab/auth/ldap/ldap_connection_error.rb new file mode 100644 index 00000000000..ef0a695742b --- /dev/null +++ b/lib/gitlab/auth/ldap/ldap_connection_error.rb @@ -0,0 +1,7 @@ +module Gitlab + module Auth + module LDAP + LDAPConnectionError = Class.new(StandardError) + end + end +end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index b6a96081278..d0c6b0386ba 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -124,6 +124,9 @@ module Gitlab Gitlab::Auth::LDAP::Person.find_by_uid(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_email(auth_hash.uid, adapter) || Gitlab::Auth::LDAP::Person.find_by_dn(auth_hash.uid, adapter) + + rescue Gitlab::Auth::LDAP::LDAPConnectionError + nil end def ldap_config diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index fd5cbf76e47..a357538a885 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -96,7 +96,7 @@ module Gitlab commit_hash.merge( merge_request_diff_id: merge_request_diff.id, relative_order: index, - sha: sha_attribute.type_cast_for_database(sha) + sha: sha_attribute.serialize(sha) ) end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb new file mode 100644 index 00000000000..e5e8837221e --- /dev/null +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Ensures services which previously recieved all notes events continue + # to recieve confidential ones. + class SetConfidentialNoteEventsOnServices + class Service < ActiveRecord::Base + self.table_name = 'services' + + include ::EachBatch + + def self.services_to_update + where(confidential_note_events: nil, note_events: true) + end + end + + def perform(start_id, stop_id) + Service.services_to_update + .where(id: start_id..stop_id) + .update_all(confidential_note_events: true) + end + end + end +end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb new file mode 100644 index 00000000000..171c8ef21b7 --- /dev/null +++ b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Ensures hooks which previously recieved all notes events continue + # to recieve confidential ones. + class SetConfidentialNoteEventsOnWebhooks + class WebHook < ActiveRecord::Base + self.table_name = 'web_hooks' + + include ::EachBatch + + def self.hooks_to_update + where(confidential_note_events: nil, note_events: true) + end + end + + def perform(start_id, stop_id) + WebHook.hooks_to_update + .where(id: start_id..stop_id) + .update_all(confidential_note_events: true) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index bffbcb86137..f3999e690fa 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -63,7 +63,7 @@ module Gitlab disk_path = project.wiki.disk_path import_url = project.import_url.sub(/\.git\z/, ".git/wiki") - gitlab_shell.import_repository(project.repository_storage_path, disk_path, import_url) + gitlab_shell.import_repository(project.repository_storage, disk_path, import_url) rescue StandardError => e errors << { type: :wiki, errors: e.message } end diff --git a/lib/gitlab/checks/lfs_integrity.rb b/lib/gitlab/checks/lfs_integrity.rb index f7276a380dc..f0e5773ec3c 100644 --- a/lib/gitlab/checks/lfs_integrity.rb +++ b/lib/gitlab/checks/lfs_integrity.rb @@ -15,8 +15,7 @@ module Gitlab return false unless new_lfs_pointers.present? - existing_count = @project.lfs_storage_project - .lfs_objects + existing_count = @project.all_lfs_objects .where(oid: new_lfs_pointers.map(&:lfs_oid)) .count diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index b20d374288f..782f6c4c0af 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -9,7 +9,7 @@ module Gitlab end end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) pipeline.has_kubernetes_active? end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index eadc0948d2f..4aa5dc89f47 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -7,7 +7,7 @@ module Gitlab @patterns = Array(refs) end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) @patterns.any? do |pattern| pattern, path = pattern.split('@', 2) diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index c317291f29d..f09ba42c074 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -15,7 +15,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline) + def satisfied_by?(pipeline, seed = nil) raise NotImplementedError end end diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb new file mode 100644 index 00000000000..9d2a362b7d4 --- /dev/null +++ b/lib/gitlab/ci/build/policy/variables.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Build + module Policy + class Variables < Policy::Specification + def initialize(expressions) + @expressions = Array(expressions) + end + + def satisfied_by?(pipeline, seed) + variables = seed.to_resource.scoped_variables_hash + + statements = @expressions.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement + .new(statement, variables) + end + + statements.any?(&:truthful?) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 0027e9ec8c5..09e8e52b60f 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -25,15 +25,31 @@ module Gitlab include Entry::Validatable include Entry::Attributable - attributes :refs, :kubernetes + attributes :refs, :kubernetes, :variables validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs kubernetes] + validates :config, allowed_keys: %i[refs kubernetes variables] + validate :variables_expressions_syntax with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] + validates :variables, array_of_strings: true + end + + def variables_expressions_syntax + return unless variables.is_a?(Array) + + statements = variables.map do |statement| + ::Gitlab::Ci::Pipeline::Expression::Statement.new(statement) + end + + statements.each do |statement| + unless statement.valid? + errors.add(:variables, "Invalid expression syntax") + end + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index b2b00c8cb4b..d299a5677de 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -17,8 +17,6 @@ module Gitlab # Populate pipeline with all stages and builds from pipeline seeds. # pipeline.stage_seeds.each do |stage| - stage.user = current_user - pipeline.stages << stage.to_resource stage.seeds.each do |build| diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 48bde213d44..346c92dc51e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -4,7 +4,7 @@ module Gitlab module Expression module Lexeme class String < Lexeme::Value - PATTERN = /("(?<string>.+?)")|('(?<string>.+?)')/.freeze + PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze def initialize(value) @value = value diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index b781c15fd67..37643c8ef53 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -11,7 +11,7 @@ module Gitlab end def evaluate(variables = {}) - HashWithIndifferentAccess.new(variables).fetch(@name, nil) + variables.with_indifferent_access.fetch(@name, nil) end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb index 4f0e101b730..09a7c98464b 100644 --- a/lib/gitlab/ci/pipeline/expression/statement.rb +++ b/lib/gitlab/ci/pipeline/expression/statement.rb @@ -14,12 +14,9 @@ module Gitlab %w[variable] ].freeze - def initialize(statement, pipeline) + def initialize(statement, variables = {}) @lexer = Expression::Lexer.new(statement) - - @variables = pipeline.variables.map do |variable| - [variable.key, variable.value] - end + @variables = variables.with_indifferent_access end def parse_tree @@ -35,6 +32,16 @@ module Gitlab def evaluate parse_tree.evaluate(@variables.to_h) end + + def truthful? + evaluate.present? + end + + def valid? + parse_tree.is_a?(Lexeme::Base) + rescue StatementError + false + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 7cd7c864448..6980b0b7aff 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,21 +11,16 @@ module Gitlab @pipeline = pipeline @attributes = attributes - @only = attributes.delete(:only) - @except = attributes.delete(:except) - end - - def user=(current_user) - @attributes.merge!(user: current_user) + @only = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:only)) + @except = Gitlab::Ci::Build::Policy + .fabricate(attributes.delete(:except)) end def included? strong_memoize(:inclusion) do - only_specs = Gitlab::Ci::Build::Policy.fabricate(@only) - except_specs = Gitlab::Ci::Build::Policy.fabricate(@except) - - only_specs.all? { |spec| spec.satisfied_by?(@pipeline) } && - except_specs.none? { |spec| spec.satisfied_by?(@pipeline) } + @only.all? { |spec| spec.satisfied_by?(@pipeline, self) } && + @except.none? { |spec| spec.satisfied_by?(@pipeline, self) } end end @@ -33,6 +28,7 @@ module Gitlab @attributes.merge( pipeline: @pipeline, project: @pipeline.project, + user: @pipeline.user, ref: @pipeline.ref, tag: @pipeline.tag, trigger_request: @pipeline.legacy_trigger, diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 1fcbdc1b15a..c101f30d6e8 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -17,10 +17,6 @@ module Gitlab end end - def user=(current_user) - @builds.each { |seed| seed.user = current_user } - end - def attributes { name: @attributes.fetch(:name), pipeline: @pipeline, diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 2d9166d6bdd..024047d4983 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -23,6 +23,10 @@ module Gitlab 'Cancel' end + def action_button_title + _('Cancel this job') + end + def self.matches?(build, user) build.cancelable? end diff --git a/lib/gitlab/ci/status/build/canceled.rb b/lib/gitlab/ci/status/build/canceled.rb new file mode 100644 index 00000000000..c83e2734a73 --- /dev/null +++ b/lib/gitlab/ci/status/build/canceled.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Canceled < Status::Extended + def illustration + { + image: 'illustrations/canceled-job_empty.svg', + size: 'svg-430', + title: _('This job has been canceled') + } + end + + def self.matches?(build, user) + build.canceled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/created.rb b/lib/gitlab/ci/status/build/created.rb new file mode 100644 index 00000000000..5be8e9de425 --- /dev/null +++ b/lib/gitlab/ci/status/build/created.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Created < Status::Extended + def illustration + { + image: 'illustrations/job_not_triggered.svg', + size: 'svg-306', + title: _('This job has not been triggered yet'), + content: _('This job depends on upstream jobs that need to succeed in order for this job to be triggered') + } + end + + def self.matches?(build, user) + build.created? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb new file mode 100644 index 00000000000..3a5113b16b6 --- /dev/null +++ b/lib/gitlab/ci/status/build/erased.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Erased < Status::Extended + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('Job has been erased') + } + end + + def self.matches?(build, user) + build.erased? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index c852d607373..2b26ebb45a1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -4,12 +4,20 @@ module Gitlab module Build class Factory < Status::Factory def self.extended_statuses - [[Status::Build::Cancelable, + [[Status::Build::Erased, + Status::Build::Manual, + Status::Build::Canceled, + Status::Build::Created, + Status::Build::Pending, + Status::Build::Skipped], + [Status::Build::Cancelable, Status::Build::Retryable], + [Status::Build::Failed], [Status::Build::FailedAllowed, Status::Build::Play, Status::Build::Stop], - [Status::Build::Action]] + [Status::Build::Action], + [Status::Build::Retried]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb new file mode 100644 index 00000000000..155f4fc1343 --- /dev/null +++ b/lib/gitlab/ci/status/build/failed.rb @@ -0,0 +1,40 @@ +module Gitlab + module Ci + module Status + module Build + class Failed < Status::Extended + REASONS = { + 'unknown_failure' => 'unknown failure', + 'script_failure' => 'script failure', + 'api_failure' => 'API failure', + 'stuck_or_timeout_failure' => 'stuck or timeout failure', + 'runner_system_failure' => 'runner system failure', + 'missing_dependency_failure' => 'missing dependency failure' + }.freeze + + def status_tooltip + base_message + end + + def badge_tooltip + base_message + end + + def self.matches?(build, user) + build.failed? + end + + private + + def base_message + "#{s_('CiStatusLabel|failed')} #{description}" + end + + def description + "<br> (#{REASONS[subject.failure_reason]})" + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index dc90f398c7e..ca0046fb1f7 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -4,7 +4,7 @@ module Gitlab module Build class FailedAllowed < Status::Extended def label - 'failed (allowed to fail)' + "failed #{allowed_to_fail_title}" end def icon @@ -15,9 +15,19 @@ module Gitlab 'failed_with_warnings' end + def status_tooltip + "#{@status.status_tooltip} #{allowed_to_fail_title}" + end + def self.matches?(build, user) build.failed? && build.allow_failure? end + + private + + def allowed_to_fail_title + "(allowed to fail)" + end end end end diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb new file mode 100644 index 00000000000..042da6392d3 --- /dev/null +++ b/lib/gitlab/ci/status/build/manual.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Manual < Status::Extended + def illustration + { + image: 'illustrations/manual_action.svg', + size: 'svg-394', + title: _('This job requires a manual action'), + content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + } + end + + def self.matches?(build, user) + build.playable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/pending.rb b/lib/gitlab/ci/status/build/pending.rb new file mode 100644 index 00000000000..9dd9a27ad57 --- /dev/null +++ b/lib/gitlab/ci/status/build/pending.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Pending < Status::Extended + def illustration + { + image: 'illustrations/pending_job_empty.svg', + size: 'svg-430', + title: _('This job has not started yet'), + content: _('This job is in pending state and is waiting to be picked by a runner') + } + end + + def self.matches?(build, user) + build.pending? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index b7b45466d3b..a8b9ebf0803 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -19,6 +19,10 @@ module Gitlab 'Play' end + def action_button_title + _('Trigger this manual action') + end + def action_path play_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/retried.rb b/lib/gitlab/ci/status/build/retried.rb new file mode 100644 index 00000000000..6e190e4ee3c --- /dev/null +++ b/lib/gitlab/ci/status/build/retried.rb @@ -0,0 +1,17 @@ +module Gitlab + module Ci + module Status + module Build + class Retried < Status::Extended + def status_tooltip + @status.status_tooltip + " (retried)" + end + + def self.matches?(build, user) + build.retried? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 44ffe783e50..5aeb8e51480 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -15,6 +15,10 @@ module Gitlab 'Retry' end + def action_button_title + _('Retry this job') + end + def action_path retry_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/skipped.rb b/lib/gitlab/ci/status/build/skipped.rb new file mode 100644 index 00000000000..3e678d0baee --- /dev/null +++ b/lib/gitlab/ci/status/build/skipped.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Skipped < Status::Extended + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('This job has been skipped') + } + end + + def self.matches?(build, user) + build.skipped? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 46e730797e4..dea838bfa39 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -19,6 +19,10 @@ module Gitlab 'Stop' end + def action_button_title + _('Stop this environment') + end + def action_path play_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index d4fd83b93f8..9d6a2f51c11 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -22,6 +22,10 @@ module Gitlab raise NotImplementedError end + def illustration + raise NotImplementedError + end + def label raise NotImplementedError end @@ -57,6 +61,20 @@ module Gitlab def action_title raise NotImplementedError end + + def action_button_title + raise NotImplementedError + end + + # Hint that appears on all the pipeline graph tooltips and builds on the right sidebar in Job detail view + def status_tooltip + label + end + + # Hint that appears on the build badges + def badge_tooltip + subject.status + end end end end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index b3fe3ef1c4d..54894a46077 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :stream - delegate :close, :tell, :seek, :size, :path, :url, :truncate, to: :stream, allow_nil: true + delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true delegate :valid?, to: :stream, as: :present?, allow_nil: true @@ -25,6 +25,10 @@ module Gitlab self.path.present? end + def path + self.stream.path if self.stream.respond_to?(:path) + end + def limit(last_bytes = LIMIT_SIZE) if last_bytes < size stream.seek(-last_bytes, IO::SEEK_END) diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 0deca55fe8f..ad30b3f427c 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -30,7 +30,13 @@ module Gitlab end def to_runner_variables - self.map(&:to_hash) + self.map(&:to_runner_variable) + end + + def to_hash + self.to_runner_variables + .map { |env| [env.fetch(:key), env.fetch(:value)] } + .to_h.with_indifferent_access end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 939912981e6..23ed71db8b0 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -17,7 +17,7 @@ module Gitlab end def ==(other) - to_hash == self.class.fabricate(other).to_hash + to_runner_variable == self.class.fabricate(other).to_runner_variable end ## @@ -25,7 +25,7 @@ module Gitlab # don't expose `file` attribute at all (stems from what the runner # expects). # - def to_hash + def to_runner_variable @variable.reject do |hash_key, hash_value| hash_key == :file && hash_value == false end diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index 50fea1232af..f573368e572 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -9,6 +9,7 @@ module Gitlab # # data = { # object_kind: "note", + # event_type: "confidential_note", # user: { # name: String, # username: String, @@ -51,8 +52,11 @@ module Gitlab end def build_base_data(project, user, note) + event_type = note.confidential? ? 'confidential_note' : 'note' + base_data = { object_kind: "note", + event_type: event_type, user: user.hook_attrs, project_id: project.id, project: project.hook_attrs, diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 1634fe4e9cb..77079e5e72b 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -860,7 +860,7 @@ into similar problems in the future (e.g. when new tables are created). # Each job is scheduled with a `delay_interval` in between. # If you use a small interval, then some jobs may run at the same time. # - # model_class - The table being iterated over + # model_class - The table or relation being iterated over # job_class_name - The background migration job class as a string # delay_interval - The duration between each job's scheduled time (must respond to `to_f`) # batch_size - The maximum number of rows per job diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index d9400e04b83..b2d8ee81977 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -1,12 +1,20 @@ module Gitlab module Database - BINARY_TYPE = if Gitlab::Database.postgresql? - # PostgreSQL defines its own class with slightly different - # behaviour from the default Binary type. - ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea - else - ActiveRecord::Type::Binary - end + BINARY_TYPE = + if Gitlab::Database.postgresql? + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea + else + # In Rails 5.0 `Type` has been moved from `ActiveRecord` to `ActiveModel` + # https://github.com/rails/rails/commit/9cc8c6f3730df3d94c81a55be9ee1b7b4ffd29f6#diff-f8ba7983a51d687976e115adcd95822b + # Remove this method and leave just `ActiveModel::Type::Binary` when removing Gitlab.rails5? code. + if Gitlab.rails5? + ActiveModel::Type::Binary + else + ActiveRecord::Type::Binary + end + end # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). # @@ -16,18 +24,39 @@ module Gitlab class ShaAttribute < BINARY_TYPE PACK_FORMAT = 'H*'.freeze - # Casts binary data to a SHA1 in hexadecimal. + # It is called from activerecord-4.2.10/lib/active_record internal methods. + # Remove this method when removing Gitlab.rails5? code. def type_cast_from_database(value) - value = super + unpack_sha(super) + end + + # It is called from activerecord-4.2.10/lib/active_record internal methods. + # Remove this method when removing Gitlab.rails5? code. + def type_cast_for_database(value) + serialize(value) + end + # It is called from activerecord-5.0.6/lib/active_record/attribute.rb + # Remove this method when removing Gitlab.rails5? code.. + def deserialize(value) + value = Gitlab.rails5? ? super : method(:type_cast_from_database).super_method.call(value) + + unpack_sha(value) + end + + # Rename this method to `deserialize(value)` removing Gitlab.rails5? code. + # Casts binary data to a SHA1 in hexadecimal. + def unpack_sha(value) + # Uncomment this line when removing Gitlab.rails5? code. + # value = super value ? value.unpack(PACK_FORMAT)[0] : nil end # Casts a SHA1 in hexadecimal to the proper binary format. - def type_cast_for_database(value) + def serialize(value) arg = value ? [value].pack(PACK_FORMAT) : nil - super(arg) + Gitlab.rails5? ? super(arg) : method(:type_cast_for_database).super_method.call(arg) end end end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 010b4be7b40..81e91ea0ab7 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -1,11 +1,14 @@ module Gitlab module Diff class InlineDiffMarker < Gitlab::StringRangeMarker + def initialize(line, rich_line = nil) + super(line, rich_line || line) + end + def mark(line_inline_diffs, mode: nil) - mark = super(line_inline_diffs) do |text, left:, right:| + super(line_inline_diffs) do |text, left:, right:| %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>} end - mark.html_safe end private diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index a616a80e8f5..05a60deb7d3 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -14,7 +14,7 @@ module Gitlab end def can_handle? - !incoming_email_token.nil? + !incoming_email_token.nil? && !incoming_email_token.include?("+") && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX) end def execute diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 93037ed8d90..0fb82441bf8 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -231,7 +231,8 @@ module Gitlab # relation to each other. The last 10 commits for a branch for example, # should go through .where def batch_by_oid(repo, oids) - repo.gitaly_migrate(:list_commits_by_oid) do |is_enabled| + repo.gitaly_migrate(:list_commits_by_oid, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled repo.gitaly_commit_client.list_commits_by_oid(oids) else diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 07b7e811a34..c3cb0264112 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -23,7 +23,7 @@ module Gitlab end rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) - rescue Rugged::OdbError, GRPC::BadStatus => e + rescue Rugged::ReferenceError, Rugged::OdbError, GRPC::BadStatus => e raise Gitlab::Git::CommandError.new(e) end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index dc0bc8518bc..099709620b3 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -4,20 +4,14 @@ module Gitlab include Gitlab::Git::Popen include Gitlab::Utils::StrongMemoize - ShardNameNotFoundError = Class.new(StandardError) - - # Absolute path to directory where repositories are stored. - # Example: /home/git/repositories - attr_reader :shard_path + # Name of shard where repositories are stored. + # Example: nfs-file06 + attr_reader :shard_name # Relative path is a directory name for repository with .git at the end. # Example: gitlab-org/gitlab-test.git attr_reader :repository_relative_path - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - attr_reader :repository_absolute_path - # This is the path at which the gitlab-shell hooks directory can be found. # It's essential for integration between git and GitLab proper. All new # repositories should have their hooks directory symlinked here. @@ -25,13 +19,12 @@ module Gitlab attr_reader :logger - def initialize(shard_path, repository_relative_path, global_hooks_path:, logger:) - @shard_path = shard_path + def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) + @shard_name = shard_name @repository_relative_path = repository_relative_path @logger = logger @global_hooks_path = global_hooks_path - @repository_absolute_path = File.join(shard_path, repository_relative_path) @output = StringIO.new end @@ -41,6 +34,22 @@ module Gitlab io.read end + # Absolute path to the repository. + # Example: /home/git/repositorities/gitlab-org/gitlab-test.git + # Probably will be removed when we fully migrate to Gitaly, part of + # https://gitlab.com/gitlab-org/gitaly/issues/1124. + def repository_absolute_path + strong_memoize(:repository_absolute_path) do + File.join(shard_path, repository_relative_path) + end + end + + def shard_path + strong_memoize(:shard_path) do + Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path + end + end + # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) @@ -53,12 +62,12 @@ module Gitlab end end - def fork_repository(new_shard_path, new_repository_relative_path) + def fork_repository(new_shard_name, new_repository_relative_path) Gitlab::GitalyClient.migrate(:fork_repository) do |is_enabled| if is_enabled - gitaly_fork_repository(new_shard_path, new_repository_relative_path) + gitaly_fork_repository(new_shard_name, new_repository_relative_path) else - git_fork_repository(new_shard_path, new_repository_relative_path) + git_fork_repository(new_shard_name, new_repository_relative_path) end end end @@ -205,17 +214,6 @@ module Gitlab private - def shard_name - strong_memoize(:shard_name) do - shard_name_from_shard_path(shard_path) - end - end - - def shard_name_from_shard_path(shard_path) - Gitlab.config.repositories.storages.find { |_, info| info.legacy_disk_path == shard_path }&.first || - raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") - end - def git_import_repository(source, timeout) # Skip import if repo already exists return false if File.exist?(repository_absolute_path) @@ -252,8 +250,9 @@ module Gitlab false end - def git_fork_repository(new_shard_path, new_repository_relative_path) + def git_fork_repository(new_shard_name, new_repository_relative_path) from_path = repository_absolute_path + new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path to_path = File.join(new_shard_path, new_repository_relative_path) # The repository cannot already exist @@ -271,8 +270,8 @@ module Gitlab run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end - def gitaly_fork_repository(new_shard_path, new_repository_relative_path) - target_repository = Gitlab::Git::Repository.new(shard_name_from_shard_path(new_shard_path), new_repository_relative_path, nil) + def gitaly_fork_repository(new_shard_name, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(new_shard_name, new_repository_relative_path, nil) raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) diff --git a/lib/gitlab/git/gitmodules_parser.rb b/lib/gitlab/git/gitmodules_parser.rb index 4a43b9b444d..4b505312f60 100644 --- a/lib/gitlab/git/gitmodules_parser.rb +++ b/lib/gitlab/git/gitmodules_parser.rb @@ -46,6 +46,8 @@ module Gitlab iterator = State.new @content.split("\n").each_with_object(iterator) do |text, iterator| + text.chomp! + next if text =~ /^\s*#/ if text =~ /\A\[submodule "(?<name>[^"]+)"\]\z/ @@ -55,7 +57,7 @@ module Gitlab next unless text =~ /\A\s*(?<key>\w+)\s*=\s*(?<value>.*)\z/ - value = $~[:value].chomp + value = $~[:value] iterator.set_attribute($~[:key], value) end end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 24f027d8da4..7c201c6169b 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -95,13 +95,13 @@ module Gitlab args = [ref, oldrev, newrev] stdout, stderr, status = Open3.capture3(env, path, *args, options) - [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] + [status.success?, Gitlab::Utils.nlbr(stderr.presence || stdout)] end def retrieve_error_message(stderr, stdout) err_message = stderr.read err_message = err_message.blank? ? stdout.read : err_message - err_message.gsub(/\R/, "<br>").html_safe + Gitlab::Utils.nlbr(err_message) end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e692c9ce342..f1b575bd872 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -23,6 +23,7 @@ module Gitlab SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout + EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -31,6 +32,7 @@ module Gitlab DeleteBranchError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) TagExistsError = Class.new(StandardError) + ChecksumError = Class.new(StandardError) class << self # Unlike `new`, `create` takes the repository path @@ -96,7 +98,7 @@ module Gitlab storage_path = Gitlab.config.repositories.storages[@storage].legacy_disk_path @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage_path, + storage, relative_path, global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, logger: Rails.logger @@ -394,17 +396,24 @@ module Gitlab nil end - def archive_prefix(ref, sha) + def archive_prefix(ref, sha, append_sha:) + append_sha = (ref != sha) if append_sha.nil? + project_name = self.name.chomp('.git') - "#{project_name}-#{ref.tr('/', '-')}-#{sha}" + formatted_ref = ref.tr('/', '-') + + prefix_segments = [project_name, formatted_ref] + prefix_segments << sha if append_sha + + prefix_segments.join('-') end - def archive_metadata(ref, storage_path, format = "tar.gz") + def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) return {} if commit.nil? - prefix = archive_prefix(ref, commit.id) + prefix = archive_prefix(ref, commit.id, append_sha: append_sha) { 'RepoPath' => path, @@ -885,7 +894,8 @@ module Gitlab end def delete_refs(*ref_names) - gitaly_migrate(:delete_refs) do |is_enabled| + gitaly_migrate(:delete_refs, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_delete_refs(*ref_names) else @@ -1035,7 +1045,8 @@ module Gitlab end def license_short_name - gitaly_migrate(:license_short_name) do |is_enabled| + gitaly_migrate(:license_short_name, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled gitaly_repository_client.license_short_name else @@ -1361,6 +1372,18 @@ module Gitlab raise CommandError.new(e) end + def clean_stale_repository_files + gitaly_migrate(:repository_cleanup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + gitaly_repository_client.cleanup if is_enabled && exists? + end + rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup + Rails.logger.error("Unable to clean repository on storage #{storage} with path #{path}: #{e.message}") + Gitlab::Metrics.counter( + :failed_repository_cleanup_total, + 'Number of failed repository cleanup events' + ).increment + end + def branch_names_contains_sha(sha) gitaly_migrate(:branch_names_contains_sha) do |is_enabled| if is_enabled @@ -1455,6 +1478,43 @@ module Gitlab run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) end + def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) + base_args = %w(worktree add --detach) + + # Note that we _don't_ want to test for `.present?` here: If the caller + # passes an non nil empty value it means it still wants sparse checkout + # but just isn't interested in any file, perhaps because it wants to + # checkout files in by a changeset but that changeset only adds files. + if sparse_checkout_files + # Create worktree without checking out + run_git!(base_args + ['--no-checkout', worktree_path], env: env) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp + + configure_sparse_checkout(worktree_git_path, sparse_checkout_files) + + # After sparse checkout configuration, checkout `branch` in worktree + run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) + else + # Create worktree and checkout `branch` in it + run_git!(base_args + [worktree_path, branch], env: env) + end + + yield + ensure + FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) + FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) + end + + def checksum + gitaly_migrate(:calculate_checksum) do |is_enabled| + if is_enabled + gitaly_repository_client.calculate_checksum + else + calculate_checksum_by_shelling_out + end + end + end + private def local_write_ref(ref_path, ref, old_ref: nil, shell: true) @@ -1541,33 +1601,6 @@ module Gitlab File.exist?(path) && !clean_stuck_worktree(path) end - def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) - base_args = %w(worktree add --detach) - - # Note that we _don't_ want to test for `.present?` here: If the caller - # passes an non nil empty value it means it still wants sparse checkout - # but just isn't interested in any file, perhaps because it wants to - # checkout files in by a changeset but that changeset only adds files. - if sparse_checkout_files - # Create worktree without checking out - run_git!(base_args + ['--no-checkout', worktree_path], env: env) - worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp - - configure_sparse_checkout(worktree_git_path, sparse_checkout_files) - - # After sparse checkout configuration, checkout `branch` in worktree - run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) - else - # Create worktree and checkout `branch` in it - run_git!(base_args + [worktree_path, branch], env: env) - end - - yield - ensure - FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) - FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) - end - def clean_stuck_worktree(path) return false unless File.mtime(path) < 15.minutes.ago @@ -2400,6 +2433,34 @@ module Gitlab def sha_from_ref(ref) rev_parse_target(ref).oid end + + def calculate_checksum_by_shelling_out + raise NoRepository unless exists? + + args = %W(--git-dir=#{path} show-ref --heads --tags) + output, status = run_git(args) + + if status.nil? || !status.zero? + # Empty repositories return with a non-zero status and an empty output. + return EMPTY_REPOSITORY_CHECKSUM if output&.empty? + + raise ChecksumError, output + end + + refs = output.split("\n") + + result = refs.inject(nil) do |checksum, ref| + value = Digest::SHA1.hexdigest(ref).hex + + if checksum.nil? + value + else + checksum ^ value + end + end + + result.to_s(16) + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index ed0644f6cf1..0d1ee73ca1a 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -29,9 +29,9 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path, :auth_result_type - def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil) + def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil, auth_result_type: nil) @actor = actor @project = project @protocol = protocol @@ -39,6 +39,7 @@ module Gitlab @namespace_path = namespace_path @project_path = project_path @redirected_path = redirected_path + @auth_result_type = auth_result_type end def check(cmd, changes) @@ -78,6 +79,12 @@ module Gitlab authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) end + def request_from_ci_build? + return false unless protocol == 'http' + + auth_result_type == :build || auth_result_type == :ci + end + def protocol_allowed? Gitlab::ProtocolAccess.allowed?(protocol) end @@ -93,6 +100,8 @@ module Gitlab end def check_protocol! + return if request_from_ci_build? + unless protocol_allowed? raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" end @@ -199,6 +208,7 @@ module Gitlab def check_download_access! passed = deploy_key? || + deploy_token? || user_can_download_code? || build_can_download_code? || guest_can_download_code? @@ -229,6 +239,11 @@ module Gitlab end def check_change_access!(changes) + # If there are worktrees with a HEAD pointing to a non-existent object, + # calls to `git rev-list --all` will fail in git 2.15+. This should also + # clear stale lock files. + project.repository.clean_stale_repository_files + changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied @@ -260,6 +275,14 @@ module Gitlab actor.is_a?(DeployKey) end + def deploy_token + actor if deploy_token? + end + + def deploy_token? + actor.is_a?(DeployToken) + end + def ci? actor == :ci end @@ -267,6 +290,8 @@ module Gitlab def can_read_project? if deploy_key? deploy_key.has_access_to?(project) + elsif deploy_token? + deploy_token.has_access_to?(project) elsif user user.can?(:read_project, project) elsif ci? diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index e1bc2f9ab61..6441065f5fe 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -19,6 +19,11 @@ module Gitlab response.exists end + def cleanup + request = Gitaly::CleanupRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :cleanup, request) + end + def garbage_collect(create_bitmap) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) GitalyClient.call(@storage, :repository_service, :garbage_collect, request) @@ -257,6 +262,12 @@ module Gitlab response.license_short_name.presence end + + def calculate_checksum + request = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request) + response.checksum.presence + end end end end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index b1b283e98b5..01168abde6c 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -56,9 +56,8 @@ module Gitlab def import_wiki_repository wiki_path = "#{project.disk_path}.wiki" - storage_path = project.repository_storage_path - gitlab_shell.import_repository(storage_path, wiki_path, wiki_url) + gitlab_shell.import_repository(project.repository_storage, wiki_path, wiki_url) true rescue Gitlab::Shell::Error => e diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb index 4febb0ab430..6ab36676127 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -11,7 +11,8 @@ module Gitlab def build(user: nil, changes: {}) hook_data = { - object_kind: issuable.class.name.underscore, + object_kind: object_kind, + event_type: event_type, user: user.hook_attrs, project: issuable.project.hook_attrs, object_attributes: issuable.hook_attrs, @@ -36,6 +37,18 @@ module Gitlab private + def object_kind + issuable.class.name.underscore + end + + def event_type + if issuable.try(:confidential?) + "confidential_#{object_kind}" + else + object_kind + end + end + def issuable_builder case issuable when Issue diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 96558872a37..9aca3b0fb26 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -4,6 +4,8 @@ # calling internal IP or services. module Gitlab class HTTP + BlockedUrlError = Class.new(StandardError) + include HTTParty # rubocop:disable Gitlab/HTTParty connection_adapter ProxyHTTPConnectionAdapter diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb new file mode 100644 index 00000000000..aef371d81eb --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -0,0 +1,83 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class BaseAfterExportStrategy + include ActiveModel::Validations + extend Forwardable + + StrategyError = Class.new(StandardError) + + AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'.freeze + + private + + attr_reader :project, :current_user + + public + + def initialize(attributes = {}) + @options = OpenStruct.new(attributes) + + self.class.instance_eval do + def_delegators :@options, *attributes.keys + end + end + + def execute(current_user, project) + return unless project&.export_project_path + + @project = project + @current_user = current_user + + if invalid? + log_validation_errors + + return + end + + create_or_update_after_export_lock + strategy_execute + + true + rescue => e + project.import_export_shared.error(e) + false + ensure + delete_after_export_lock + end + + def to_json(options = {}) + @options.to_h.merge!(klass: self.class.name).to_json + end + + def self.lock_file_path(project) + return unless project&.export_path + + File.join(project.export_path, AFTER_EXPORT_LOCK_FILE_NAME) + end + + protected + + def strategy_execute + raise NotImplementedError + end + + private + + def create_or_update_after_export_lock + FileUtils.touch(self.class.lock_file_path(project)) + end + + def delete_after_export_lock + lock_file = self.class.lock_file_path(project) + + FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file) + end + + def log_validation_errors + errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb new file mode 100644 index 00000000000..4371a7eff56 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb @@ -0,0 +1,17 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class DownloadNotificationStrategy < BaseAfterExportStrategy + private + + def strategy_execute + notification_service.project_exported(project, current_user) + end + + def notification_service + @notification_service ||= NotificationService.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb new file mode 100644 index 00000000000..938664a95a1 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -0,0 +1,61 @@ +module Gitlab + module ImportExport + module AfterExportStrategies + class WebUploadStrategy < BaseAfterExportStrategy + PUT_METHOD = 'PUT'.freeze + POST_METHOD = 'POST'.freeze + INVALID_HTTP_METHOD = 'invalid. Only PUT and POST methods allowed.'.freeze + + validates :url, url: true + + validate do + unless [PUT_METHOD, POST_METHOD].include?(http_method.upcase) + errors.add(:http_method, INVALID_HTTP_METHOD) + end + end + + def initialize(url:, http_method: PUT_METHOD) + super + end + + protected + + def strategy_execute + handle_response_error(send_file) + + project.remove_exported_project_file + end + + def handle_response_error(response) + unless response.success? + error_code = response.dig('Error', 'Code') || response.code + error_message = response.dig('Error', 'Message') || response.message + + raise StrategyError.new("Error uploading the project. Code #{error_code}: #{error_message}") + end + end + + private + + def send_file + export_file = File.open(project.export_project_path) + + Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options(export_file)) # rubocop:disable GitlabSecurity/PublicSend + ensure + export_file.close if export_file + end + + def send_file_options(export_file) + { + body_stream: export_file, + headers: headers + } + end + + def headers + { 'Content-Length' => File.size(project.export_project_path).to_s } + end + end + end + end +end diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb new file mode 100644 index 00000000000..7eabcae2380 --- /dev/null +++ b/lib/gitlab/import_export/after_export_strategy_builder.rb @@ -0,0 +1,24 @@ +module Gitlab + module ImportExport + class AfterExportStrategyBuilder + StrategyNotFoundError = Class.new(StandardError) + + def self.build!(strategy_klass, attributes = {}) + return default_strategy.new unless strategy_klass + + attributes ||= {} + klass = strategy_klass.constantize rescue nil + + unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy + raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found") + end + + klass.new(**attributes.symbolize_keys) + end + + def self.default_strategy + AfterExportStrategies::DownloadNotificationStrategy + end + end + end +end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 4bdd01f5e94..cd840bd5b01 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -105,6 +105,7 @@ excluded_attributes: - :last_repository_updated_at - :last_repository_check_at - :storage_version + - :description_html snippets: - :expired_at merge_request_diff: @@ -124,6 +125,8 @@ excluded_attributes: - :trace - :token - :when + - :artifacts_file + - :artifacts_metadata push_event_payload: - :event_id project_badges: @@ -144,8 +147,6 @@ methods: - :diff_head_sha - :source_branch_sha - :target_branch_sha - project: - - :description_html events: - :action push_event_payload: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c38df9102eb..63cab07324a 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,6 +1,9 @@ module Gitlab module ImportExport class Importer + include Gitlab::Allowable + include Gitlab::Utils::StrongMemoize + def self.imports_repository? true end @@ -13,17 +16,24 @@ module Gitlab end def execute - if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore) + if import_file && check_version! && restorers.all?(&:restore) && overwrite_project project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) end - + rescue => e + raise Projects::ImportService::Error.new(e.message) + ensure remove_import_file end private + def restorers + [repo_restorer, wiki_restorer, project_tree, avatar_restorer, + uploads_restorer, lfs_restorer, statistics_restorer] + end + def import_file Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, shared: @shared) @@ -60,6 +70,14 @@ module Gitlab Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) end + def lfs_restorer + Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + + def statistics_restorer + Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + def path_with_namespace File.join(@project.namespace.full_path, @project.path) end @@ -75,6 +93,33 @@ module Gitlab def remove_import_file FileUtils.rm_rf(@archive_file) end + + def overwrite_project + project = project_tree.restored_project + + return unless can?(@current_user, :admin_namespace, project.namespace) + + if overwrite_project? + ::Projects::OverwriteProjectService.new(project, @current_user) + .execute(project_to_overwrite) + end + + true + end + + def original_path + @project.import_data&.data&.fetch('original_path', nil) + end + + def overwrite_project? + original_path.present? && project_to_overwrite.present? + end + + def project_to_overwrite + strong_memoize(:project_to_overwrite) do + Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}") + end + end end end end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb new file mode 100644 index 00000000000..b28c3c161b7 --- /dev/null +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -0,0 +1,43 @@ +module Gitlab + module ImportExport + class LfsRestorer + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + return true if lfs_file_paths.empty? + + lfs_file_paths.each do |file_path| + link_or_create_lfs_object!(file_path) + end + + true + rescue => e + @shared.error(e) + false + end + + private + + def link_or_create_lfs_object!(path) + size = File.size(path) + oid = LfsObject.calculate_oid(path) + + lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size) + lfs_object.file = File.open(path) unless lfs_object.file&.exists? + + @project.all_lfs_objects << lfs_object + end + + def lfs_file_paths + @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*") + end + + def lfs_storage_path + File.join(@shared.export_path, 'lfs-objects') + end + end + end +end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb new file mode 100644 index 00000000000..29410e2331c --- /dev/null +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -0,0 +1,55 @@ +module Gitlab + module ImportExport + class LfsSaver + include Gitlab::ImportExport::CommandLineUtil + + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def save + @project.all_lfs_objects.each do |lfs_object| + save_lfs_object(lfs_object) + end + + true + rescue => e + @shared.error(e) + + false + end + + private + + def save_lfs_object(lfs_object) + if lfs_object.local_store? + copy_file_for_lfs_object(lfs_object) + else + download_file_for_lfs_object(lfs_object) + end + end + + def download_file_for_lfs_object(lfs_object) + destination = destination_path_for_object(lfs_object) + mkdir_p(File.dirname(destination)) + + File.open(destination, 'w') do |file| + IO.copy_stream(URI.parse(lfs_object.file.url).open, file) + end + end + + def copy_file_for_lfs_object(lfs_object) + copy_files(lfs_object.file.path, destination_path_for_object(lfs_object)) + end + + def destination_path_for_object(lfs_object) + File.join(lfs_export_path, lfs_object.oid) + end + + def lfs_export_path + File.join(@shared.export_path, 'lfs-objects') + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 8f5bb8f9597..d5590dde40f 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -77,24 +77,31 @@ module Gitlab end def default_relation_list - Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model| + reader.tree.reject do |model| model.is_a?(Hash) && model[:project_members] end end def restore_project - params = project_params - - if params[:description].present? - params[:description_html] = nil - end - - @project.update_columns(params) + @project.update_columns(project_params) @project end def project_params - @tree_hash.reject do |key, value| + @project_params ||= json_params.merge(override_params) + end + + def override_params + return {} unless params = @project.import_data&.data&.fetch('override_params', nil) + + @override_params ||= params.select do |key, _value| + Project.column_names.include?(key.to_s) && + !reader.project_tree[:except].include?(key.to_sym) + end + end + + def json_params + @json_params ||= @tree_hash.reject do |key, value| # return params that are not 1 to many or 1 to 1 relations value.respond_to?(:each) && !Project.column_names.include?(key) end @@ -181,6 +188,10 @@ module Gitlab relation_hash.merge(params) end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 791a54e1b69..598832fb2df 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -19,7 +19,7 @@ module Gitlab custom_attributes: 'ProjectCustomAttribute', project_badges: 'Badge' }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id closed_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 3d3d998a6a3..6d7c36ce38b 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -22,7 +22,7 @@ module Gitlab def error(error) error_out(error.message, caller[0].dup) - @errors << error.message + add_error_message(error.message) # Debug: if error.backtrace @@ -32,6 +32,14 @@ module Gitlab end end + def add_error_message(error_message) + @errors << error_message + end + + def after_export_in_progress? + File.exist?(after_export_lock_file) + end + private def relative_path @@ -45,6 +53,10 @@ module Gitlab def error_out(message, caller) Rails.logger.error("Import/Export error raised on #{caller}: #{message}") end + + def after_export_lock_file + AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project) + end end end end diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb new file mode 100644 index 00000000000..bcdd9c12c85 --- /dev/null +++ b/lib/gitlab/import_export/statistics_restorer.rb @@ -0,0 +1,17 @@ +module Gitlab + module ImportExport + class StatisticsRestorer + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + @project.statistics.refresh! + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 0526ef9eb13..7edd0ad2033 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -259,7 +259,7 @@ module Gitlab def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(project.repository_storage_path, wiki.disk_path, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage, wiki.disk_path, wiki.import_url) end rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, diff --git a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb index db8bdde74b2..47b4af5d649 100644 --- a/lib/gitlab/metrics/sidekiq_metrics_exporter.rb +++ b/lib/gitlab/metrics/sidekiq_metrics_exporter.rb @@ -4,6 +4,8 @@ require 'prometheus/client/rack/exporter' module Gitlab module Metrics class SidekiqMetricsExporter < Daemon + LOG_FILENAME = File.join(Rails.root, 'log', 'sidekiq_exporter.log') + def enabled? Gitlab::Metrics.metrics_folder_present? && settings.enabled end @@ -17,7 +19,13 @@ module Gitlab attr_reader :server def start_working - @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address) + logger = WEBrick::Log.new(LOG_FILENAME) + access_log = [ + [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] + ] + + @server = ::WEBrick::HTTPServer.new(Port: settings.port, BindAddress: settings.address, + Logger: logger, AccessLog: access_log) server.mount "/", Rack::Handler::WEBrick, rack_app server.start end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index d4c54049b74..a5f5d719cc1 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -82,7 +82,7 @@ module Gitlab end def open_file(path, name) - ::UploadedFile.new(path, name || File.basename(path), 'application/octet-stream') + ::UploadedFile.new(path, filename: name || File.basename(path), content_type: 'application/octet-stream') end end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index 6c2b2036074..92a308a12dc 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -5,6 +5,7 @@ module Gitlab def self.enabled?(user = nil) return true if Rails.env.development? + return true if user&.admin? return false unless user && allowed_group_id allowed_user_ids.include?(user.id) diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index aad76e335af..f5879de1e94 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -79,7 +79,7 @@ module Gitlab def common_query_context(environment, timeframe_start:, timeframe_end:) base_query_context(timeframe_start, timeframe_end).merge({ ci_environment_slug: environment.slug, - kube_namespace: environment.project.deployment_platform&.actual_namespace || '', + kube_namespace: environment.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} }) end diff --git a/lib/gitlab/proxy_http_connection_adapter.rb b/lib/gitlab/proxy_http_connection_adapter.rb index c70d6f4cd84..d682289b632 100644 --- a/lib/gitlab/proxy_http_connection_adapter.rb +++ b/lib/gitlab/proxy_http_connection_adapter.rb @@ -10,8 +10,12 @@ module Gitlab class ProxyHTTPConnectionAdapter < HTTParty::ConnectionAdapter def connection - if !allow_local_requests? && blocked_url? - raise URI::InvalidURIError + unless allow_local_requests? + begin + Gitlab::UrlBlocker.validate!(uri, allow_local_network: false) + rescue Gitlab::UrlBlocker::BlockedUrlError => e + raise Gitlab::HTTP::BlockedUrlError, "URL '#{uri}' is blocked: #{e.message}" + end end super @@ -19,10 +23,6 @@ module Gitlab private - def blocked_url? - Gitlab::UrlBlocker.blocked_url?(uri, allow_private_networks: false) - end - def allow_local_requests? options.fetch(:allow_local_requests, allow_settings_local_requests?) end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index c8c15b9684a..67407b651a5 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -93,12 +93,12 @@ module Gitlab # Import repository # - # storage - project's storage path + # storage - project's storage name # name - project disk path # url - URL to import from # # Ex. - # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") + # import_repository("nfs-file06", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) @@ -131,8 +131,7 @@ module Gitlab if is_enabled repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) else - storage_path = Gitlab.config.repositories.storages[repository.storage].legacy_disk_path - local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) + local_fetch_remote(repository.storage, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end end end @@ -156,13 +155,13 @@ module Gitlab end # Fork repository to new path - # forked_from_storage - forked-from project's storage path - # forked_from_disk_path - project disk path - # forked_to_storage - forked-to project's storage path - # forked_to_disk_path - forked project disk path + # forked_from_storage - forked-from project's storage name + # forked_from_disk_path - project disk relative path + # forked_to_storage - forked-to project's storage name + # forked_to_disk_path - forked project disk relative path # # Ex. - # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") + # fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci") # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) @@ -420,16 +419,16 @@ module Gitlab private - def gitlab_projects(shard_path, disk_path) + def gitlab_projects(shard_name, disk_path) Gitlab::Git::GitlabProjects.new( - shard_path, + shard_name, disk_path, global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, logger: Rails.logger ) end - def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + def local_fetch_remote(storage_name, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) vars = { force: forced, tags: !no_tags, prune: prune } if ssh_auth&.ssh_import? @@ -442,7 +441,7 @@ module Gitlab end end - cmd = gitlab_projects(storage_path, repository_relative_path) + cmd = gitlab_projects(storage_name, repository_relative_path) success = cmd.fetch_remote(remote, git_timeout, vars) diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb new file mode 100644 index 00000000000..98f8222fd03 --- /dev/null +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -0,0 +1,21 @@ +module Gitlab + module SidekiqLogging + class JSONFormatter + def call(severity, timestamp, progname, data) + output = { + severity: severity, + time: timestamp.utc.iso8601(3) + } + + case data + when String + output[:message] = data + when Hash + output.merge!(data) + end + + output.to_json + "\n" + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb new file mode 100644 index 00000000000..9a89ae70b98 --- /dev/null +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -0,0 +1,96 @@ +module Gitlab + module SidekiqLogging + class StructuredLogger + START_TIMESTAMP_FIELDS = %w[created_at enqueued_at].freeze + DONE_TIMESTAMP_FIELDS = %w[started_at retried_at failed_at completed_at].freeze + + def call(job, queue) + started_at = current_time + base_payload = parse_job(job) + + Sidekiq.logger.info log_job_start(started_at, base_payload) + + yield + + Sidekiq.logger.info log_job_done(started_at, base_payload) + rescue => job_exception + Sidekiq.logger.warn log_job_done(started_at, base_payload, job_exception) + + raise + end + + private + + def base_message(payload) + "#{payload['class']} JID-#{payload['jid']}" + end + + def log_job_start(started_at, payload) + payload['message'] = "#{base_message(payload)}: start" + payload['job_status'] = 'start' + + payload + end + + def log_job_done(started_at, payload, job_exception = nil) + payload = payload.dup + payload['duration'] = elapsed(started_at) + payload['completed_at'] = Time.now.utc + + message = base_message(payload) + + if job_exception + payload['message'] = "#{message}: fail: #{payload['duration']} sec" + payload['job_status'] = 'fail' + payload['error_message'] = job_exception.message + payload['error'] = job_exception.class + payload['error_backtrace'] = backtrace_cleaner.clean(job_exception.backtrace) + else + payload['message'] = "#{message}: done: #{payload['duration']} sec" + payload['job_status'] = 'done' + end + + convert_to_iso8601(payload, DONE_TIMESTAMP_FIELDS) + + payload + end + + def parse_job(job) + job = job.dup + + # Add process id params + job['pid'] = ::Process.pid + + job.delete('args') unless ENV['SIDEKIQ_LOG_ARGUMENTS'] + + convert_to_iso8601(job, START_TIMESTAMP_FIELDS) + + job + end + + def convert_to_iso8601(payload, keys) + keys.each do |key| + payload[key] = format_time(payload[key]) if payload[key] + end + end + + def elapsed(start) + (current_time - start).round(3) + end + + def current_time + Gitlab::Metrics::System.monotonic_time + end + + def backtrace_cleaner + @backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new + end + + def format_time(timestamp) + return timestamp if timestamp.is_a?(String) + + Time.at(timestamp).utc.iso8601(3) + end + end + end +end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 0f9f939e204..db97f65bd54 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -2,48 +2,84 @@ require 'resolv' module Gitlab class UrlBlocker - class << self - def blocked_url?(url, allow_private_networks: true, valid_ports: []) - return false if url.nil? + BlockedUrlError = Class.new(StandardError) - blocked_ips = ["127.0.0.1", "::1", "0.0.0.0"] - blocked_ips.concat(Socket.ip_address_list.map(&:ip_address)) + class << self + def validate!(url, allow_localhost: false, allow_local_network: true, valid_ports: []) + return true if url.nil? begin uri = Addressable::URI.parse(url) - # Allow imports from the GitLab instance itself but only from the configured ports - return false if internal?(uri) + rescue Addressable::URI::InvalidURIError + raise BlockedUrlError, "URI is invalid" + end - return true if blocked_port?(uri.port, valid_ports) - return true if blocked_user_or_hostname?(uri.user) - return true if blocked_user_or_hostname?(uri.hostname) + # Allow imports from the GitLab instance itself but only from the configured ports + return true if internal?(uri) - addrs_info = Addrinfo.getaddrinfo(uri.hostname, 80, nil, :STREAM) - server_ips = addrs_info.map(&:ip_address) + port = uri.port || uri.default_port + validate_port!(port, valid_ports) if valid_ports.any? + validate_user!(uri.user) + validate_hostname!(uri.hostname) - return true if (blocked_ips & server_ips).any? - return true if !allow_private_networks && private_network?(addrs_info) - rescue Addressable::URI::InvalidURIError - return true + begin + addrs_info = Addrinfo.getaddrinfo(uri.hostname, port, nil, :STREAM) rescue SocketError - return false + return true end + validate_localhost!(addrs_info) unless allow_localhost + validate_local_network!(addrs_info) unless allow_local_network + + true + end + + def blocked_url?(*args) + validate!(*args) + false + rescue BlockedUrlError + true end private - def blocked_port?(port, valid_ports) - return false if port.blank? || valid_ports.blank? + def validate_port!(port, valid_ports) + return if port.blank? + # Only ports under 1024 are restricted + return if port >= 1024 + return if valid_ports.include?(port) - port < 1024 && !valid_ports.include?(port) + raise BlockedUrlError, "Only allowed ports are #{valid_ports.join(', ')}, and any over 1024" end - def blocked_user_or_hostname?(value) - return false if value.blank? + def validate_user!(value) + return if value.blank? + return if value =~ /\A\p{Alnum}/ - value !~ /\A\p{Alnum}/ + raise BlockedUrlError, "Username needs to start with an alphanumeric character" + end + + def validate_hostname!(value) + return if value.blank? + return if value =~ /\A\p{Alnum}/ + + raise BlockedUrlError, "Hostname needs to start with an alphanumeric character" + end + + def validate_localhost!(addrs_info) + local_ips = ["127.0.0.1", "::1", "0.0.0.0"] + local_ips.concat(Socket.ip_address_list.map(&:ip_address)) + + return if (local_ips & addrs_info.map(&:ip_address)).empty? + + raise BlockedUrlError, "Requests to localhost are not allowed" + end + + def validate_local_network!(addrs_info) + return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } + + raise BlockedUrlError, "Requests to the local network are not allowed" end def internal?(uri) @@ -60,10 +96,6 @@ module Gitlab (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) end - def private_network?(addrs_info) - addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? } - end - def config Gitlab.config end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 37d3512990e..8c0a4d55ea2 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -30,6 +30,7 @@ module Gitlab usage_data end + # rubocop:disable Metrics/AbcSize def system_usage_data { counts: { @@ -50,6 +51,12 @@ module Gitlab clusters: ::Clusters::Cluster.count, clusters_enabled: ::Clusters::Cluster.enabled.count, clusters_disabled: ::Clusters::Cluster.disabled.count, + clusters_platforms_gke: ::Clusters::Cluster.gcp_installed.enabled.count, + clusters_platforms_user: ::Clusters::Cluster.user_provided.enabled.count, + clusters_applications_helm: ::Clusters::Applications::Helm.installed.count, + clusters_applications_ingress: ::Clusters::Applications::Ingress.installed.count, + clusters_applications_prometheus: ::Clusters::Applications::Prometheus.installed.count, + clusters_applications_runner: ::Clusters::Applications::Runner.installed.count, in_review_folder: ::Environment.in_review_folder.count, groups: Group.count, issues: Issue.count, diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index dc9391f32cf..b0a492eaa58 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -27,6 +27,11 @@ module Gitlab .gsub(/(\A-+|-+\z)/, '') end + # Converts newlines into HTML line break elements + def nlbr(str) + ActionView::Base.full_sanitizer.sanitize(str, tags: []).gsub(/\r?\n/, '<br>').html_safe + end + def remove_line_breaks(str) str.gsub(/\r?\n/, '') end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index b102812ec12..153cb2a8bb1 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -36,10 +36,6 @@ module Gitlab } end - def artifact_upload_ok - { TempPath: JobArtifactUploader.workhorse_upload_path } - end - def send_git_blob(repository, blob) params = if Gitlab::GitalyClient.feature_enabled?(:workhorse_raw_show, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) { @@ -63,10 +59,10 @@ module Gitlab ] end - def send_git_archive(repository, ref:, format:) + def send_git_archive(repository, ref:, format:, append_sha:) format ||= 'tar.gz' format.downcase! - params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) + params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha) raise "Repository or ref not found" if params.empty? if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) |