diff options
Diffstat (limited to 'lib/gitlab')
27 files changed, 408 insertions, 137 deletions
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 6af763faf10..2a44e11efb6 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -5,7 +5,7 @@ module Gitlab REGISTRY_SCOPES = [:read_registry].freeze # Scopes used for GitLab API access - API_SCOPES = [:api, :read_user, :sudo].freeze + API_SCOPES = [:api, :read_user, :sudo, :read_repository].freeze # Scopes used for OpenID Connect OPENID_SCOPES = [:openid].freeze @@ -26,6 +26,7 @@ module Gitlab lfs_token_check(login, password, project) || oauth_access_token_check(login, password) || personal_access_token_check(password) || + deploy_token_check(login, password) || user_with_password_for_git(login, password) || Gitlab::Auth::Result.new @@ -163,7 +164,8 @@ module Gitlab def abilities_for_scopes(scopes) abilities_by_scope = { api: full_authentication_abilities, - read_registry: [:read_container_image] + read_registry: [:read_container_image], + read_repository: [:download_code] } scopes.flat_map do |scope| @@ -171,6 +173,22 @@ module Gitlab end.uniq end + def deploy_token_check(login, password) + return unless password.present? + + token = + DeployToken.active.find_by(token: password) + + return unless token && login + return if login != token.username + + scopes = abilities_for_scopes(token.scopes) + + if valid_scoped_token?(token, available_scopes) + Gitlab::Auth::Result.new(token, token.project, :deploy_token, scopes) + end + end + def lfs_token_check(login, password, project) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb index fd5cbf76e47..a357538a885 100644 --- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb +++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb @@ -96,7 +96,7 @@ module Gitlab commit_hash.merge( merge_request_diff_id: merge_request_diff.id, relative_order: index, - sha: sha_attribute.type_cast_for_database(sha) + sha: sha_attribute.serialize(sha) ) end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 2d9166d6bdd..024047d4983 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -23,6 +23,10 @@ module Gitlab 'Cancel' end + def action_button_title + _('Cancel this job') + end + def self.matches?(build, user) build.cancelable? end diff --git a/lib/gitlab/ci/status/build/canceled.rb b/lib/gitlab/ci/status/build/canceled.rb new file mode 100644 index 00000000000..c83e2734a73 --- /dev/null +++ b/lib/gitlab/ci/status/build/canceled.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Canceled < Status::Extended + def illustration + { + image: 'illustrations/canceled-job_empty.svg', + size: 'svg-430', + title: _('This job has been canceled') + } + end + + def self.matches?(build, user) + build.canceled? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index c0c7c7f5b5d..c1fc70ac266 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -3,6 +3,14 @@ module Gitlab module Status module Build module Common + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('This job does not have a trace.') + } + end + def has_details? can?(user, :read_build, subject) end diff --git a/lib/gitlab/ci/status/build/created.rb b/lib/gitlab/ci/status/build/created.rb new file mode 100644 index 00000000000..5be8e9de425 --- /dev/null +++ b/lib/gitlab/ci/status/build/created.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Created < Status::Extended + def illustration + { + image: 'illustrations/job_not_triggered.svg', + size: 'svg-306', + title: _('This job has not been triggered yet'), + content: _('This job depends on upstream jobs that need to succeed in order for this job to be triggered') + } + end + + def self.matches?(build, user) + build.created? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb new file mode 100644 index 00000000000..3a5113b16b6 --- /dev/null +++ b/lib/gitlab/ci/status/build/erased.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Erased < Status::Extended + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('Job has been erased') + } + end + + def self.matches?(build, user) + build.erased? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 20a319caf86..2b26ebb45a1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -4,7 +4,13 @@ module Gitlab module Build class Factory < Status::Factory def self.extended_statuses - [[Status::Build::Cancelable, + [[Status::Build::Erased, + Status::Build::Manual, + Status::Build::Canceled, + Status::Build::Created, + Status::Build::Pending, + Status::Build::Skipped], + [Status::Build::Cancelable, Status::Build::Retryable], [Status::Build::Failed], [Status::Build::FailedAllowed, diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb new file mode 100644 index 00000000000..042da6392d3 --- /dev/null +++ b/lib/gitlab/ci/status/build/manual.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Manual < Status::Extended + def illustration + { + image: 'illustrations/manual_action.svg', + size: 'svg-394', + title: _('This job requires a manual action'), + content: _('This job depends on a user to trigger its process. Often they are used to deploy code to production environments') + } + end + + def self.matches?(build, user) + build.playable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/pending.rb b/lib/gitlab/ci/status/build/pending.rb new file mode 100644 index 00000000000..9dd9a27ad57 --- /dev/null +++ b/lib/gitlab/ci/status/build/pending.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module Build + class Pending < Status::Extended + def illustration + { + image: 'illustrations/pending_job_empty.svg', + size: 'svg-430', + title: _('This job has not started yet'), + content: _('This job is in pending state and is waiting to be picked by a runner') + } + end + + def self.matches?(build, user) + build.pending? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index b7b45466d3b..a8b9ebf0803 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -19,6 +19,10 @@ module Gitlab 'Play' end + def action_button_title + _('Trigger this manual action') + end + def action_path play_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 44ffe783e50..5aeb8e51480 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -15,6 +15,10 @@ module Gitlab 'Retry' end + def action_button_title + _('Retry this job') + end + def action_path retry_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/build/skipped.rb b/lib/gitlab/ci/status/build/skipped.rb new file mode 100644 index 00000000000..3e678d0baee --- /dev/null +++ b/lib/gitlab/ci/status/build/skipped.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Skipped < Status::Extended + def illustration + { + image: 'illustrations/skipped-job_empty.svg', + size: 'svg-430', + title: _('This job has been skipped') + } + end + + def self.matches?(build, user) + build.skipped? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 46e730797e4..dea838bfa39 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -19,6 +19,10 @@ module Gitlab 'Stop' end + def action_button_title + _('Stop this environment') + end + def action_path play_project_job_path(subject.project, subject) end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index daab6bb2de5..9d6a2f51c11 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -22,6 +22,10 @@ module Gitlab raise NotImplementedError end + def illustration + raise NotImplementedError + end + def label raise NotImplementedError end @@ -58,6 +62,10 @@ module Gitlab raise NotImplementedError end + def action_button_title + raise NotImplementedError + end + # Hint that appears on all the pipeline graph tooltips and builds on the right sidebar in Job detail view def status_tooltip label diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb index d9400e04b83..b2d8ee81977 100644 --- a/lib/gitlab/database/sha_attribute.rb +++ b/lib/gitlab/database/sha_attribute.rb @@ -1,12 +1,20 @@ module Gitlab module Database - BINARY_TYPE = if Gitlab::Database.postgresql? - # PostgreSQL defines its own class with slightly different - # behaviour from the default Binary type. - ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea - else - ActiveRecord::Type::Binary - end + BINARY_TYPE = + if Gitlab::Database.postgresql? + # PostgreSQL defines its own class with slightly different + # behaviour from the default Binary type. + ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea + else + # In Rails 5.0 `Type` has been moved from `ActiveRecord` to `ActiveModel` + # https://github.com/rails/rails/commit/9cc8c6f3730df3d94c81a55be9ee1b7b4ffd29f6#diff-f8ba7983a51d687976e115adcd95822b + # Remove this method and leave just `ActiveModel::Type::Binary` when removing Gitlab.rails5? code. + if Gitlab.rails5? + ActiveModel::Type::Binary + else + ActiveRecord::Type::Binary + end + end # Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa). # @@ -16,18 +24,39 @@ module Gitlab class ShaAttribute < BINARY_TYPE PACK_FORMAT = 'H*'.freeze - # Casts binary data to a SHA1 in hexadecimal. + # It is called from activerecord-4.2.10/lib/active_record internal methods. + # Remove this method when removing Gitlab.rails5? code. def type_cast_from_database(value) - value = super + unpack_sha(super) + end + + # It is called from activerecord-4.2.10/lib/active_record internal methods. + # Remove this method when removing Gitlab.rails5? code. + def type_cast_for_database(value) + serialize(value) + end + # It is called from activerecord-5.0.6/lib/active_record/attribute.rb + # Remove this method when removing Gitlab.rails5? code.. + def deserialize(value) + value = Gitlab.rails5? ? super : method(:type_cast_from_database).super_method.call(value) + + unpack_sha(value) + end + + # Rename this method to `deserialize(value)` removing Gitlab.rails5? code. + # Casts binary data to a SHA1 in hexadecimal. + def unpack_sha(value) + # Uncomment this line when removing Gitlab.rails5? code. + # value = super value ? value.unpack(PACK_FORMAT)[0] : nil end # Casts a SHA1 in hexadecimal to the proper binary format. - def type_cast_for_database(value) + def serialize(value) arg = value ? [value].pack(PACK_FORMAT) : nil - super(arg) + Gitlab.rails5? ? super(arg) : method(:type_cast_for_database).super_method.call(arg) end end end diff --git a/lib/gitlab/git/checksum.rb b/lib/gitlab/git/checksum.rb deleted file mode 100644 index 3ef0f0a8854..00000000000 --- a/lib/gitlab/git/checksum.rb +++ /dev/null @@ -1,82 +0,0 @@ -module Gitlab - module Git - class Checksum - include Gitlab::Git::Popen - - EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze - - Failure = Class.new(StandardError) - - attr_reader :path, :relative_path, :storage, :storage_path - - def initialize(storage, relative_path) - @storage = storage - @storage_path = Gitlab.config.repositories.storages[storage].legacy_disk_path - @relative_path = "#{relative_path}.git" - @path = File.join(storage_path, @relative_path) - end - - def calculate - unless repository_exists? - failure!(Gitlab::Git::Repository::NoRepository, 'No repository for such path') - end - - calculate_checksum_by_shelling_out - end - - private - - def repository_exists? - raw_repository.exists? - end - - def calculate_checksum_by_shelling_out - args = %W(--git-dir=#{path} show-ref --heads --tags) - output, status = run_git(args) - - if status&.zero? - refs = output.split("\n") - - result = refs.inject(nil) do |checksum, ref| - value = Digest::SHA1.hexdigest(ref).hex - - if checksum.nil? - value - else - checksum ^ value - end - end - - result.to_s(16) - else - # Empty repositories return with a non-zero status and an empty output. - if output&.empty? - EMPTY_REPOSITORY_CHECKSUM - else - failure!(Gitlab::Git::Checksum::Failure, output) - end - end - end - - def failure!(klass, message) - Gitlab::GitLogger.error("'git show-ref --heads --tags' in #{path}: #{message}") - - raise klass.new("Could not calculate the checksum for #{path}: #{message}") - end - - def circuit_breaker - @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) - end - - def raw_repository - Gitlab::Git::Repository.new(storage, relative_path, nil) - end - - def run_git(args) - circuit_breaker.perform do - popen([Gitlab.config.git.bin_path, *args], path) - end - end - end - end -end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 07b7e811a34..c3cb0264112 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -23,7 +23,7 @@ module Gitlab end rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) - rescue Rugged::OdbError, GRPC::BadStatus => e + rescue Rugged::ReferenceError, Rugged::OdbError, GRPC::BadStatus => e raise Gitlab::Git::CommandError.new(e) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 8d97bfb0e6a..79cacd9f6f5 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -23,6 +23,7 @@ module Gitlab SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout + EMPTY_REPOSITORY_CHECKSUM = '0000000000000000000000000000000000000000'.freeze NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -31,6 +32,7 @@ module Gitlab DeleteBranchError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) TagExistsError = Class.new(StandardError) + ChecksumError = Class.new(StandardError) class << self # Unlike `new`, `create` takes the repository path @@ -394,17 +396,24 @@ module Gitlab nil end - def archive_prefix(ref, sha) + def archive_prefix(ref, sha, append_sha:) + append_sha = (ref != sha) if append_sha.nil? + project_name = self.name.chomp('.git') - "#{project_name}-#{ref.tr('/', '-')}-#{sha}" + formatted_ref = ref.tr('/', '-') + + prefix_segments = [project_name, formatted_ref] + prefix_segments << sha if append_sha + + prefix_segments.join('-') end - def archive_metadata(ref, storage_path, format = "tar.gz") + def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) return {} if commit.nil? - prefix = archive_prefix(ref, commit.id) + prefix = archive_prefix(ref, commit.id, append_sha: append_sha) { 'RepoPath' => path, @@ -1362,6 +1371,18 @@ module Gitlab raise CommandError.new(e) end + def clean_stale_repository_files + gitaly_migrate(:repository_cleanup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + gitaly_repository_client.cleanup if is_enabled && exists? + end + rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup + Rails.logger.error("Unable to clean repository on storage #{storage} with path #{path}: #{e.message}") + Gitlab::Metrics.counter( + :failed_repository_cleanup_total, + 'Number of failed repository cleanup events' + ).increment + end + def branch_names_contains_sha(sha) gitaly_migrate(:branch_names_contains_sha) do |is_enabled| if is_enabled @@ -1456,6 +1477,43 @@ module Gitlab run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) end + def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) + base_args = %w(worktree add --detach) + + # Note that we _don't_ want to test for `.present?` here: If the caller + # passes an non nil empty value it means it still wants sparse checkout + # but just isn't interested in any file, perhaps because it wants to + # checkout files in by a changeset but that changeset only adds files. + if sparse_checkout_files + # Create worktree without checking out + run_git!(base_args + ['--no-checkout', worktree_path], env: env) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp + + configure_sparse_checkout(worktree_git_path, sparse_checkout_files) + + # After sparse checkout configuration, checkout `branch` in worktree + run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) + else + # Create worktree and checkout `branch` in it + run_git!(base_args + [worktree_path, branch], env: env) + end + + yield + ensure + FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) + FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) + end + + def checksum + gitaly_migrate(:calculate_checksum) do |is_enabled| + if is_enabled + gitaly_repository_client.calculate_checksum + else + calculate_checksum_by_shelling_out + end + end + end + private def local_write_ref(ref_path, ref, old_ref: nil, shell: true) @@ -1542,33 +1600,6 @@ module Gitlab File.exist?(path) && !clean_stuck_worktree(path) end - def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) - base_args = %w(worktree add --detach) - - # Note that we _don't_ want to test for `.present?` here: If the caller - # passes an non nil empty value it means it still wants sparse checkout - # but just isn't interested in any file, perhaps because it wants to - # checkout files in by a changeset but that changeset only adds files. - if sparse_checkout_files - # Create worktree without checking out - run_git!(base_args + ['--no-checkout', worktree_path], env: env) - worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp - - configure_sparse_checkout(worktree_git_path, sparse_checkout_files) - - # After sparse checkout configuration, checkout `branch` in worktree - run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env) - else - # Create worktree and checkout `branch` in it - run_git!(base_args + [worktree_path, branch], env: env) - end - - yield - ensure - FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path) - FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path) - end - def clean_stuck_worktree(path) return false unless File.mtime(path) < 15.minutes.ago @@ -2401,6 +2432,34 @@ module Gitlab def sha_from_ref(ref) rev_parse_target(ref).oid end + + def calculate_checksum_by_shelling_out + raise NoRepository unless exists? + + args = %W(--git-dir=#{path} show-ref --heads --tags) + output, status = run_git(args) + + if status.nil? || !status.zero? + # Empty repositories return with a non-zero status and an empty output. + return EMPTY_REPOSITORY_CHECKSUM if output&.empty? + + raise ChecksumError, output + end + + refs = output.split("\n") + + result = refs.inject(nil) do |checksum, ref| + value = Digest::SHA1.hexdigest(ref).hex + + if checksum.nil? + value + else + checksum ^ value + end + end + + result.to_s(16) + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 6a01957184d..0d1ee73ca1a 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -208,6 +208,7 @@ module Gitlab def check_download_access! passed = deploy_key? || + deploy_token? || user_can_download_code? || build_can_download_code? || guest_can_download_code? @@ -238,6 +239,11 @@ module Gitlab end def check_change_access!(changes) + # If there are worktrees with a HEAD pointing to a non-existent object, + # calls to `git rev-list --all` will fail in git 2.15+. This should also + # clear stale lock files. + project.repository.clean_stale_repository_files + changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied @@ -269,6 +275,14 @@ module Gitlab actor.is_a?(DeployKey) end + def deploy_token + actor if deploy_token? + end + + def deploy_token? + actor.is_a?(DeployToken) + end + def ci? actor == :ci end @@ -276,6 +290,8 @@ module Gitlab def can_read_project? if deploy_key? deploy_key.has_access_to?(project) + elsif deploy_token? + deploy_token.has_access_to?(project) elsif user user.can?(:read_project, project) elsif ci? diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index e1bc2f9ab61..6441065f5fe 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -19,6 +19,11 @@ module Gitlab response.exists end + def cleanup + request = Gitaly::CleanupRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :cleanup, request) + end + def garbage_collect(create_bitmap) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) GitalyClient.call(@storage, :repository_service, :garbage_collect, request) @@ -257,6 +262,12 @@ module Gitlab response.license_short_name.presence end + + def calculate_checksum + request = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo) + response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request) + response.checksum.presence + end end end end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 0d8dd5cb8f4..7a698e4b3f3 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -136,7 +136,7 @@ module Gitlab wiki_file = nil response.each do |message| - next unless message.name.present? + next unless message.name.present? || wiki_file if wiki_file wiki_file.raw_data << message.raw_data diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c490bf059d2..63cab07324a 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,6 +1,9 @@ module Gitlab module ImportExport class Importer + include Gitlab::Allowable + include Gitlab::Utils::StrongMemoize + def self.imports_repository? true end @@ -13,12 +16,14 @@ module Gitlab end def execute - if import_file && check_version! && restorers.all?(&:restore) + if import_file && check_version! && restorers.all?(&:restore) && overwrite_project project_tree.restored_project else raise Projects::ImportService::Error.new(@shared.errors.join(', ')) end - + rescue => e + raise Projects::ImportService::Error.new(e.message) + ensure remove_import_file end @@ -26,7 +31,7 @@ module Gitlab def restorers [repo_restorer, wiki_restorer, project_tree, avatar_restorer, - uploads_restorer, lfs_restorer] + uploads_restorer, lfs_restorer, statistics_restorer] end def import_file @@ -69,6 +74,10 @@ module Gitlab Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared) end + def statistics_restorer + Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared) + end + def path_with_namespace File.join(@project.namespace.full_path, @project.path) end @@ -84,6 +93,33 @@ module Gitlab def remove_import_file FileUtils.rm_rf(@archive_file) end + + def overwrite_project + project = project_tree.restored_project + + return unless can?(@current_user, :admin_namespace, project.namespace) + + if overwrite_project? + ::Projects::OverwriteProjectService.new(project, @current_user) + .execute(project_to_overwrite) + end + + true + end + + def original_path + @project.import_data&.data&.fetch('original_path', nil) + end + + def overwrite_project? + original_path.present? && project_to_overwrite.present? + end + + def project_to_overwrite + strong_memoize(:project_to_overwrite) do + Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}") + end + end end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 2c315207298..d5590dde40f 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -92,7 +92,7 @@ module Gitlab end def override_params - return {} unless params = @project.import_data&.data&.fetch('override_params') + return {} unless params = @project.import_data&.data&.fetch('override_params', nil) @override_params ||= params.select do |key, _value| Project.column_names.include?(key.to_s) && diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb new file mode 100644 index 00000000000..bcdd9c12c85 --- /dev/null +++ b/lib/gitlab/import_export/statistics_restorer.rb @@ -0,0 +1,17 @@ +module Gitlab + module ImportExport + class StatisticsRestorer + def initialize(project:, shared:) + @project = project + @shared = shared + end + + def restore + @project.statistics.refresh! + rescue => e + @shared.error(e) + false + end + end + end +end diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index aad76e335af..f5879de1e94 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -79,7 +79,7 @@ module Gitlab def common_query_context(environment, timeframe_start:, timeframe_end:) base_query_context(timeframe_start, timeframe_end).merge({ ci_environment_slug: environment.slug, - kube_namespace: environment.project.deployment_platform&.actual_namespace || '', + kube_namespace: environment.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} }) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 2faeaf16d55..153cb2a8bb1 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -59,10 +59,10 @@ module Gitlab ] end - def send_git_archive(repository, ref:, format:) + def send_git_archive(repository, ref:, format:, append_sha:) format ||= 'tar.gz' format.downcase! - params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format) + params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha) raise "Repository or ref not found" if params.empty? if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) |