summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/api/helpers.rb4
-rw-r--r--lib/api/project_import.rb4
-rw-r--r--lib/api/repositories.rb2
-rw-r--r--lib/api/v3/repositories.rb2
-rw-r--r--lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb2
-rw-r--r--lib/gitlab/database/sha_attribute.rb51
-rw-r--r--lib/gitlab/git/repository.rb81
-rw-r--r--lib/gitlab/git_access.rb5
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb5
-rw-r--r--lib/gitlab/import_export/importer.rb42
-rw-r--r--lib/gitlab/import_export/project_tree_restorer.rb2
-rw-r--r--lib/gitlab/import_export/statistics_restorer.rb17
-rw-r--r--lib/gitlab/prometheus/queries/query_additional_metrics.rb2
-rw-r--r--lib/gitlab/workhorse.rb4
14 files changed, 168 insertions, 55 deletions
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index a582aa0ec2c..61dab1dd5cb 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -468,8 +468,8 @@ module API
header(*Gitlab::Workhorse.send_git_blob(repository, blob))
end
- def send_git_archive(repository, ref:, format:)
- header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format))
+ def send_git_archive(repository, **kwargs)
+ header(*Gitlab::Workhorse.send_git_archive(repository, **kwargs))
end
def send_artifacts_entry(build, entry)
diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb
index 303b58a5942..bc5152e539f 100644
--- a/lib/api/project_import.rb
+++ b/lib/api/project_import.rb
@@ -26,6 +26,7 @@ module API
requires :path, type: String, desc: 'The new project path and name'
requires :file, type: File, desc: 'The project export file to be imported'
optional :namespace, type: String, desc: "The ID or name of the namespace that the project will be imported into. Defaults to the current user's namespace."
+ optional :overwrite, type: Boolean, default: false, desc: 'If there is a project in the same namespace and with the same name overwrite it'
optional :override_params,
type: Hash,
desc: 'New project params to override values in the export' do
@@ -50,7 +51,8 @@ module API
project_params = {
path: import_params[:path],
namespace_id: namespace.id,
- file: import_params[:file]['tempfile']
+ file: import_params[:file]['tempfile'],
+ overwrite: import_params[:overwrite]
}
override_params = import_params.delete(:override_params)
diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb
index 9638c53a1df..2396dc73f0e 100644
--- a/lib/api/repositories.rb
+++ b/lib/api/repositories.rb
@@ -88,7 +88,7 @@ module API
end
get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do
begin
- send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
+ send_git_archive user_project.repository, ref: params[:sha], format: params[:format], append_sha: true
rescue
not_found!('File')
end
diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb
index 5b54734bb45..f701d64e886 100644
--- a/lib/api/v3/repositories.rb
+++ b/lib/api/v3/repositories.rb
@@ -75,7 +75,7 @@ module API
end
get ':id/repository/archive', requirements: { format: Gitlab::PathRegex.archive_formats_regex } do
begin
- send_git_archive user_project.repository, ref: params[:sha], format: params[:format]
+ send_git_archive user_project.repository, ref: params[:sha], format: params[:format], append_sha: true
rescue
not_found!('File')
end
diff --git a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
index fd5cbf76e47..a357538a885 100644
--- a/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
+++ b/lib/gitlab/background_migration/deserialize_merge_request_diffs_and_commits.rb
@@ -96,7 +96,7 @@ module Gitlab
commit_hash.merge(
merge_request_diff_id: merge_request_diff.id,
relative_order: index,
- sha: sha_attribute.type_cast_for_database(sha)
+ sha: sha_attribute.serialize(sha)
)
end
diff --git a/lib/gitlab/database/sha_attribute.rb b/lib/gitlab/database/sha_attribute.rb
index d9400e04b83..b2d8ee81977 100644
--- a/lib/gitlab/database/sha_attribute.rb
+++ b/lib/gitlab/database/sha_attribute.rb
@@ -1,12 +1,20 @@
module Gitlab
module Database
- BINARY_TYPE = if Gitlab::Database.postgresql?
- # PostgreSQL defines its own class with slightly different
- # behaviour from the default Binary type.
- ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
- else
- ActiveRecord::Type::Binary
- end
+ BINARY_TYPE =
+ if Gitlab::Database.postgresql?
+ # PostgreSQL defines its own class with slightly different
+ # behaviour from the default Binary type.
+ ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Bytea
+ else
+ # In Rails 5.0 `Type` has been moved from `ActiveRecord` to `ActiveModel`
+ # https://github.com/rails/rails/commit/9cc8c6f3730df3d94c81a55be9ee1b7b4ffd29f6#diff-f8ba7983a51d687976e115adcd95822b
+ # Remove this method and leave just `ActiveModel::Type::Binary` when removing Gitlab.rails5? code.
+ if Gitlab.rails5?
+ ActiveModel::Type::Binary
+ else
+ ActiveRecord::Type::Binary
+ end
+ end
# Class for casting binary data to hexadecimal SHA1 hashes (and vice-versa).
#
@@ -16,18 +24,39 @@ module Gitlab
class ShaAttribute < BINARY_TYPE
PACK_FORMAT = 'H*'.freeze
- # Casts binary data to a SHA1 in hexadecimal.
+ # It is called from activerecord-4.2.10/lib/active_record internal methods.
+ # Remove this method when removing Gitlab.rails5? code.
def type_cast_from_database(value)
- value = super
+ unpack_sha(super)
+ end
+
+ # It is called from activerecord-4.2.10/lib/active_record internal methods.
+ # Remove this method when removing Gitlab.rails5? code.
+ def type_cast_for_database(value)
+ serialize(value)
+ end
+ # It is called from activerecord-5.0.6/lib/active_record/attribute.rb
+ # Remove this method when removing Gitlab.rails5? code..
+ def deserialize(value)
+ value = Gitlab.rails5? ? super : method(:type_cast_from_database).super_method.call(value)
+
+ unpack_sha(value)
+ end
+
+ # Rename this method to `deserialize(value)` removing Gitlab.rails5? code.
+ # Casts binary data to a SHA1 in hexadecimal.
+ def unpack_sha(value)
+ # Uncomment this line when removing Gitlab.rails5? code.
+ # value = super
value ? value.unpack(PACK_FORMAT)[0] : nil
end
# Casts a SHA1 in hexadecimal to the proper binary format.
- def type_cast_for_database(value)
+ def serialize(value)
arg = value ? [value].pack(PACK_FORMAT) : nil
- super(arg)
+ Gitlab.rails5? ? super(arg) : method(:type_cast_for_database).super_method.call(arg)
end
end
end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 8d97bfb0e6a..5678c28cf3a 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -394,17 +394,24 @@ module Gitlab
nil
end
- def archive_prefix(ref, sha)
+ def archive_prefix(ref, sha, append_sha:)
+ append_sha = (ref != sha) if append_sha.nil?
+
project_name = self.name.chomp('.git')
- "#{project_name}-#{ref.tr('/', '-')}-#{sha}"
+ formatted_ref = ref.tr('/', '-')
+
+ prefix_segments = [project_name, formatted_ref]
+ prefix_segments << sha if append_sha
+
+ prefix_segments.join('-')
end
- def archive_metadata(ref, storage_path, format = "tar.gz")
+ def archive_metadata(ref, storage_path, format = "tar.gz", append_sha:)
ref ||= root_ref
commit = Gitlab::Git::Commit.find(self, ref)
return {} if commit.nil?
- prefix = archive_prefix(ref, commit.id)
+ prefix = archive_prefix(ref, commit.id, append_sha: append_sha)
{
'RepoPath' => path,
@@ -1362,6 +1369,18 @@ module Gitlab
raise CommandError.new(e)
end
+ def clean_stale_repository_files
+ gitaly_migrate(:repository_cleanup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled|
+ gitaly_repository_client.cleanup if is_enabled && exists?
+ end
+ rescue Gitlab::Git::CommandError => e # Don't fail if we can't cleanup
+ Rails.logger.error("Unable to clean repository on storage #{storage} with path #{path}: #{e.message}")
+ Gitlab::Metrics.counter(
+ :failed_repository_cleanup_total,
+ 'Number of failed repository cleanup events'
+ ).increment
+ end
+
def branch_names_contains_sha(sha)
gitaly_migrate(:branch_names_contains_sha) do |is_enabled|
if is_enabled
@@ -1456,6 +1475,33 @@ module Gitlab
run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"])
end
+ def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:)
+ base_args = %w(worktree add --detach)
+
+ # Note that we _don't_ want to test for `.present?` here: If the caller
+ # passes an non nil empty value it means it still wants sparse checkout
+ # but just isn't interested in any file, perhaps because it wants to
+ # checkout files in by a changeset but that changeset only adds files.
+ if sparse_checkout_files
+ # Create worktree without checking out
+ run_git!(base_args + ['--no-checkout', worktree_path], env: env)
+ worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp
+
+ configure_sparse_checkout(worktree_git_path, sparse_checkout_files)
+
+ # After sparse checkout configuration, checkout `branch` in worktree
+ run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env)
+ else
+ # Create worktree and checkout `branch` in it
+ run_git!(base_args + [worktree_path, branch], env: env)
+ end
+
+ yield
+ ensure
+ FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path)
+ FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path)
+ end
+
private
def local_write_ref(ref_path, ref, old_ref: nil, shell: true)
@@ -1542,33 +1588,6 @@ module Gitlab
File.exist?(path) && !clean_stuck_worktree(path)
end
- def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:)
- base_args = %w(worktree add --detach)
-
- # Note that we _don't_ want to test for `.present?` here: If the caller
- # passes an non nil empty value it means it still wants sparse checkout
- # but just isn't interested in any file, perhaps because it wants to
- # checkout files in by a changeset but that changeset only adds files.
- if sparse_checkout_files
- # Create worktree without checking out
- run_git!(base_args + ['--no-checkout', worktree_path], env: env)
- worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp
-
- configure_sparse_checkout(worktree_git_path, sparse_checkout_files)
-
- # After sparse checkout configuration, checkout `branch` in worktree
- run_git!(%W(checkout --detach #{branch}), chdir: worktree_path, env: env)
- else
- # Create worktree and checkout `branch` in it
- run_git!(base_args + [worktree_path, branch], env: env)
- end
-
- yield
- ensure
- FileUtils.rm_rf(worktree_path) if File.exist?(worktree_path)
- FileUtils.rm_rf(worktree_git_path) if worktree_git_path && File.exist?(worktree_git_path)
- end
-
def clean_stuck_worktree(path)
return false unless File.mtime(path) < 15.minutes.ago
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 6a01957184d..01f8b22b2b6 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -238,6 +238,11 @@ module Gitlab
end
def check_change_access!(changes)
+ # If there are worktrees with a HEAD pointing to a non-existent object,
+ # calls to `git rev-list --all` will fail in git 2.15+. This should also
+ # clear stale lock files.
+ project.repository.clean_stale_repository_files
+
changes_list = Gitlab::ChangesList.new(changes)
# Iterate over all changes to find if user allowed all of them to be applied
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index e1bc2f9ab61..b5a734aaef6 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -19,6 +19,11 @@ module Gitlab
response.exists
end
+ def cleanup
+ request = Gitaly::CleanupRequest.new(repository: @gitaly_repo)
+ GitalyClient.call(@storage, :repository_service, :cleanup, request)
+ end
+
def garbage_collect(create_bitmap)
request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap)
GitalyClient.call(@storage, :repository_service, :garbage_collect, request)
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index c490bf059d2..63cab07324a 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -1,6 +1,9 @@
module Gitlab
module ImportExport
class Importer
+ include Gitlab::Allowable
+ include Gitlab::Utils::StrongMemoize
+
def self.imports_repository?
true
end
@@ -13,12 +16,14 @@ module Gitlab
end
def execute
- if import_file && check_version! && restorers.all?(&:restore)
+ if import_file && check_version! && restorers.all?(&:restore) && overwrite_project
project_tree.restored_project
else
raise Projects::ImportService::Error.new(@shared.errors.join(', '))
end
-
+ rescue => e
+ raise Projects::ImportService::Error.new(e.message)
+ ensure
remove_import_file
end
@@ -26,7 +31,7 @@ module Gitlab
def restorers
[repo_restorer, wiki_restorer, project_tree, avatar_restorer,
- uploads_restorer, lfs_restorer]
+ uploads_restorer, lfs_restorer, statistics_restorer]
end
def import_file
@@ -69,6 +74,10 @@ module Gitlab
Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
end
+ def statistics_restorer
+ Gitlab::ImportExport::StatisticsRestorer.new(project: project_tree.restored_project, shared: @shared)
+ end
+
def path_with_namespace
File.join(@project.namespace.full_path, @project.path)
end
@@ -84,6 +93,33 @@ module Gitlab
def remove_import_file
FileUtils.rm_rf(@archive_file)
end
+
+ def overwrite_project
+ project = project_tree.restored_project
+
+ return unless can?(@current_user, :admin_namespace, project.namespace)
+
+ if overwrite_project?
+ ::Projects::OverwriteProjectService.new(project, @current_user)
+ .execute(project_to_overwrite)
+ end
+
+ true
+ end
+
+ def original_path
+ @project.import_data&.data&.fetch('original_path', nil)
+ end
+
+ def overwrite_project?
+ original_path.present? && project_to_overwrite.present?
+ end
+
+ def project_to_overwrite
+ strong_memoize(:project_to_overwrite) do
+ Project.find_by_full_path("#{@project.namespace.full_path}/#{original_path}")
+ end
+ end
end
end
end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 2c315207298..d5590dde40f 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -92,7 +92,7 @@ module Gitlab
end
def override_params
- return {} unless params = @project.import_data&.data&.fetch('override_params')
+ return {} unless params = @project.import_data&.data&.fetch('override_params', nil)
@override_params ||= params.select do |key, _value|
Project.column_names.include?(key.to_s) &&
diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb
new file mode 100644
index 00000000000..bcdd9c12c85
--- /dev/null
+++ b/lib/gitlab/import_export/statistics_restorer.rb
@@ -0,0 +1,17 @@
+module Gitlab
+ module ImportExport
+ class StatisticsRestorer
+ def initialize(project:, shared:)
+ @project = project
+ @shared = shared
+ end
+
+ def restore
+ @project.statistics.refresh!
+ rescue => e
+ @shared.error(e)
+ false
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
index aad76e335af..f5879de1e94 100644
--- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb
+++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb
@@ -79,7 +79,7 @@ module Gitlab
def common_query_context(environment, timeframe_start:, timeframe_end:)
base_query_context(timeframe_start, timeframe_end).merge({
ci_environment_slug: environment.slug,
- kube_namespace: environment.project.deployment_platform&.actual_namespace || '',
+ kube_namespace: environment.deployment_platform&.actual_namespace || '',
environment_filter: %{container_name!="POD",environment="#{environment.slug}"}
})
end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 2faeaf16d55..153cb2a8bb1 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -59,10 +59,10 @@ module Gitlab
]
end
- def send_git_archive(repository, ref:, format:)
+ def send_git_archive(repository, ref:, format:, append_sha:)
format ||= 'tar.gz'
format.downcase!
- params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format)
+ params = repository.archive_metadata(ref, Gitlab.config.gitlab.repository_downloads_path, format, append_sha: append_sha)
raise "Repository or ref not found" if params.empty?
if Gitlab::GitalyClient.feature_enabled?(:workhorse_archive, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT)
View Lorry Controller Status