diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-02 21:08:01 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-02 21:08:01 +0000 |
commit | 561e1b470f0a99fe6304c8f197348c47a637d594 (patch) | |
tree | 6b361b6b0b412b70450aca167079c50a13bd88d8 /app/services/projects | |
parent | 7b52c7cb634ef7047d30b0337fe477bcdcedf41d (diff) | |
download | gitlab-ce-561e1b470f0a99fe6304c8f197348c47a637d594.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'app/services/projects')
4 files changed, 146 insertions, 17 deletions
diff --git a/app/services/projects/container_repository/cleanup_tags_service.rb b/app/services/projects/container_repository/cleanup_tags_service.rb index 046745d725e..6eb8f5c27d9 100644 --- a/app/services/projects/container_repository/cleanup_tags_service.rb +++ b/app/services/projects/container_repository/cleanup_tags_service.rb @@ -61,10 +61,15 @@ module Projects end def filter_by_name(tags) - regex = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex']}\\z") + # Technical Debt: https://gitlab.com/gitlab-org/gitlab/issues/207267 + # name_regex to be removed when container_expiration_policies is updated + # to have both regex columns + regex_delete = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_delete'] || params['name_regex']}\\z") + regex_retain = Gitlab::UntrustedRegexp.new("\\A#{params['name_regex_keep']}\\z") tags.select do |tag| - regex.scan(tag.name).any? + # regex_retain will override any overlapping matches by regex_delete + regex_delete.match?(tag.name) && !regex_retain.match?(tag.name) end end diff --git a/app/services/projects/lsif_data_service.rb b/app/services/projects/lsif_data_service.rb index 971885b680e..142a5a910d4 100644 --- a/app/services/projects/lsif_data_service.rb +++ b/app/services/projects/lsif_data_service.rb @@ -2,20 +2,22 @@ module Projects class LsifDataService - attr_reader :file, :project, :path, :commit_id, - :docs, :doc_ranges, :ranges, :def_refs, :hover_refs + attr_reader :file, :project, :commit_id, :docs, + :doc_ranges, :ranges, :def_refs, :hover_refs CACHE_EXPIRE_IN = 1.hour - def initialize(file, project, params) + def initialize(file, project, commit_id) @file = file @project = project - @path = params[:path] - @commit_id = params[:commit_id] - end + @commit_id = commit_id - def execute fetch_data! + end + + def execute(path) + doc_id = find_doc_id(docs, path) + dir_absolute_path = docs[doc_id]&.delete_suffix(path) doc_ranges[doc_id]&.map do |range_id| location, ref_id = ranges[range_id].values_at('loc', 'ref_id') @@ -26,7 +28,7 @@ module Projects end_line: line_data.last, start_char: column_data.first, end_char: column_data.last, - definition_url: definition_url_for(def_refs[ref_id]), + definition_url: definition_url_for(def_refs[ref_id], dir_absolute_path), hover: highlighted_hover(hover_refs[ref_id]) } end @@ -58,8 +60,8 @@ module Projects @hover_refs = data['hover_refs'] end - def doc_id - @doc_id ||= docs.reduce(nil) do |doc_id, (id, doc_path)| + def find_doc_id(docs, path) + docs.reduce(nil) do |doc_id, (id, doc_path)| next doc_id unless doc_path =~ /#{path}$/ if doc_id.nil? || docs[doc_id].size > doc_path.size @@ -70,11 +72,7 @@ module Projects end end - def dir_absolute_path - @dir_absolute_path ||= docs[doc_id]&.delete_suffix(path) - end - - def definition_url_for(ref_id) + def definition_url_for(ref_id, dir_absolute_path) return unless range = ranges[ref_id] def_doc_id, location = range.values_at('doc_id', 'loc') diff --git a/app/services/projects/update_repository_storage_service.rb b/app/services/projects/update_repository_storage_service.rb new file mode 100644 index 00000000000..30b99e85304 --- /dev/null +++ b/app/services/projects/update_repository_storage_service.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +module Projects + class UpdateRepositoryStorageService < BaseService + include Gitlab::ShellAdapter + + RepositoryAlreadyMoved = Class.new(StandardError) + + def initialize(project) + @project = project + end + + def execute(new_repository_storage_key) + # Raising an exception is a little heavy handed but this behavior (doing + # nothing if the repo is already on the right storage) prevents data + # loss, so it is valuable for us to be able to observe it via the + # exception. + raise RepositoryAlreadyMoved if project.repository_storage == new_repository_storage_key + + if mirror_repositories(new_repository_storage_key) + mark_old_paths_for_archive + + project.update(repository_storage: new_repository_storage_key, repository_read_only: false) + project.leave_pool_repository + project.track_project_repository + + enqueue_housekeeping + else + project.update(repository_read_only: false) + end + end + + private + + def mirror_repositories(new_repository_storage_key) + result = mirror_repository(new_repository_storage_key) + + if project.wiki.repository_exists? + result &&= mirror_repository(new_repository_storage_key, type: Gitlab::GlRepository::WIKI) + end + + result + end + + def mirror_repository(new_storage_key, type: Gitlab::GlRepository::PROJECT) + return false unless wait_for_pushes(type) + + repository = type.repository_for(project) + full_path = repository.full_path + raw_repository = repository.raw + + # Initialize a git repository on the target path + gitlab_shell.create_repository(new_storage_key, raw_repository.relative_path, full_path) + new_repository = Gitlab::Git::Repository.new(new_storage_key, + raw_repository.relative_path, + raw_repository.gl_repository, + full_path) + + new_repository.fetch_repository_as_mirror(raw_repository) + end + + def mark_old_paths_for_archive + old_repository_storage = project.repository_storage + new_project_path = moved_path(project.disk_path) + + # Notice that the block passed to `run_after_commit` will run with `project` + # as its context + project.run_after_commit do + GitlabShellWorker.perform_async(:mv_repository, + old_repository_storage, + disk_path, + new_project_path) + + if wiki.repository_exists? + GitlabShellWorker.perform_async(:mv_repository, + old_repository_storage, + wiki.disk_path, + "#{new_project_path}.wiki") + end + end + end + + def moved_path(path) + "#{path}+#{project.id}+moved+#{Time.now.to_i}" + end + + # The underlying FetchInternalRemote call uses a `git fetch` to move data + # to the new repository, which leaves it in a less-well-packed state, + # lacking bitmaps and commit graphs. Housekeeping will boost performance + # significantly. + def enqueue_housekeeping + return unless Gitlab::CurrentSettings.housekeeping_enabled? + return unless Feature.enabled?(:repack_after_shard_migration, project) + + Projects::HousekeepingService.new(project, :gc).execute + rescue Projects::HousekeepingService::LeaseTaken + # No action required + end + + def wait_for_pushes(type) + reference_counter = project.reference_counter(type: type) + + # Try for 30 seconds, polling every 10 + 3.times do + return true if reference_counter.value == 0 + + sleep 10 + end + + false + end + end +end + +Projects::UpdateRepositoryStorageService.prepend_if_ee('EE::Projects::UpdateRepositoryStorageService') diff --git a/app/services/projects/update_service.rb b/app/services/projects/update_service.rb index aedd7252f63..e10dede632a 100644 --- a/app/services/projects/update_service.rb +++ b/app/services/projects/update_service.rb @@ -13,6 +13,10 @@ module Projects ensure_wiki_exists if enabling_wiki? + if changing_storage_size? + project.change_repository_storage(params.delete(:repository_storage)) + end + yield if block_given? validate_classification_label(project, :external_authorization_classification_label) @@ -140,6 +144,13 @@ module Projects def changing_pages_https_only? project.previous_changes.include?(:pages_https_only) end + + def changing_storage_size? + new_repository_storage = params[:repository_storage] + + new_repository_storage && project.repository.exists? && + can?(current_user, :change_repository_storage, project) + end end end |