diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-10-01 12:05:59 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-10-01 12:05:59 +0000 |
commit | 9e27f0d920cc3891fa7644c5cc0bc280c519fb20 (patch) | |
tree | 9784dd99270f2009159b19077412bf83d13123a4 /lib/gitlab | |
parent | 1bab0ba591263cd739af2d2c7c3f1b03678a59b6 (diff) | |
download | gitlab-ce-9e27f0d920cc3891fa7644c5cc0bc280c519fb20.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/background_migration/migrate_pages_metadata.rb | 38 | ||||
-rw-r--r-- | lib/gitlab/ci/status/composite.rb | 120 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_tree_restorer.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_factory.rb | 30 |
4 files changed, 208 insertions, 2 deletions
diff --git a/lib/gitlab/background_migration/migrate_pages_metadata.rb b/lib/gitlab/background_migration/migrate_pages_metadata.rb new file mode 100644 index 00000000000..68fd0c17d29 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_pages_metadata.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will insert record into project_pages_metadata + # for each existing project + class MigratePagesMetadata + def perform(start_id, stop_id) + perform_on_relation(Project.where(id: start_id..stop_id)) + end + + def perform_on_relation(relation) + successful_pages_deploy = <<~SQL + SELECT TRUE + FROM ci_builds + WHERE ci_builds.type = 'GenericCommitStatus' + AND ci_builds.status = 'success' + AND ci_builds.stage = 'deploy' + AND ci_builds.name = 'pages:deploy' + AND ci_builds.project_id = projects.id + LIMIT 1 + SQL + + select_from = relation + .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)") + .to_sql + + ActiveRecord::Base.connection_pool.with_connection do |connection| + connection.execute <<~SQL + INSERT INTO project_pages_metadata (project_id, deployed) + #{select_from} + ON CONFLICT (project_id) DO NOTHING + SQL + end + end + end + end +end diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb new file mode 100644 index 00000000000..3c00b67911f --- /dev/null +++ b/lib/gitlab/ci/status/composite.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + class Composite + include Gitlab::Utils::StrongMemoize + + # This class accepts an array of arrays/hashes/or objects + def initialize(all_statuses, with_allow_failure: true) + unless all_statuses.respond_to?(:pluck) + raise ArgumentError, "all_statuses needs to respond to `.pluck`" + end + + @status_set = Set.new + @status_key = 0 + @allow_failure_key = 1 if with_allow_failure + + consume_all_statuses(all_statuses) + end + + # The status calculation is order dependent, + # 1. In some cases we assume that that status is exact + # if the we only have given statues, + # 2. In other cases we assume that status is of that type + # based on what statuses are no longer valid based on the + # data set that we have + def status + return if none? + + strong_memoize(:status) do + if only_of?(:skipped, :ignored) + 'skipped' + elsif only_of?(:success, :skipped, :success_with_warnings, :ignored) + 'success' + elsif only_of?(:created, :success_with_warnings, :ignored) + 'created' + elsif only_of?(:preparing, :success_with_warnings, :ignored) + 'preparing' + elsif only_of?(:canceled, :success, :skipped, :success_with_warnings, :ignored) + 'canceled' + elsif only_of?(:pending, :created, :skipped, :success_with_warnings, :ignored) + 'pending' + elsif any_of?(:running, :pending) + 'running' + elsif any_of?(:manual) + 'manual' + elsif any_of?(:scheduled) + 'scheduled' + elsif any_of?(:preparing) + 'preparing' + elsif any_of?(:created) + 'running' + else + 'failed' + end + end + end + + def warnings? + @status_set.include?(:success_with_warnings) + end + + private + + def none? + @status_set.empty? + end + + def any_of?(*names) + names.any? { |name| @status_set.include?(name) } + end + + def only_of?(*names) + matching = names.count { |name| @status_set.include?(name) } + matching > 0 && + matching == @status_set.size + end + + def consume_all_statuses(all_statuses) + columns = [] + columns[@status_key] = :status + columns[@allow_failure_key] = :allow_failure if @allow_failure_key + + all_statuses + .pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord + .each(&method(:consume_status)) + end + + def consume_status(description) + # convert `"status"` into `["status"]` + description = Array(description) + + status = + if success_with_warnings?(description) + :success_with_warnings + elsif ignored_status?(description) + :ignored + else + description[@status_key].to_sym + end + + @status_set.add(status) + end + + def success_with_warnings?(status) + @allow_failure_key && + status[@allow_failure_key] && + HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key]) + end + + def ignored_status?(status) + @allow_failure_key && + status[@allow_failure_key] && + HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key]) + end + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 39a243bd433..017e536c3e7 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -52,6 +52,11 @@ module Gitlab project: restored_project) end + # A Hash of the imported merge request ID -> imported ID. + def merge_requests_mapping + @merge_requests_mapping ||= {} + end + # Loops through the tree of models defined in import_export.yml and # finds them in the imported JSON so they can be instantiated and saved # in the DB. The structure and relationships between models are guessed from @@ -80,10 +85,26 @@ module Gitlab @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) + save_id_mappings(relation_key, relation_hash_batch, relation_hash) + # Restore the project again, extra query that skips holding the AR objects in memory @restored_project = Project.find(@project_id) end + # Older, serialized CI pipeline exports may only have a + # merge_request_id and not the full hash of the merge request. To + # import these pipelines, we need to preserve the mapping between + # the old and new the merge request ID. + def save_id_mappings(relation_key, relation_hash_batch, relation_hash) + return unless relation_key == 'merge_requests' + + relation_hash = Array(relation_hash) + + Array(relation_hash_batch).each_with_index do |raw_data, index| + merge_requests_mapping[raw_data['id']] = relation_hash[index]['id'] + end + end + # Remove project models that became group models as we found them at group level. # This no longer required saving them at the root project level. # For example, in the case of an existing group label that matched the title. @@ -222,6 +243,7 @@ module Gitlab relation_sym: relation_key.to_sym, relation_hash: relation_hash, members_mapper: members_mapper, + merge_requests_mapping: merge_requests_mapping, user: @user, project: @restored_project, excluded_keys: excluded_keys_for_relation(relation_key)) diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 37f625288a9..9ec244b0960 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -55,10 +55,11 @@ module Gitlab relation_name.to_s.constantize end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: []) + def initialize(relation_sym:, relation_hash:, members_mapper:, merge_requests_mapping:, user:, project:, excluded_keys: []) @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper + @merge_requests_mapping = merge_requests_mapping @user = user @project = project @imported_object_retries = 0 @@ -109,7 +110,10 @@ module Gitlab update_group_references remove_duplicate_assignees - setup_pipeline if @relation_name == :'Ci::Pipeline' + if @relation_name == :'Ci::Pipeline' + update_merge_request_references + setup_pipeline + end reset_tokens! remove_encrypted_attributes! @@ -194,6 +198,28 @@ module Gitlab @relation_hash['group_id'] = @project.namespace_id end + # This code is a workaround for broken project exports that don't + # export merge requests with CI pipelines (i.e. exports that were + # generated from + # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844). + # This method can be removed in GitLab 12.6. + def update_merge_request_references + # If a merge request was properly created, we don't need to fix + # up this export. + return if @relation_hash['merge_request'] + + merge_request_id = @relation_hash['merge_request_id'] + + return unless merge_request_id + + new_merge_request_id = @merge_requests_mapping[merge_request_id] + + return unless new_merge_request_id + + @relation_hash['merge_request_id'] = new_merge_request_id + parsed_relation_hash['merge_request_id'] = new_merge_request_id + end + def reset_tokens! return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name) |