diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-10-01 12:05:59 +0000 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-10-01 12:05:59 +0000 |
commit | 9e27f0d920cc3891fa7644c5cc0bc280c519fb20 (patch) | |
tree | 9784dd99270f2009159b19077412bf83d13123a4 /lib | |
parent | 1bab0ba591263cd739af2d2c7c3f1b03678a59b6 (diff) | |
download | gitlab-ce-9e27f0d920cc3891fa7644c5cc0bc280c519fb20.tar.gz |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/helpers/graphql_helpers.rb | 2 | ||||
-rw-r--r-- | lib/api/version.rb | 5 | ||||
-rw-r--r-- | lib/banzai/filter/video_link_filter.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/background_migration/migrate_pages_metadata.rb | 38 | ||||
-rw-r--r-- | lib/gitlab/ci/status/composite.rb | 120 | ||||
-rw-r--r-- | lib/gitlab/import_export/project_tree_restorer.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_factory.rb | 30 |
7 files changed, 228 insertions, 11 deletions
diff --git a/lib/api/helpers/graphql_helpers.rb b/lib/api/helpers/graphql_helpers.rb index bd60470fbd6..3ddef0c16b3 100644 --- a/lib/api/helpers/graphql_helpers.rb +++ b/lib/api/helpers/graphql_helpers.rb @@ -6,7 +6,7 @@ module API # against the graphql API. Helper code for the graphql server implementation # should be in app/graphql/ or lib/gitlab/graphql/ module GraphqlHelpers - def conditionally_graphql!(fallback:, query:, context: {}, transform: nil) + def run_graphql!(query:, context: {}, transform: nil) result = GitlabSchema.execute(query, context: context) if transform diff --git a/lib/api/version.rb b/lib/api/version.rb index eca1b529094..f79bb3428f2 100644 --- a/lib/api/version.rb +++ b/lib/api/version.rb @@ -19,11 +19,10 @@ module API detail 'This feature was introduced in GitLab 8.13.' end get '/version' do - conditionally_graphql!( + run_graphql!( query: METADATA_QUERY, context: { current_user: current_user }, - transform: ->(result) { result.dig('data', 'metadata') }, - fallback: -> { { version: Gitlab::VERSION, revision: Gitlab.revision } } + transform: ->(result) { result.dig('data', 'metadata') } ) end end diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index b3d5d2c95d7..a35b0d7a0b5 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -8,8 +8,8 @@ module Banzai # a "Download" link in the case the video cannot be played. class VideoLinkFilter < HTML::Pipeline::Filter def call - doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |el| - el.replace(video_node(doc, el)) if has_video_extension?(el) + doc.xpath(query).each do |el| + el.replace(video_node(doc, el)) end doc @@ -17,10 +17,22 @@ module Banzai private - def has_video_extension?(element) - src_attr = context[:asset_proxy_enabled] ? 'data-canonical-src' : 'src' + def query + @query ||= begin + src_query = UploaderHelper::SAFE_VIDEO_EXT.map do |ext| + "'.#{ext}' = substring(@src, string-length(@src) - #{ext.size})" + end - element.attr(src_attr).downcase.end_with?(*UploaderHelper::SAFE_VIDEO_EXT) + if context[:asset_proxy_enabled].present? + src_query.concat( + UploaderHelper::SAFE_VIDEO_EXT.map do |ext| + "'.#{ext}' = substring(@data-canonical-src, string-length(@data-canonical-src) - #{ext.size})" + end + ) + end + + "descendant-or-self::img[not(ancestor::a) and (#{src_query.join(' or ')})]" + end end def video_node(doc, element) diff --git a/lib/gitlab/background_migration/migrate_pages_metadata.rb b/lib/gitlab/background_migration/migrate_pages_metadata.rb new file mode 100644 index 00000000000..68fd0c17d29 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_pages_metadata.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will insert record into project_pages_metadata + # for each existing project + class MigratePagesMetadata + def perform(start_id, stop_id) + perform_on_relation(Project.where(id: start_id..stop_id)) + end + + def perform_on_relation(relation) + successful_pages_deploy = <<~SQL + SELECT TRUE + FROM ci_builds + WHERE ci_builds.type = 'GenericCommitStatus' + AND ci_builds.status = 'success' + AND ci_builds.stage = 'deploy' + AND ci_builds.name = 'pages:deploy' + AND ci_builds.project_id = projects.id + LIMIT 1 + SQL + + select_from = relation + .select("projects.id", "COALESCE((#{successful_pages_deploy}), FALSE)") + .to_sql + + ActiveRecord::Base.connection_pool.with_connection do |connection| + connection.execute <<~SQL + INSERT INTO project_pages_metadata (project_id, deployed) + #{select_from} + ON CONFLICT (project_id) DO NOTHING + SQL + end + end + end + end +end diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb new file mode 100644 index 00000000000..3c00b67911f --- /dev/null +++ b/lib/gitlab/ci/status/composite.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + class Composite + include Gitlab::Utils::StrongMemoize + + # This class accepts an array of arrays/hashes/or objects + def initialize(all_statuses, with_allow_failure: true) + unless all_statuses.respond_to?(:pluck) + raise ArgumentError, "all_statuses needs to respond to `.pluck`" + end + + @status_set = Set.new + @status_key = 0 + @allow_failure_key = 1 if with_allow_failure + + consume_all_statuses(all_statuses) + end + + # The status calculation is order dependent, + # 1. In some cases we assume that that status is exact + # if the we only have given statues, + # 2. In other cases we assume that status is of that type + # based on what statuses are no longer valid based on the + # data set that we have + def status + return if none? + + strong_memoize(:status) do + if only_of?(:skipped, :ignored) + 'skipped' + elsif only_of?(:success, :skipped, :success_with_warnings, :ignored) + 'success' + elsif only_of?(:created, :success_with_warnings, :ignored) + 'created' + elsif only_of?(:preparing, :success_with_warnings, :ignored) + 'preparing' + elsif only_of?(:canceled, :success, :skipped, :success_with_warnings, :ignored) + 'canceled' + elsif only_of?(:pending, :created, :skipped, :success_with_warnings, :ignored) + 'pending' + elsif any_of?(:running, :pending) + 'running' + elsif any_of?(:manual) + 'manual' + elsif any_of?(:scheduled) + 'scheduled' + elsif any_of?(:preparing) + 'preparing' + elsif any_of?(:created) + 'running' + else + 'failed' + end + end + end + + def warnings? + @status_set.include?(:success_with_warnings) + end + + private + + def none? + @status_set.empty? + end + + def any_of?(*names) + names.any? { |name| @status_set.include?(name) } + end + + def only_of?(*names) + matching = names.count { |name| @status_set.include?(name) } + matching > 0 && + matching == @status_set.size + end + + def consume_all_statuses(all_statuses) + columns = [] + columns[@status_key] = :status + columns[@allow_failure_key] = :allow_failure if @allow_failure_key + + all_statuses + .pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord + .each(&method(:consume_status)) + end + + def consume_status(description) + # convert `"status"` into `["status"]` + description = Array(description) + + status = + if success_with_warnings?(description) + :success_with_warnings + elsif ignored_status?(description) + :ignored + else + description[@status_key].to_sym + end + + @status_set.add(status) + end + + def success_with_warnings?(status) + @allow_failure_key && + status[@allow_failure_key] && + HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key]) + end + + def ignored_status?(status) + @allow_failure_key && + status[@allow_failure_key] && + HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key]) + end + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 39a243bd433..017e536c3e7 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -52,6 +52,11 @@ module Gitlab project: restored_project) end + # A Hash of the imported merge request ID -> imported ID. + def merge_requests_mapping + @merge_requests_mapping ||= {} + end + # Loops through the tree of models defined in import_export.yml and # finds them in the imported JSON so they can be instantiated and saved # in the DB. The structure and relationships between models are guessed from @@ -80,10 +85,26 @@ module Gitlab @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) + save_id_mappings(relation_key, relation_hash_batch, relation_hash) + # Restore the project again, extra query that skips holding the AR objects in memory @restored_project = Project.find(@project_id) end + # Older, serialized CI pipeline exports may only have a + # merge_request_id and not the full hash of the merge request. To + # import these pipelines, we need to preserve the mapping between + # the old and new the merge request ID. + def save_id_mappings(relation_key, relation_hash_batch, relation_hash) + return unless relation_key == 'merge_requests' + + relation_hash = Array(relation_hash) + + Array(relation_hash_batch).each_with_index do |raw_data, index| + merge_requests_mapping[raw_data['id']] = relation_hash[index]['id'] + end + end + # Remove project models that became group models as we found them at group level. # This no longer required saving them at the root project level. # For example, in the case of an existing group label that matched the title. @@ -222,6 +243,7 @@ module Gitlab relation_sym: relation_key.to_sym, relation_hash: relation_hash, members_mapper: members_mapper, + merge_requests_mapping: merge_requests_mapping, user: @user, project: @restored_project, excluded_keys: excluded_keys_for_relation(relation_key)) diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 37f625288a9..9ec244b0960 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -55,10 +55,11 @@ module Gitlab relation_name.to_s.constantize end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project:, excluded_keys: []) + def initialize(relation_sym:, relation_hash:, members_mapper:, merge_requests_mapping:, user:, project:, excluded_keys: []) @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper + @merge_requests_mapping = merge_requests_mapping @user = user @project = project @imported_object_retries = 0 @@ -109,7 +110,10 @@ module Gitlab update_group_references remove_duplicate_assignees - setup_pipeline if @relation_name == :'Ci::Pipeline' + if @relation_name == :'Ci::Pipeline' + update_merge_request_references + setup_pipeline + end reset_tokens! remove_encrypted_attributes! @@ -194,6 +198,28 @@ module Gitlab @relation_hash['group_id'] = @project.namespace_id end + # This code is a workaround for broken project exports that don't + # export merge requests with CI pipelines (i.e. exports that were + # generated from + # https://gitlab.com/gitlab-org/gitlab/merge_requests/17844). + # This method can be removed in GitLab 12.6. + def update_merge_request_references + # If a merge request was properly created, we don't need to fix + # up this export. + return if @relation_hash['merge_request'] + + merge_request_id = @relation_hash['merge_request_id'] + + return unless merge_request_id + + new_merge_request_id = @merge_requests_mapping[merge_request_id] + + return unless new_merge_request_id + + @relation_hash['merge_request_id'] = new_merge_request_id + parsed_relation_hash['merge_request_id'] = new_merge_request_id + end + def reset_tokens! return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name) |