diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml | 1 | ||||
-rw-r--r-- | lib/gitlab/import_export/group/tree_restorer.rb | 25 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/legacy_reader.rb | 104 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_loader.rb | 74 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_restorer.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/import_export/reader.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_tree_restorer.rb | 34 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_tree_saver.rb | 2 |
8 files changed, 143 insertions, 127 deletions
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index bbfcf53b3d4..c6c8256b4bb 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -40,6 +40,7 @@ stop_review: environment: name: review/$CI_COMMIT_REF_NAME action: stop + dependencies: [] when: manual allow_failure: true only: diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index cbaa6929efa..247e39a68b9 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -17,9 +17,17 @@ module Gitlab end def restore - @tree_hash = @group_hash || read_tree_hash - @group_members = @tree_hash.delete('members') - @children = @tree_hash.delete('children') + @relation_reader ||= + if @group_hash.present? + ImportExport::JSON::LegacyReader::User.new(@group_hash, reader.group_relation_names) + else + ImportExport::JSON::LegacyReader::File.new(@path, reader.group_relation_names) + end + + @group_members = @relation_reader.consume_relation('members') + @children = @relation_reader.consume_attribute('children') + @relation_reader.consume_attribute('name') + @relation_reader.consume_attribute('path') if members_mapper.map && restorer.restore @children&.each do |group_hash| @@ -45,21 +53,12 @@ module Gitlab private - def read_tree_hash - json = IO.read(@path) - ActiveSupport::JSON.decode(json) - rescue => e - @shared.error(e) - - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - def restorer @relation_tree_restorer ||= RelationTreeRestorer.new( user: @user, shared: @shared, importable: @group, - tree_hash: @tree_hash.except('name', 'path'), + relation_reader: @relation_reader, members_mapper: members_mapper, object_builder: object_builder, relation_factory: relation_factory, diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb new file mode 100644 index 00000000000..477e41ae3eb --- /dev/null +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class LegacyReader + class File < LegacyReader + def initialize(path, relation_names) + @path = path + super(relation_names) + end + + def valid? + ::File.exist?(@path) + end + + private + + def tree_hash + @tree_hash ||= read_hash + end + + def read_hash + ActiveSupport::JSON.decode(IO.read(@path)) + rescue => e + Gitlab::ErrorTracking.log_exception(e) + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + end + + class User < LegacyReader + def initialize(tree_hash, relation_names) + @tree_hash = tree_hash + super(relation_names) + end + + def valid? + @tree_hash.present? + end + + protected + + attr_reader :tree_hash + end + + def initialize(relation_names) + @relation_names = relation_names.map(&:to_s) + end + + def valid? + raise NotImplementedError + end + + def legacy? + true + end + + def root_attributes(excluded_attributes = []) + attributes.except(*excluded_attributes.map(&:to_s)) + end + + def consume_relation(key) + value = relations.delete(key) + + return value unless block_given? + + return if value.nil? + + if value.is_a?(Array) + value.each.with_index do |item, idx| + yield(item, idx) + end + else + yield(value, 0) + end + end + + def consume_attribute(key) + attributes.delete(key) + end + + def sort_ci_pipelines_by_id + relations['ci_pipelines']&.sort_by! { |hash| hash['id'] } + end + + private + + attr_reader :relation_names + + def tree_hash + raise NotImplementedError + end + + def attributes + @attributes ||= tree_hash.slice!(*relation_names) + end + + def relations + @relations ||= tree_hash.extract!(*relation_names) + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_loader.rb b/lib/gitlab/import_export/project/tree_loader.rb deleted file mode 100644 index 6d4737a2d00..00000000000 --- a/lib/gitlab/import_export/project/tree_loader.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Project - class TreeLoader - def load(path, dedup_entries: false) - tree_hash = ActiveSupport::JSON.decode(IO.read(path)) - - if dedup_entries - dedup_tree(tree_hash) - else - tree_hash - end - end - - private - - # This function removes duplicate entries from the given tree recursively - # by caching nodes it encounters repeatedly. We only consider nodes for - # which there can actually be multiple equivalent instances (e.g. strings, - # hashes and arrays, but not `nil`s, numbers or booleans.) - # - # The algorithm uses a recursive depth-first descent with 3 cases, starting - # with a root node (the tree/hash itself): - # - a node has already been cached; in this case we return it from the cache - # - a node has not been cached yet but should be; descend into its children - # - a node is neither cached nor qualifies for caching; this is a no-op - def dedup_tree(node, nodes_seen = {}) - if nodes_seen.key?(node) && distinguishable?(node) - yield nodes_seen[node] - elsif should_dedup?(node) - nodes_seen[node] = node - - case node - when Array - node.each_index do |idx| - dedup_tree(node[idx], nodes_seen) do |cached_node| - node[idx] = cached_node - end - end - when Hash - node.each do |k, v| - dedup_tree(v, nodes_seen) do |cached_node| - node[k] = cached_node - end - end - end - else - node - end - end - - # We do not need to consider nodes for which there cannot be multiple instances - def should_dedup?(node) - node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass)) - end - - # We can only safely de-dup values that are distinguishable. True value objects - # are always distinguishable by nature. Hashes however can represent entities, - # which are identified by ID, not value. We therefore disallow de-duping hashes - # that do not have an `id` field, since we might risk dropping entities that - # have equal attributes yet different identities. - def distinguishable?(node) - if node.is_a?(Hash) - node.key?('id') - else - true - end - end - end - end - end -end diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index 295e0d5f348..f8d25e14c02 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -4,8 +4,6 @@ module Gitlab module ImportExport module Project class TreeRestorer - LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte - attr_reader :user attr_reader :shared attr_reader :project @@ -14,12 +12,12 @@ module Gitlab @user = user @shared = shared @project = project - @tree_loader = TreeLoader.new end def restore - @tree_hash = read_tree_hash - @project_members = @tree_hash.delete('project_members') + @relation_reader = ImportExport::JSON::LegacyReader::File.new(File.join(shared.export_path, 'project.json'), reader.project_relation_names) + + @project_members = @relation_reader.consume_relation('project_members') if relation_tree_restorer.restore import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do @@ -37,24 +35,12 @@ module Gitlab private - def large_project?(path) - File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES - end - - def read_tree_hash - path = File.join(@shared.export_path, 'project.json') - @tree_loader.load(path, dedup_entries: large_project?(path)) - rescue => e - Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') - end - def relation_tree_restorer @relation_tree_restorer ||= RelationTreeRestorer.new( user: @user, shared: @shared, importable: @project, - tree_hash: @tree_hash, + relation_reader: @relation_reader, object_builder: object_builder, members_mapper: members_mapper, relation_factory: relation_factory, diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 1390770acef..8d36d05ca6f 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -17,10 +17,18 @@ module Gitlab tree_by_key(:project) end + def project_relation_names + attributes_finder.find_relations_tree(:project).keys + end + def group_tree tree_by_key(:group) end + def group_relation_names + attributes_finder.find_relations_tree(:group).keys + end + def group_members_tree tree_by_key(:group_members) end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 8359eefc846..466cb03862e 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -9,13 +9,13 @@ module Gitlab attr_reader :user attr_reader :shared attr_reader :importable - attr_reader :tree_hash + attr_reader :relation_reader - def initialize(user:, shared:, importable:, tree_hash:, members_mapper:, object_builder:, relation_factory:, reader:) + def initialize(user:, shared:, importable:, relation_reader:, members_mapper:, object_builder:, relation_factory:, reader:) @user = user @shared = shared @importable = importable - @tree_hash = tree_hash + @relation_reader = relation_reader @members_mapper = members_mapper @object_builder = object_builder @relation_factory = relation_factory @@ -30,7 +30,7 @@ module Gitlab bulk_inserts_enabled = @importable.class == ::Project && Feature.enabled?(:import_bulk_inserts, @importable.group) BulkInsertableAssociations.with_bulk_insert(enabled: bulk_inserts_enabled) do - update_relation_hashes! + fix_ci_pipelines_not_sorted_on_legacy_project_json! create_relations! end end @@ -57,18 +57,8 @@ module Gitlab end def process_relation!(relation_key, relation_definition) - data_hashes = @tree_hash.delete(relation_key) - return unless data_hashes - - # we do not care if we process array or hash - data_hashes = [data_hashes] unless data_hashes.is_a?(Array) - - relation_index = 0 - - # consume and remove objects from memory - while data_hash = data_hashes.shift + @relation_reader.consume_relation(relation_key) do |data_hash, relation_index| process_relation_item!(relation_key, relation_definition, relation_index, data_hash) - relation_index += 1 end end @@ -103,10 +93,7 @@ module Gitlab end def update_params! - params = @tree_hash.reject do |key, _| - relations.include?(key) - end - + params = @relation_reader.root_attributes(relations.keys) params = params.merge(present_override_params) # Cleaning all imported and overridden params @@ -223,8 +210,13 @@ module Gitlab } end - def update_relation_hashes! - @tree_hash['ci_pipelines']&.sort_by! { |hash| hash['id'] } + # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json + # This should be removed once legacy JSON format is deprecated. + # Ndjson export file will fix the order during project export. + def fix_ci_pipelines_not_sorted_on_legacy_project_json! + return unless relation_reader.legacy? + + relation_reader.sort_ci_pipelines_by_id end end end diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/relation_tree_saver.rb index a0452071ccf..ed5392c13d0 100644 --- a/lib/gitlab/import_export/relation_tree_saver.rb +++ b/lib/gitlab/import_export/relation_tree_saver.rb @@ -18,7 +18,7 @@ module Gitlab def save(tree, dir_path, filename) mkdir_p(dir_path) - tree_json = JSON.generate(tree) + tree_json = ::JSON.generate(tree) File.write(File.join(dir_path, filename), tree_json) end |