Add latest changes from gitlab-org/gitlab@13-1-stable-ee

13 files changed, 180 insertions, 28 deletions

diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb
index bab473741b1..1e98595bb07 100644
--- a/lib/gitlab/import_export/attributes_finder.rb
+++ b/lib/gitlab/import_export/attributes_finder.rb
@@ -3,6 +3,8 @@
 module Gitlab
   module ImportExport
     class AttributesFinder
+      attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads
+
       def initialize(config:)
         @tree = config[:tree] || {}
         @included_attributes = config[:included_attributes] || {}
diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb
new file mode 100644
index 00000000000..86f51add504
--- /dev/null
+++ b/lib/gitlab/import_export/attributes_permitter.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+# AttributesPermitter builds a hash of permitted attributes for
+# every model defined in import_export.yml that is used to validate and
+# filter out any attributes that are not permitted when doing Project/Group Import
+#
+# Each model's list includes:
+#   - attributes defined under included_attributes section
+#   - associations defined under project/group tree
+#   - methods defined under methods section
+#
+# Given the following import_export.yml example:
+# ```
+#   tree:
+#     project:
+#       - labels:
+#         - :priorities
+#   included_attributes:
+#     labels:
+#       - :title
+#       - :description
+#    methods:
+#      labels:
+#        - :type
+# ```
+#
+# Produces a list of permitted attributes:
+# ```
+#   Gitlab::ImportExport::AttributesPermitter.new.permitted_attributes
+#
+#   => { labels: [:priorities, :title, :description, :type] }
+# ```
+#
+# Filters out any other attributes from specific relation hash:
+# ```
+#   Gitlab::ImportExport::AttributesPermitter.new.permit(:labels, {id: 5, type: 'opened', description: 'test', sensitive_attribute: 'my_sensitive_attribute'})
+#
+#   => {:type=>"opened", :description=>"test"}
+# ```
+module Gitlab
+  module ImportExport
+    class AttributesPermitter
+      attr_reader :permitted_attributes
+
+      def initialize(config: ImportExport::Config.new.to_h)
+        @config = config
+        @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config)
+        @permitted_attributes = {}
+
+        build_permitted_attributes
+      end
+
+      def permit(relation_name, relation_hash)
+        permitted_attributes = permitted_attributes_for(relation_name)
+
+        relation_hash.select do |key, _|
+          permitted_attributes.include?(key)
+        end
+      end
+
+      def permitted_attributes_for(relation_name)
+        @permitted_attributes[relation_name] || []
+      end
+
+      private
+
+      def build_permitted_attributes
+        build_associations
+        build_attributes
+        build_methods
+      end
+
+      # Deep traverse relations tree to build a list of allowed model relations
+      def build_associations
+        stack = @attributes_finder.tree.to_a
+
+        while stack.any?
+          model_name, relations = stack.pop
+
+          if relations.is_a?(Hash)
+            add_permitted_attributes(model_name, relations.keys)
+
+            stack.concat(relations.to_a)
+          end
+        end
+
+        @permitted_attributes
+      end
+
+      def build_attributes
+        @attributes_finder.included_attributes.each(&method(:add_permitted_attributes))
+      end
+
+      def build_methods
+        @attributes_finder.methods.each(&method(:add_permitted_attributes))
+      end
+
+      def add_permitted_attributes(model_name, attributes)
+        @permitted_attributes[model_name] ||= []
+
+        @permitted_attributes[model_name].concat(attributes) if attributes.any?
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index b1219384732..7b8689069d8 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -24,8 +24,14 @@ module Gitlab
           raise Projects::ImportService::Error.new(shared.errors.to_sentence)
         end
       rescue => e
+        # If some exception was raised could mean that the SnippetsRepoRestorer
+        # was not called. This would leave us with snippets without a repository.
+        # This is a state we don't want them to be, so we better delete them.
+        remove_non_migrated_snippets
+
         raise Projects::ImportService::Error.new(e.message)
       ensure
+        remove_base_tmp_dir
         remove_import_file
       end
 
@@ -148,6 +154,18 @@ module Gitlab
           ::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
         end
       end
+
+      def remove_base_tmp_dir
+        FileUtils.rm_rf(@shared.base_path)
+      end
+
+      def remove_non_migrated_snippets
+        project
+          .snippets
+          .left_joins(:snippet_repository)
+          .where(snippet_repositories: { snippet_id: nil })
+          .delete_all
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb
index 7f55a0a3821..20f9c668b9c 100644
--- a/lib/gitlab/import_export/json/streaming_serializer.rb
+++ b/lib/gitlab/import_export/json/streaming_serializer.rb
@@ -7,6 +7,15 @@ module Gitlab
         include Gitlab::ImportExport::CommandLineUtil
 
         BATCH_SIZE = 100
+        SMALLER_BATCH_SIZE = 20
+
+        def self.batch_size(exportable)
+          if Feature.enabled?(:export_reduce_relation_batch_size, exportable)
+            SMALLER_BATCH_SIZE
+          else
+            BATCH_SIZE
+          end
+        end
 
         class Raw < String
           def to_json(*_args)
@@ -60,7 +69,7 @@ module Gitlab
             key_preloads = preloads&.dig(key)
             records = records.preload(key_preloads) if key_preloads
 
-            records.find_each(batch_size: BATCH_SIZE) do |record|
+            records.find_each(batch_size: batch_size) do |record|
               items << Raw.new(record.to_json(options))
             end
           end
@@ -91,6 +100,10 @@ module Gitlab
         def preloads
           relations_schema[:preload]
         end
+
+        def batch_size
+          @batch_size ||= self.class.batch_size(@exportable)
+        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
index cf75a2c7fa8..f8b8b74ffd7 100644
--- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb
+++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
@@ -7,7 +7,7 @@ module Gitlab
 
       def serialize(exportable, relations_tree)
         Gitlab::ImportExport::FastHashSerializer
-          .new(exportable, relations_tree)
+          .new(exportable, relations_tree, batch_size: batch_size(exportable))
           .execute
       end
 
@@ -18,6 +18,12 @@ module Gitlab
 
         File.write(File.join(dir_path, filename), tree_json)
       end
+
+      private
+
+      def batch_size(exportable)
+        Gitlab::ImportExport::JSON::StreamingSerializer.batch_size(exportable)
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index 263c49c509f..31d1f7b48bd 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -49,7 +49,7 @@ module Gitlab
       def ensure_default_member!
         return if user_already_member?
 
-        @importable.members.destroy_all # rubocop: disable DestroyAll
+        @importable.members.destroy_all # rubocop: disable Cop/DestroyAll
 
         relation_class.create!(user: @user, access_level: highest_access_level, source_id: @importable.id, importing: true)
       rescue => e
diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb
index f735b9612aa..4643742b607 100644
--- a/lib/gitlab/import_export/merge_request_parser.rb
+++ b/lib/gitlab/import_export/merge_request_parser.rb
@@ -41,7 +41,13 @@ module Gitlab
       def create_source_branch
         @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha)
       rescue => err
-        Rails.logger.warn("Import/Export warning: Failed to create source branch #{@merge_request.source_branch} => #{@diff_head_sha} for MR #{@merge_request.iid}: #{err}") # rubocop:disable Gitlab/RailsLogger
+        Gitlab::Import::Logger.warn(
+          message: 'Import warning: Failed to create source branch',
+          source_branch: @merge_request.source_branch,
+          diff_head_sha: @diff_head_sha,
+          merge_request_iid: @merge_request.iid,
+          error: err.message
+        )
       end
 
       def create_target_branch
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 8851b106ad5..f0b733d7e95 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -312,6 +312,7 @@ excluded_attributes:
     - :pipeline_schedule_id
     - :merge_request_id
     - :external_pull_request_id
+    - :ci_ref_id
   stages:
     - :pipeline_id
   merge_access_levels:
@@ -397,3 +398,4 @@ ee:
       - protected_environments:
         - :deploy_access_levels
       - :service_desk_setting
+      - :security_setting
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index 3123687453f..9e10e7aea13 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -16,6 +16,8 @@ module Gitlab
 
         repository.create_from_bundle(path_to_bundle)
       rescue => e
+        Repositories::DestroyService.new(repository).execute
+
         shared.error(e)
         false
       end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index ae82c380755..e4724659eff 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -11,14 +11,16 @@ module Gitlab
 
       def initialize(exportable:, shared:)
         @exportable = exportable
-        @shared     = shared
+        @shared = shared
       end
 
       def save
         if compress_and_save
-          remove_export_path
-
-          Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger
+          Gitlab::Export::Logger.info(
+            message: 'Export archive saved',
+            exportable_class: @exportable.class.to_s,
+            archive_file: archive_file
+          )
 
           save_upload
         else
@@ -29,8 +31,7 @@ module Gitlab
         @shared.error(e)
         false
       ensure
-        remove_archive
-        remove_export_path
+        remove_base_tmp_dir
       end
 
       private
@@ -39,12 +40,8 @@ module Gitlab
         tar_czf(archive: archive_file, dir: @shared.export_path)
       end
 
-      def remove_export_path
-        FileUtils.rm_rf(@shared.export_path)
-      end
-
-      def remove_archive
-        FileUtils.rm_rf(@shared.archive_path)
+      def remove_base_tmp_dir
+        FileUtils.rm_rf(@shared.base_path)
       end
 
       def archive_file
diff --git a/lib/gitlab/import_export/snippet_repo_restorer.rb b/lib/gitlab/import_export/snippet_repo_restorer.rb
index b58ea14a3a8..334d13a13ae 100644
--- a/lib/gitlab/import_export/snippet_repo_restorer.rb
+++ b/lib/gitlab/import_export/snippet_repo_restorer.rb
@@ -5,6 +5,8 @@ module Gitlab
     class SnippetRepoRestorer < RepoRestorer
       attr_reader :snippet
 
+      SnippetRepositoryError = Class.new(StandardError)
+
       def initialize(snippet:, user:, shared:, path_to_bundle:)
         @snippet = snippet
         @user = user
@@ -34,14 +36,11 @@ module Gitlab
       end
 
       def create_repository_from_db
-        snippet.create_repository
-
-        commit_attrs = {
-          branch_name: 'master',
-          message: 'Initial commit'
-        }
+        Gitlab::BackgroundMigration::BackfillSnippetRepositories.new.perform_by_ids([snippet.id])
 
-        repository.create_file(@user, snippet.file_name, snippet.content, commit_attrs)
+        unless snippet.reset.snippet_repository
+          raise SnippetRepositoryError, _("Error creating repository for snippet with id %{snippet_id}") % { snippet_id: snippet.id }
+        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/snippets_repo_restorer.rb b/lib/gitlab/import_export/snippets_repo_restorer.rb
index 9ff3e74a6b1..5ab28f8dd83 100644
--- a/lib/gitlab/import_export/snippets_repo_restorer.rb
+++ b/lib/gitlab/import_export/snippets_repo_restorer.rb
@@ -10,15 +10,13 @@ module Gitlab
       end
 
       def restore
-        return true unless Dir.exist?(snippets_repo_bundle_path)
-
-        @project.snippets.find_each.all? do |snippet|
+        @project.snippets.find_each.map do |snippet|
           Gitlab::ImportExport::SnippetRepoRestorer.new(snippet: snippet,
                                                         user: @user,
                                                         shared: @shared,
                                                         path_to_bundle: snippet_repo_bundle_path(snippet))
                                                    .restore
-        end
+        end.all?(true)
       end
 
       private
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
index 86ea7a30e69..4154d4fe775 100644
--- a/lib/gitlab/import_export/version_checker.rb
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -36,7 +36,11 @@ module Gitlab
       def different_version?(version)
         Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version)
       rescue => e
-        Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
+        Gitlab::Import::Logger.error(
+          message: 'Import error',
+          error: e.message
+        )
+
         raise Gitlab::ImportExport::Error.new('Incorrect VERSION format')
       end
     end