Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42

10 files changed, 99 insertions, 75 deletions

diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb
index 219821a7150..37f1bdc3009 100644
--- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb
+++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb
@@ -1,24 +1,16 @@
 # frozen_string_literal: true
 
-require 'zlib'
-
 module Gitlab
   module ImportExport
     class DecompressedArchiveSizeValidator
       include Gitlab::Utils::StrongMemoize
 
       DEFAULT_MAX_BYTES = 10.gigabytes.freeze
-      CHUNK_SIZE = 4096.freeze
-
-      attr_reader :error
+      TIMEOUT_LIMIT = 60.seconds
 
       def initialize(archive_path:, max_bytes: self.class.max_bytes)
         @archive_path = archive_path
         @max_bytes = max_bytes
-        @bytes_read = 0
-        @total_reads = 0
-        @denominator = 5
-        @error = nil
       end
 
       def valid?
@@ -31,59 +23,62 @@ module Gitlab
         DEFAULT_MAX_BYTES
       end
 
-      def archive_file
-        @archive_file ||= File.open(@archive_path)
-      end
-
       private
 
       def validate
-        until archive_file.eof?
-          compressed_chunk = archive_file.read(CHUNK_SIZE)
+        pgrp = nil
+        valid_archive = true
 
-          inflate_stream.inflate(compressed_chunk) do |chunk|
-            @bytes_read += chunk.size
-            @total_reads += 1
-          end
+        Timeout.timeout(TIMEOUT_LIMIT) do
+          stdin, stdout, stderr, wait_thr = Open3.popen3(command, pgroup: true)
+          stdin.close
+          pgrp = Process.getpgid(wait_thr[:pid])
+          status = wait_thr.value
 
-          # Start garbage collection every 5 reads in order
-          # to prevent memory bloat during archive decompression
-          GC.start if gc_start?
+          if status.success?
+            result = stdout.readline
 
-          if @bytes_read > @max_bytes
-            @error = error_message
+            if result.to_i > @max_bytes
+              valid_archive = false
 
-            return false
+              log_error('Decompressed archive size limit reached')
+            end
+          else
+            valid_archive = false
+
+            log_error(stderr.readline)
           end
+
+        ensure
+          stdout.close
+          stderr.close
         end
 
-        true
-      rescue => e
-        @error = error_message
+        valid_archive
+      rescue Timeout::Error
+        log_error('Timeout reached during archive decompression')
 
-        Gitlab::ErrorTracking.track_exception(e)
-
-        Gitlab::Import::Logger.info(
-          message: @error,
-          error: e.message
-        )
+        Process.kill(-1, pgrp) if pgrp
 
         false
-      ensure
-        inflate_stream.close
-        archive_file.close
-      end
+      rescue => e
+        log_error(e.message)
 
-      def inflate_stream
-        @inflate_stream ||= Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+        Process.kill(-1, pgrp) if pgrp
+
+        false
       end
 
-      def gc_start?
-        @total_reads % @denominator == 0
+      def command
+        "gzip -dc #{@archive_path} | wc -c"
       end
 
-      def error_message
-        _('Decompressed archive size validation failed.')
+      def log_error(error)
+        Gitlab::Import::Logger.info(
+          message: error,
+          import_upload_archive_path: @archive_path,
+          import_upload_archive_size: File.size(@archive_path)
+        )
       end
     end
   end
diff --git a/lib/gitlab/import_export/design_repo_restorer.rb b/lib/gitlab/import_export/design_repo_restorer.rb
index a702c58a7c2..e093b4b0697 100644
--- a/lib/gitlab/import_export/design_repo_restorer.rb
+++ b/lib/gitlab/import_export/design_repo_restorer.rb
@@ -3,10 +3,11 @@
 module Gitlab
   module ImportExport
     class DesignRepoRestorer < RepoRestorer
-      def initialize(project:, shared:, path_to_bundle:)
-        super(project: project, shared: shared, path_to_bundle: path_to_bundle)
+      extend ::Gitlab::Utils::Override
 
-        @repository = project.design_repository
+      override :repository
+      def repository
+        @repository ||= importable.design_repository
       end
 
       # `restore` method is handled in super class
diff --git a/lib/gitlab/import_export/design_repo_saver.rb b/lib/gitlab/import_export/design_repo_saver.rb
index db9ebee6a13..b400aedc205 100644
--- a/lib/gitlab/import_export/design_repo_saver.rb
+++ b/lib/gitlab/import_export/design_repo_saver.rb
@@ -3,16 +3,18 @@
 module Gitlab
   module ImportExport
     class DesignRepoSaver < RepoSaver
-      def save
-        @repository = project.design_repository
+      extend ::Gitlab::Utils::Override
 
-        super
+      override :repository
+      def repository
+        @repository ||= exportable.design_repository
       end
 
       private
 
-      def bundle_full_path
-        File.join(shared.export_path, ::Gitlab::ImportExport.design_repo_bundle_filename)
+      override :bundle_filename
+      def bundle_filename
+        ::Gitlab::ImportExport.design_repo_bundle_filename
       end
     end
   end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 5a6f6e017d2..51d58aae54f 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -87,7 +87,7 @@ module Gitlab
       end
 
       def validate_decompressed_archive_size
-        raise ImporterError.new(size_validator.error) unless size_validator.valid?
+        raise ImporterError.new(_('Decompressed archive size validation failed.')) unless size_validator.valid?
       end
 
       def size_validator
diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb
index dfe27118d66..925ab6680ba 100644
--- a/lib/gitlab/import_export/group/tree_restorer.rb
+++ b/lib/gitlab/import_export/group/tree_restorer.rb
@@ -6,7 +6,7 @@ module Gitlab
       class TreeRestorer
         include Gitlab::Utils::StrongMemoize
 
-        attr_reader :user, :shared
+        attr_reader :user, :shared, :groups_mapping
 
         def initialize(user:, shared:, group:)
           @user = user
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 789249c7d91..390909efe36 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -75,19 +75,19 @@ module Gitlab
       def repo_restorer
         Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path,
                                                shared: shared,
-                                               project: project)
+                                               importable: project)
       end
 
       def wiki_restorer
         Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path,
                                                shared: shared,
-                                               project: ProjectWiki.new(project))
+                                               importable: ProjectWiki.new(project))
       end
 
       def design_repo_restorer
         Gitlab::ImportExport::DesignRepoRestorer.new(path_to_bundle: design_repo_path,
                                                      shared: shared,
-                                                     project: project)
+                                                     importable: project)
       end
 
       def uploads_restorer
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index f808e30bd6e..8af7b68d78e 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -5,10 +5,12 @@ module Gitlab
     class RepoRestorer
       include Gitlab::ImportExport::CommandLineUtil
 
-      def initialize(project:, shared:, path_to_bundle:)
-        @repository = project.repository
+      attr_reader :importable
+
+      def initialize(importable:, shared:, path_to_bundle:)
         @path_to_bundle = path_to_bundle
         @shared = shared
+        @importable = importable
       end
 
       def restore
@@ -17,14 +19,25 @@ module Gitlab
         ensure_repository_does_not_exist!
 
         repository.create_from_bundle(path_to_bundle)
+        update_importable_repository_info
+
+        true
       rescue => e
         shared.error(e)
         false
       end
 
+      def repository
+        @repository ||= importable.repository
+      end
+
       private
 
-      attr_accessor :repository, :path_to_bundle, :shared
+      attr_accessor :path_to_bundle, :shared
+
+      def update_importable_repository_info
+        # No-op. Overridden in EE
+      end
 
       def ensure_repository_does_not_exist!
         if repository.exists?
@@ -38,3 +51,5 @@ module Gitlab
     end
   end
 end
+
+Gitlab::ImportExport::RepoRestorer.prepend_if_ee('EE::Gitlab::ImportExport::RepoRestorer')
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index 898cd7898ba..0fdd0722b65 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -5,12 +5,11 @@ module Gitlab
     class RepoSaver
       include Gitlab::ImportExport::CommandLineUtil
 
-      attr_reader :project, :repository, :shared
+      attr_reader :exportable, :shared
 
-      def initialize(project:, shared:)
-        @project = project
+      def initialize(exportable:, shared:)
+        @exportable = exportable
         @shared = shared
-        @repository = @project.repository
       end
 
       def save
@@ -19,6 +18,10 @@ module Gitlab
         bundle_to_disk
       end
 
+      def repository
+        @repository ||= @exportable.repository
+      end
+
       private
 
       def repository_exists?
@@ -26,11 +29,16 @@ module Gitlab
       end
 
       def bundle_full_path
-        File.join(shared.export_path, ImportExport.project_bundle_filename)
+        File.join(shared.export_path, bundle_filename)
+      end
+
+      def bundle_filename
+        ::Gitlab::ImportExport.project_bundle_filename
       end
 
       def bundle_to_disk
-        mkdir_p(shared.export_path)
+        mkdir_p(File.dirname(bundle_full_path))
+
         repository.bundle_to_disk(bundle_full_path)
       rescue => e
         shared.error(e)
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index 045ba2495bf..bb2bbda4bd6 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -31,7 +31,7 @@ module Gitlab
         @shared.error(e)
         false
       ensure
-        remove_base_tmp_dir
+        remove_archive_tmp_dir
       end
 
       private
@@ -40,8 +40,8 @@ module Gitlab
         tar_czf(archive: archive_file, dir: @shared.export_path)
       end
 
-      def remove_base_tmp_dir
-        FileUtils.rm_rf(@shared.base_path)
+      def remove_archive_tmp_dir
+        FileUtils.rm_rf(@shared.archive_path)
       end
 
       def archive_file
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 93ae6f6b02a..4b1cf4915e4 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -3,18 +3,21 @@
 module Gitlab
   module ImportExport
     class WikiRepoSaver < RepoSaver
-      def save
-        wiki = ProjectWiki.new(project)
-        @repository = wiki.repository
+      extend ::Gitlab::Utils::Override
 
-        super
+      override :repository
+      def repository
+        @repository ||= exportable.wiki.repository
       end
 
       private
 
-      def bundle_full_path
-        File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename)
+      override :bundle_filename
+      def bundle_filename
+        ::Gitlab::ImportExport.wiki_repo_bundle_filename
       end
     end
   end
 end
+
+Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver')