Merge remote-tracking branch 'upstream/master' into feature/1376-allow-write-access-deploy-keys

* upstream/master: (3852 commits)
  Grapify token API
  Fix cache for commit status in commits list to respect branches
  Grapify milestones API
  Grapify runners API
  Improve EeCompatCheck, cache EE repo and keep artifacts for the ee_compat_check task
  Use 'Forking in progress' title when appropriate
  Fix CHANGELOG after 8.14.0-rc1 tag
  Update CHANGELOG.md for 8.14.0-rc1
  Fix YAML syntax on CHANGELOG entry
  Remove redundant rescue from repository keep_around
  Remove redundant space from repository model code
  Remove order-dependent expectation
  Minor CHANGELOG.md cleanups
  Add a link to Git cheatsheet PDF in docs readme
  Grapify the session API
  Add 8.13.5, 8.12.9, and 8.11.11 CHANGELOG
  Merge branch 'unauthenticated-container-registry-access' into 'security'
  Merge branch '23403-fix-events-for-private-project-features' into 'security'
  Merge branch 'fix-unathorized-cloning' into 'security'
  Merge branch 'markdown-xss-fix-option-2.1' into 'security'
  ...

1 files changed, 77 insertions, 17 deletions

diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb
index f117fc3d37d..d746070913d 100644
--- a/lib/backup/repository.rb
+++ b/lib/backup/repository.rb
@@ -2,11 +2,14 @@ require 'yaml'
 
 module Backup
   class Repository
+
     def dump
       prepare
 
       Project.find_each(batch_size: 1000) do |project|
         $progress.print " * #{project.path_with_namespace} ... "
+        path_to_project_repo = path_to_repo(project)
+        path_to_project_bundle = path_to_bundle(project)
 
         # Create namespace dir if missing
         FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace
@@ -14,8 +17,22 @@ module Backup
         if project.empty_repo?
           $progress.puts "[SKIPPED]".color(:cyan)
         else
-          cmd = %W(tar -cf #{path_to_bundle(project)} -C #{path_to_repo(project)} .)
+          in_path(path_to_project_repo) do |dir|
+            FileUtils.mkdir_p(path_to_tars(project))
+            cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
+            output, status = Gitlab::Popen.popen(cmd)
+
+            unless status.zero?
+              puts "[FAILED]".color(:red)
+              puts "failed: #{cmd.join(' ')}"
+              puts output
+              abort 'Backup failed'
+            end
+          end
+
+          cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all)
           output, status = Gitlab::Popen.popen(cmd)
+
           if status.zero?
             $progress.puts "[DONE]".color(:green)
           else
@@ -27,19 +44,22 @@ module Backup
         end
 
         wiki = ProjectWiki.new(project)
+        path_to_wiki_repo = path_to_repo(wiki)
+        path_to_wiki_bundle = path_to_bundle(wiki)
 
-        if File.exist?(path_to_repo(wiki))
+        if File.exist?(path_to_wiki_repo)
           $progress.print " * #{wiki.path_with_namespace} ... "
           if wiki.repository.empty?
             $progress.puts " [SKIPPED]".color(:cyan)
           else
-            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all)
+            cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all)
             output, status = Gitlab::Popen.popen(cmd)
             if status.zero?
               $progress.puts " [DONE]".color(:green)
             else
               puts " [FAILED]".color(:red)
               puts "failed: #{cmd.join(' ')}"
+              puts output
               abort 'Backup failed'
             end
           end
@@ -55,45 +75,64 @@ module Backup
         bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s)
         FileUtils.mv(path, bk_repos_path)
         # This is expected from gitlab:check
-        FileUtils.mkdir_p(path, mode: 2770)
+        FileUtils.mkdir_p(path, mode: 02770)
       end
 
       Project.find_each(batch_size: 1000) do |project|
         $progress.print " * #{project.path_with_namespace} ... "
+        path_to_project_repo = path_to_repo(project)
+        path_to_project_bundle = path_to_bundle(project)
 
         project.ensure_dir_exist
 
-        if File.exist?(path_to_bundle(project))
-          FileUtils.mkdir_p(path_to_repo(project))
-          cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)})
+        if File.exists?(path_to_project_bundle)
+          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo})
         else
-          cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)})
+          cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo})
         end
 
-        if system(*cmd, silent)
+        output, status = Gitlab::Popen.popen(cmd)
+        if status.zero?
           $progress.puts "[DONE]".color(:green)
         else
           puts "[FAILED]".color(:red)
           puts "failed: #{cmd.join(' ')}"
+          puts output
           abort 'Restore failed'
         end
 
+        in_path(path_to_tars(project)) do |dir|
+          cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir})
+
+          output, status = Gitlab::Popen.popen(cmd)
+          unless status.zero?
+            puts "[FAILED]".color(:red)
+            puts "failed: #{cmd.join(' ')}"
+            puts output
+            abort 'Restore failed'
+          end
+        end
+
         wiki = ProjectWiki.new(project)
+        path_to_wiki_repo = path_to_repo(wiki)
+        path_to_wiki_bundle = path_to_bundle(wiki)
 
-        if File.exist?(path_to_bundle(wiki))
+        if File.exist?(path_to_wiki_bundle)
           $progress.print " * #{wiki.path_with_namespace} ... "
 
           # If a wiki bundle exists, first remove the empty repo
           # that was initialized with ProjectWiki.new() and then
           # try to restore with 'git clone --bare'.
-          FileUtils.rm_rf(path_to_repo(wiki))
-          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)})
+          FileUtils.rm_rf(path_to_wiki_repo)
+          cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo})
 
-          if system(*cmd, silent)
+          output, status = Gitlab::Popen.popen(cmd)
+          if status.zero?
             $progress.puts " [DONE]".color(:green)
           else
             puts " [FAILED]".color(:red)
             puts "failed: #{cmd.join(' ')}"
+            puts output
             abort 'Restore failed'
           end
         end
@@ -101,13 +140,15 @@ module Backup
 
       $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow)
       cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args
-      if system(*cmd)
+
+      output, status = Gitlab::Popen.popen(cmd)
+      if status.zero?
         $progress.puts " [DONE]".color(:green)
       else
         puts " [FAILED]".color(:red)
         puts "failed: #{cmd}"
+        puts output
       end
-
     end
 
     protected
@@ -117,11 +158,30 @@ module Backup
     end
 
     def path_to_bundle(project)
-      File.join(backup_repos_path, project.path_with_namespace + ".bundle")
+      File.join(backup_repos_path, project.path_with_namespace + '.bundle')
+    end
+
+    def path_to_tars(project, dir = nil)
+      path = File.join(backup_repos_path, project.path_with_namespace)
+
+      if dir
+        File.join(path, "#{dir}.tar")
+      else
+        path
+      end
     end
 
     def backup_repos_path
-      File.join(Gitlab.config.backup.path, "repositories")
+      File.join(Gitlab.config.backup.path, 'repositories')
+    end
+
+    def in_path(path)
+      return unless Dir.exist?(path)
+
+      dir_entries = Dir.entries(path)
+      %w[annex custom_hooks].each do |entry|
+        yield(entry) if dir_entries.include?(entry)
+      end
     end
 
     def prepare