Create CI migration task on GitLab side

3 files changed, 122 insertions, 21 deletions

diff --git a/lib/ci/migrate/builds.rb b/lib/ci/migrate/builds.rb
new file mode 100644
index 00000000000..fdc143cfad5
--- /dev/null
+++ b/lib/ci/migrate/builds.rb
@@ -0,0 +1,29 @@
+module Ci
+  module Migrate
+    class Builds
+      attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
+
+      def initialize
+        @app_builds_dir = Settings.gitlab_ci.builds_path
+        @backup_dir = Gitlab.config.backup.path
+        @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
+      end
+
+      def restore
+        backup_existing_builds_dir
+
+        FileUtils.mkdir_p(app_builds_dir, mode: 0700)
+        unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball)
+          abort 'Restore failed'.red
+        end
+      end
+
+      def backup_existing_builds_dir
+        timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
+        if File.exists?(app_builds_dir)
+          FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb
index 74f592dcaea..bf9b80f1f62 100644
--- a/lib/ci/migrate/database.rb
+++ b/lib/ci/migrate/database.rb
@@ -9,32 +9,32 @@ module Ci
         @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
       end
 
-      def restore(ci_dump)
-        puts 'Deleting all CI related data ... '
-        truncate_ci_tables
+      def restore
+        decompress_rd, decompress_wr = IO.pipe
+        decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+        decompress_wr.close
 
-        puts 'Restoring CI data ... '
-        case config["adapter"]
-          when /^mysql/ then
-            print "Restoring MySQL database #{config['database']} ... "
-            # Workaround warnings from MySQL 5.6 about passwords on cmd line
-            ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
-            system('mysql', *mysql_args, config['database'], in: ci_dump)
-          when "postgresql" then
-            puts "Restoring PostgreSQL database #{config['database']} ... "
-            pg_env
-            system('psql', config['database'], '-f', ci_dump)
-        end
+        restore_pid = case config["adapter"]
+                        when /^mysql/ then
+                          $progress.print "Restoring MySQL database #{config['database']} ... "
+                          # Workaround warnings from MySQL 5.6 about passwords on cmd line
+                          ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
+                          spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
+                        when "postgresql" then
+                          $progress.print "Restoring PostgreSQL database #{config['database']} ... "
+                          pg_env
+                          spawn('psql', config['database'], in: decompress_rd)
+                      end
+        decompress_rd.close
+
+        success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+        abort 'Restore failed' unless success
       end
 
       protected
 
-      def truncate_ci_tables
-        c = ActiveRecord::Base.connection
-        c.tables.select { |t| t.start_with?('ci_') }.each do |table|
-          puts "Deleting data from #{table}..."
-          c.execute("DELETE FROM #{table}")
-        end
+      def db_file_name
+        File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
       end
 
       def mysql_args
diff --git a/lib/ci/migrate/manager.rb b/lib/ci/migrate/manager.rb
new file mode 100644
index 00000000000..9405397031c
--- /dev/null
+++ b/lib/ci/migrate/manager.rb
@@ -0,0 +1,72 @@
+module Ci
+  module Migrate
+    class Manager
+      VERSION = '8.0.0.pre'
+
+      def cleanup
+        $progress.print "Deleting tmp directories ... "
+
+        backup_contents.each do |dir|
+          next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
+
+          if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
+            $progress.puts "done".green
+          else
+            puts "deleting tmp directory '#{dir}' failed".red
+            abort 'Backup failed'
+          end
+        end
+      end
+
+      def unpack
+        Dir.chdir(Gitlab.config.backup.path)
+
+        # check for existing backups in the backup dir
+        file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
+        puts "no backups found" if file_list.count == 0
+
+        if file_list.count > 1 && ENV["BACKUP"].nil?
+          puts "Found more than one backup, please specify which one you want to restore:"
+          puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+          exit 1
+        end
+
+        tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
+
+        unless File.exists?(tar_file)
+          puts "The specified CI backup doesn't exist!"
+          exit 1
+        end
+
+        $progress.print "Unpacking backup ... "
+
+        unless Kernel.system(*%W(tar -xf #{tar_file}))
+          puts "unpacking backup failed".red
+          exit 1
+        else
+          $progress.puts "done".green
+        end
+
+        ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
+
+        # restoring mismatching backups can lead to unexpected problems
+        if settings[:gitlab_version] != VERSION
+          puts "GitLab CI version mismatch:".red
+          puts "  Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
+          exit 1
+        end
+      end
+
+      private
+
+      def backup_contents
+        ["db", "builds", "backup_information.yml"]
+      end
+
+      def settings
+        @settings ||= YAML.load_file("backup_information.yml")
+      end
+    end
+  end
+end
+