From 8df6ac5bc635c5b0c5eb33bdf3b8f030e521385a Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Fri, 18 Sep 2015 11:58:45 +0200 Subject: Make backups more space-efficient --- lib/backup/builds.rb | 18 +++++++++++++----- lib/backup/database.rb | 31 ++++++++++++++++++++++++------- lib/backup/manager.rb | 16 ++++++++-------- 3 files changed, 45 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb index 71e9704..285cf65 100644 --- a/lib/backup/builds.rb +++ b/lib/backup/builds.rb @@ -1,23 +1,31 @@ module Backup class Builds - attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir + attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir def initialize @app_builds_dir = File.realpath(Rails.root.join('builds')) @backup_dir = GitlabCi.config.backup.path - @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'builds') + @backup_builds_tarball = File.join(GitlabCi.config.backup.path, 'builds/builds.tar.gz') end # Copy builds from builds directory to backup/builds def dump - FileUtils.mkdir_p(backup_builds_dir) - FileUtils.cp_r(app_builds_dir, backup_dir) + FileUtils.mkdir_p(File.dirname(backup_builds_tarball)) + FileUtils.rm_f(backup_builds_tarball) + + system( + *%W(tar -C #{app_builds_dir} -czf - -- .), + out: [backup_builds_tarball, 'w', 0600] + ) end def restore backup_existing_builds_dir - FileUtils.cp_r(backup_builds_dir, app_builds_dir) + system( + *%W(tar -C #{app_builds_dir} -xzf - -- .), + in: backup_builds_tarball + ) end def backup_existing_builds_dir diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 3ef5f44..0c5a3e9 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -11,24 +11,37 @@ module Backup end def dump - success = case config["adapter"] + FileUtils.rm_f(db_file_name) + compress_rd, compress_wr = IO.pipe + compress_pid = spawn(*%W(gzip -c), in: compress_rd, out: [db_file_name, 'w', 0600]) + compress_rd.close + + dump_pid = case config["adapter"] when /^mysql/ then $progress.print "Dumping MySQL database #{config['database']} ... " - system('mysqldump', *mysql_args, config['database'], out: db_file_name) + spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) when "postgresql" then $progress.print "Dumping PostgreSQL database #{config['database']} ... " pg_env - system('pg_dump', config['database'], out: db_file_name) + spawn('pg_dump', config['database'], out: compress_wr) end + compress_wr.close + + success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? } + report_success(success) abort 'Backup failed' unless success end def restore - success = case config["adapter"] + decompress_rd, decompress_wr = IO.pipe + decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_wr.close + + restore_pid = case config["adapter"] when /^mysql/ then $progress.print "Restoring MySQL database #{config['database']} ... " - system('mysql', *mysql_args, config['database'], in: db_file_name) + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) when "postgresql" then $progress.print "Restoring PostgreSQL database #{config['database']} ... " # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE @@ -36,8 +49,12 @@ module Backup drop_all_tables drop_all_postgres_sequences pg_env - system('psql', config['database'], '-f', db_file_name) + spawn('psql', config['database'], in: decompress_rd) end + decompress_rd.close + + success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + report_success(success) abort 'Restore failed' unless success end @@ -45,7 +62,7 @@ module Backup protected def db_file_name - File.join(db_dir, 'database.sql') + File.join(db_dir, 'database.sql.gz') end def mysql_args diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 43fb362..8ad3ea6 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -7,7 +7,7 @@ module Backup s[:backup_created_at] = Time.now s[:gitlab_version] = GitlabCi::VERSION s[:tar_version] = tar_version - tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar.gz" + tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar" Dir.chdir(GitlabCi.config.backup.path) do File.open("#{GitlabCi.config.backup.path}/backup_information.yml", @@ -20,7 +20,7 @@ module Backup # create archive $progress.print "Creating backup archive: #{tar_file} ... " orig_umask = File.umask(0077) - if Kernel.system('tar', '-czf', tar_file, *backup_contents) + if Kernel.system('tar', '-cf', tar_file, *backup_contents) $progress.puts "done".green else puts "creating archive #{tar_file} failed".red @@ -78,11 +78,11 @@ module Backup removed = 0 Dir.chdir(GitlabCi.config.backup.path) do - file_list = Dir.glob('*_gitlab_ci_backup.tar.gz') - file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar.gz/ } + file_list = Dir.glob('*_gitlab_ci_backup.tar') + file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar/ } file_list.sort.each do |timestamp| if Time.at(timestamp) < (Time.now - keep_time) - if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar.gz)) + if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar)) removed += 1 end end @@ -99,7 +99,7 @@ module Backup Dir.chdir(GitlabCi.config.backup.path) # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_ci_backup.tar.gz").each.map { |f| f.split(/_/).first.to_i } + file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i } puts "no backups found" if file_list.count == 0 if file_list.count > 1 && ENV["BACKUP"].nil? @@ -108,7 +108,7 @@ module Backup exit 1 end - tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar.gz") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar.gz") + tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar") unless File.exists?(tar_file) puts "The specified backup doesn't exist!" @@ -117,7 +117,7 @@ module Backup $progress.print "Unpacking backup ... " - unless Kernel.system(*%W(tar -xzf #{tar_file})) + unless Kernel.system(*%W(tar -xf #{tar_file})) puts "unpacking backup failed".red exit 1 else -- cgit v1.2.1