From ed18e04bb3d921d14f1bfb9adf7e0a4ad4dfc0b2 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 15 Sep 2015 10:24:30 +0200 Subject: Cleanup CI backup => migrate with GitLab --- lib/backup/builds.rb | 30 +++++++ lib/ci/backup/builds.rb | 32 ------- lib/ci/backup/database.rb | 94 -------------------- lib/ci/backup/manager.rb | 158 ---------------------------------- lib/tasks/ci/backup.rake | 62 ------------- lib/tasks/gitlab/backup.rake | 21 +++++ spec/tasks/gitlab/backup_rake_spec.rb | 4 + 7 files changed, 55 insertions(+), 346 deletions(-) create mode 100644 lib/backup/builds.rb delete mode 100644 lib/ci/backup/builds.rb delete mode 100644 lib/ci/backup/database.rb delete mode 100644 lib/ci/backup/manager.rb delete mode 100644 lib/tasks/ci/backup.rake diff --git a/lib/backup/builds.rb b/lib/backup/builds.rb new file mode 100644 index 00000000000..4280438e86c --- /dev/null +++ b/lib/backup/builds.rb @@ -0,0 +1,30 @@ +module Backup + class Builds + attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir + + def initialize + @app_builds_dir = File.realpath(Rails.root.join('ci/builds')) + @backup_dir = GitlabCi.config.backup.path + @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds') + end + + # Copy builds from builds directory to backup/builds + def dump + FileUtils.mkdir_p(backup_builds_dir) + FileUtils.cp_r(app_builds_dir, backup_dir) + end + + def restore + backup_existing_builds_dir + + FileUtils.cp_r(backup_builds_dir, app_builds_dir) + end + + def backup_existing_builds_dir + timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") + if File.exists?(app_builds_dir) + FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) + end + end + end +end diff --git a/lib/ci/backup/builds.rb b/lib/ci/backup/builds.rb deleted file mode 100644 index 832a5ab8fdc..00000000000 --- a/lib/ci/backup/builds.rb +++ /dev/null @@ -1,32 +0,0 @@ -module Ci - module Backup - class Builds - attr_reader :app_builds_dir, :backup_builds_dir, :backup_dir - - def initialize - @app_builds_dir = File.realpath(Rails.root.join('ci/builds')) - @backup_dir = GitlabCi.config.backup.path - @backup_builds_dir = File.join(GitlabCi.config.backup.path, 'ci/builds') - end - - # Copy builds from builds directory to backup/builds - def dump - FileUtils.mkdir_p(backup_builds_dir) - FileUtils.cp_r(app_builds_dir, backup_dir) - end - - def restore - backup_existing_builds_dir - - FileUtils.cp_r(backup_builds_dir, app_builds_dir) - end - - def backup_existing_builds_dir - timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}") - if File.exists?(app_builds_dir) - FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path)) - end - end - end - end -end diff --git a/lib/ci/backup/database.rb b/lib/ci/backup/database.rb deleted file mode 100644 index 3f2277024e4..00000000000 --- a/lib/ci/backup/database.rb +++ /dev/null @@ -1,94 +0,0 @@ -require 'yaml' - -module Ci - module Backup - class Database - attr_reader :config, :db_dir - - def initialize - @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] - @db_dir = File.join(GitlabCi.config.backup.path, 'db') - FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir) - end - - def dump - success = case config["adapter"] - when /^mysql/ then - $progress.print "Dumping MySQL database #{config['database']} ... " - system('mysqldump', *mysql_args, config['database'], out: db_file_name) - when "postgresql" then - $progress.print "Dumping PostgreSQL database #{config['database']} ... " - pg_env - system('pg_dump', config['database'], out: db_file_name) - end - report_success(success) - abort 'Backup failed' unless success - end - - def restore - success = case config["adapter"] - when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " - system('mysql', *mysql_args, config['database'], in: db_file_name) - when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " - # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE - # statements like MySQL. - drop_all_tables - drop_all_postgres_sequences - pg_env - system('psql', config['database'], '-f', db_file_name) - end - report_success(success) - abort 'Restore failed' unless success - end - - protected - - def db_file_name - File.join(db_dir, 'database.sql') - end - - def mysql_args - args = { - 'host' => '--host', - 'port' => '--port', - 'socket' => '--socket', - 'username' => '--user', - 'encoding' => '--default-character-set', - 'password' => '--password' - } - args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact - end - - def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] - end - - def report_success(success) - if success - $progress.puts '[DONE]'.green - else - $progress.puts '[FAILED]'.red - end - end - - def drop_all_tables - connection = ActiveRecord::Base.connection - connection.tables.each do |table| - connection.drop_table(table) - end - end - - def drop_all_postgres_sequences - connection = ActiveRecord::Base.connection - connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence| - connection.execute("DROP SEQUENCE #{sequence['relname']}") - end - end - end - end -end diff --git a/lib/ci/backup/manager.rb b/lib/ci/backup/manager.rb deleted file mode 100644 index 2e9d6df7139..00000000000 --- a/lib/ci/backup/manager.rb +++ /dev/null @@ -1,158 +0,0 @@ -module Ci - module Backup - class Manager - def pack - # saving additional informations - s = {} - s[:db_version] = "#{ActiveRecord::Migrator.current_version}" - s[:backup_created_at] = Time.now - s[:gitlab_version] = GitlabCi::VERSION - s[:tar_version] = tar_version - tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar.gz" - - Dir.chdir(GitlabCi.config.backup.path) do - File.open("#{GitlabCi.config.backup.path}/backup_information.yml", - "w+") do |file| - file << s.to_yaml.gsub(/^---\n/,'') - end - - FileUtils.chmod(0700, ["db", "builds"]) - - # create archive - $progress.print "Creating backup archive: #{tar_file} ... " - orig_umask = File.umask(0077) - if Kernel.system('tar', '-czf', tar_file, *backup_contents) - $progress.puts "done".green - else - puts "creating archive #{tar_file} failed".red - abort 'Backup failed' - end - File.umask(orig_umask) - - upload(tar_file) - end - end - - def upload(tar_file) - remote_directory = GitlabCi.config.backup.upload.remote_directory - $progress.print "Uploading backup archive to remote storage #{remote_directory} ... " - - connection_settings = GitlabCi.config.backup.upload.connection - if connection_settings.blank? - $progress.puts "skipped".yellow - return - end - - connection = ::Fog::Storage.new(connection_settings) - directory = connection.directories.get(remote_directory) - - if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, - multipart_chunk_size: GitlabCi.config.backup.upload.multipart_chunk_size) - $progress.puts "done".green - else - puts "uploading backup to #{remote_directory} failed".red - abort 'Backup failed' - end - end - - def cleanup - $progress.print "Deleting tmp directories ... " - - backup_contents.each do |dir| - next unless File.exist?(File.join(GitlabCi.config.backup.path, dir)) - - if FileUtils.rm_rf(File.join(GitlabCi.config.backup.path, dir)) - $progress.puts "done".green - else - puts "deleting tmp directory '#{dir}' failed".red - abort 'Backup failed' - end - end - end - - def remove_old - # delete backups - $progress.print "Deleting old backups ... " - keep_time = GitlabCi.config.backup.keep_time.to_i - - if keep_time > 0 - removed = 0 - - Dir.chdir(GitlabCi.config.backup.path) do - file_list = Dir.glob('*_gitlab_ci_backup.tar.gz') - file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar.gz/ } - file_list.sort.each do |timestamp| - if Time.at(timestamp) < (Time.now - keep_time) - if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar.gz)) - removed += 1 - end - end - end - end - - $progress.puts "done. (#{removed} removed)".green - else - $progress.puts "skipping".yellow - end - end - - def unpack - Dir.chdir(GitlabCi.config.backup.path) - - # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_ci_backup.tar.gz").each.map { |f| f.split(/_/).first.to_i } - puts "no backups found" if file_list.count == 0 - - if file_list.count > 1 && ENV["BACKUP"].nil? - puts "Found more than one backup, please specify which one you want to restore:" - puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" - exit 1 - end - - tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar.gz") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar.gz") - - unless File.exists?(tar_file) - puts "The specified backup doesn't exist!" - exit 1 - end - - $progress.print "Unpacking backup ... " - - unless Kernel.system(*%W(tar -xzf #{tar_file})) - puts "unpacking backup failed".red - exit 1 - else - $progress.puts "done".green - end - - ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 - - # restoring mismatching backups can lead to unexpected problems - if settings[:gitlab_version] != GitlabCi::VERSION - puts "GitLab CI version mismatch:".red - puts " Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI version in the backup!".red - puts " Please switch to the following version and try again:".red - puts " version: #{settings[:gitlab_version]}".red - puts - puts "Hint: git checkout v#{settings[:gitlab_version]}" - exit 1 - end - end - - def tar_version - tar_version = `tar --version` - tar_version.force_encoding('locale').split("\n").first - end - - private - - def backup_contents - ["db", "builds", "backup_information.yml"] - end - - def settings - @settings ||= YAML.load_file("backup_information.yml") - end - end - end -end diff --git a/lib/tasks/ci/backup.rake b/lib/tasks/ci/backup.rake deleted file mode 100644 index 1cb2e43f875..00000000000 --- a/lib/tasks/ci/backup.rake +++ /dev/null @@ -1,62 +0,0 @@ -namespace :ci do - namespace :backup do - - desc "GITLAB | Create a backup of the GitLab CI database" - task create: :environment do - configure_cron_mode - - $progress.puts "Dumping database ... ".blue - Ci::Backup::Database.new.dump - $progress.puts "done".green - - $progress.puts "Dumping builds ... ".blue - Ci::Backup::Builds.new.dump - $progress.puts "done".green - - backup = Ci::Backup::Manager.new - backup.pack - backup.cleanup - backup.remove_old - end - - desc "GITLAB | Restore a previously created backup" - task restore: :environment do - configure_cron_mode - - backup = Ci::Backup::Manager.new - backup.unpack - - $progress.puts "Restoring database ... ".blue - Ci::Backup::Database.new.restore - $progress.puts "done".green - - $progress.puts "Restoring builds ... ".blue - Ci::Backup::Builds.new.restore - $progress.puts "done".green - - backup.cleanup - end - - def configure_cron_mode - if ENV['CRON'] - # We need an object we can say 'puts' and 'print' to; let's use a - # StringIO. - require 'stringio' - $progress = StringIO.new - else - $progress = $stdout - end - end - end - - # Disable colors for CRON - unless STDOUT.isatty - module Colored - extend self - - def colorize(string, options={}) - string - end - end - end -end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 4c73f90bbf2..f20c7f71ba5 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -11,6 +11,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:db:create"].invoke Rake::Task["gitlab:backup:repo:create"].invoke Rake::Task["gitlab:backup:uploads:create"].invoke + Rake::Task["gitlab:backup:builds:create"].invoke backup = Backup::Manager.new backup.pack @@ -30,6 +31,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:db:restore"].invoke unless backup.skipped?("db") Rake::Task["gitlab:backup:repo:restore"].invoke unless backup.skipped?("repositories") Rake::Task["gitlab:backup:uploads:restore"].invoke unless backup.skipped?("uploads") + Rake::Task["gitlab:backup:builds:restore"].invoke unless backup.skipped?("builds") Rake::Task["gitlab:shell:setup"].invoke backup.cleanup @@ -73,6 +75,25 @@ namespace :gitlab do end end + namespace :builds do + task create: :environment do + $progress.puts "Dumping builds ... ".blue + + if ENV["SKIP"] && ENV["SKIP"].include?("builds") + $progress.puts "[SKIPPED]".cyan + else + Backup::Builds.new.dump + $progress.puts "done".green + end + end + + task restore: :environment do + $progress.puts "Restoring builds ... ".blue + Backup::Builds.new.restore + $progress.puts "done".green + end + end + namespace :uploads do task create: :environment do $progress.puts "Dumping uploads ... ".blue diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 23f322e0a62..32adcbec71f 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -54,6 +54,7 @@ describe 'gitlab:app namespace rake task' do and_return({ gitlab_version: gitlab_version }) expect(Rake::Task["gitlab:backup:db:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:backup:repo:restore"]).to receive(:invoke) + expect(Rake::Task["gitlab:backup:builds:restore"]).to receive(:invoke) expect(Rake::Task["gitlab:shell:setup"]).to receive(:invoke) expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error end @@ -117,6 +118,7 @@ describe 'gitlab:app namespace rake task' do expect(tar_contents).to match('db/') expect(tar_contents).to match('uploads/') expect(tar_contents).to match('repositories/') + expect(tar_contents).to match('builds/') expect(tar_contents).not_to match(/^.{4,9}[rwx].* (db|uploads|repositories)\/$/) end @@ -163,6 +165,7 @@ describe 'gitlab:app namespace rake task' do expect(tar_contents).to match('db/') expect(tar_contents).to match('uploads/') + expect(tar_contents).to match('builds/') expect(tar_contents).not_to match('repositories/') end @@ -173,6 +176,7 @@ describe 'gitlab:app namespace rake task' do expect(Rake::Task["gitlab:backup:db:restore"]).to receive :invoke expect(Rake::Task["gitlab:backup:repo:restore"]).not_to receive :invoke + expect(Rake::Task["gitlab:backup:builds:restore"]).to receive :invoke expect(Rake::Task["gitlab:shell:setup"]).to receive :invoke expect { run_rake_task('gitlab:backup:restore') }.not_to raise_error end -- cgit v1.2.1