summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--app/views/admin/application_settings/_form.html.haml2
-rw-r--r--app/views/ci/projects/disabled.html.haml2
-rw-r--r--lib/ci/migrate/builds.rb29
-rw-r--r--lib/ci/migrate/database.rb42
-rw-r--r--lib/ci/migrate/manager.rb72
-rw-r--r--lib/ci/migrate/tags.rb55
-rw-r--r--lib/tasks/ci/migrate.rake62
7 files changed, 191 insertions, 73 deletions
diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml
index 1476e29524c..143cd10c543 100644
--- a/app/views/admin/application_settings/_form.html.haml
+++ b/app/views/admin/application_settings/_form.html.haml
@@ -131,7 +131,7 @@
.checkbox
= f.label :ci_enabled do
= f.check_box :ci_enabled
- Enable Continuous Integration
+ Disable to prevent CI usage until rake ci:migrate is run (8.0 only)
.form-actions
= f.submit 'Save', class: 'btn btn-primary'
diff --git a/app/views/ci/projects/disabled.html.haml b/app/views/ci/projects/disabled.html.haml
index 95276d894ed..83b0d8329e1 100644
--- a/app/views/ci/projects/disabled.html.haml
+++ b/app/views/ci/projects/disabled.html.haml
@@ -1 +1 @@
-Continuous Integration has been disabled. Please ask your administrator to enable it.
+Continuous Integration has been disabled for time of the migration.
diff --git a/lib/ci/migrate/builds.rb b/lib/ci/migrate/builds.rb
new file mode 100644
index 00000000000..fdc143cfad5
--- /dev/null
+++ b/lib/ci/migrate/builds.rb
@@ -0,0 +1,29 @@
+module Ci
+ module Migrate
+ class Builds
+ attr_reader :app_builds_dir, :backup_builds_tarball, :backup_dir
+
+ def initialize
+ @app_builds_dir = Settings.gitlab_ci.builds_path
+ @backup_dir = Gitlab.config.backup.path
+ @backup_builds_tarball = File.join(backup_dir, 'builds/builds.tar.gz')
+ end
+
+ def restore
+ backup_existing_builds_dir
+
+ FileUtils.mkdir_p(app_builds_dir, mode: 0700)
+ unless system('tar', '-C', app_builds_dir, '-zxvf', backup_builds_tarball)
+ abort 'Restore failed'.red
+ end
+ end
+
+ def backup_existing_builds_dir
+ timestamped_builds_path = File.join(app_builds_dir, '..', "builds.#{Time.now.to_i}")
+ if File.exists?(app_builds_dir)
+ FileUtils.mv(app_builds_dir, File.expand_path(timestamped_builds_path))
+ end
+ end
+ end
+ end
+end
diff --git a/lib/ci/migrate/database.rb b/lib/ci/migrate/database.rb
index 74f592dcaea..bf9b80f1f62 100644
--- a/lib/ci/migrate/database.rb
+++ b/lib/ci/migrate/database.rb
@@ -9,32 +9,32 @@ module Ci
@config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env]
end
- def restore(ci_dump)
- puts 'Deleting all CI related data ... '
- truncate_ci_tables
+ def restore
+ decompress_rd, decompress_wr = IO.pipe
+ decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name)
+ decompress_wr.close
- puts 'Restoring CI data ... '
- case config["adapter"]
- when /^mysql/ then
- print "Restoring MySQL database #{config['database']} ... "
- # Workaround warnings from MySQL 5.6 about passwords on cmd line
- ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
- system('mysql', *mysql_args, config['database'], in: ci_dump)
- when "postgresql" then
- puts "Restoring PostgreSQL database #{config['database']} ... "
- pg_env
- system('psql', config['database'], '-f', ci_dump)
- end
+ restore_pid = case config["adapter"]
+ when /^mysql/ then
+ $progress.print "Restoring MySQL database #{config['database']} ... "
+ # Workaround warnings from MySQL 5.6 about passwords on cmd line
+ ENV['MYSQL_PWD'] = config["password"].to_s if config["password"]
+ spawn('mysql', *mysql_args, config['database'], in: decompress_rd)
+ when "postgresql" then
+ $progress.print "Restoring PostgreSQL database #{config['database']} ... "
+ pg_env
+ spawn('psql', config['database'], in: decompress_rd)
+ end
+ decompress_rd.close
+
+ success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? }
+ abort 'Restore failed' unless success
end
protected
- def truncate_ci_tables
- c = ActiveRecord::Base.connection
- c.tables.select { |t| t.start_with?('ci_') }.each do |table|
- puts "Deleting data from #{table}..."
- c.execute("DELETE FROM #{table}")
- end
+ def db_file_name
+ File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz')
end
def mysql_args
diff --git a/lib/ci/migrate/manager.rb b/lib/ci/migrate/manager.rb
new file mode 100644
index 00000000000..9405397031c
--- /dev/null
+++ b/lib/ci/migrate/manager.rb
@@ -0,0 +1,72 @@
+module Ci
+ module Migrate
+ class Manager
+ VERSION = '8.0.0.pre'
+
+ def cleanup
+ $progress.print "Deleting tmp directories ... "
+
+ backup_contents.each do |dir|
+ next unless File.exist?(File.join(Gitlab.config.backup.path, dir))
+
+ if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir))
+ $progress.puts "done".green
+ else
+ puts "deleting tmp directory '#{dir}' failed".red
+ abort 'Backup failed'
+ end
+ end
+ end
+
+ def unpack
+ Dir.chdir(Gitlab.config.backup.path)
+
+ # check for existing backups in the backup dir
+ file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
+ puts "no backups found" if file_list.count == 0
+
+ if file_list.count > 1 && ENV["BACKUP"].nil?
+ puts "Found more than one backup, please specify which one you want to restore:"
+ puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+ exit 1
+ end
+
+ tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
+
+ unless File.exists?(tar_file)
+ puts "The specified CI backup doesn't exist!"
+ exit 1
+ end
+
+ $progress.print "Unpacking backup ... "
+
+ unless Kernel.system(*%W(tar -xf #{tar_file}))
+ puts "unpacking backup failed".red
+ exit 1
+ else
+ $progress.puts "done".green
+ end
+
+ ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
+
+ # restoring mismatching backups can lead to unexpected problems
+ if settings[:gitlab_version] != VERSION
+ puts "GitLab CI version mismatch:".red
+ puts " Your current GitLab CI version (#{VERSION}) differs from the GitLab CI (#{settings[:gitlab_version]}) version in the backup!".red
+ exit 1
+ end
+ end
+
+ private
+
+ def backup_contents
+ ["db", "builds", "backup_information.yml"]
+ end
+
+ def settings
+ @settings ||= YAML.load_file("backup_information.yml")
+ end
+ end
+ end
+end
+
diff --git a/lib/ci/migrate/tags.rb b/lib/ci/migrate/tags.rb
index 125a535e9a9..97e043ece27 100644
--- a/lib/ci/migrate/tags.rb
+++ b/lib/ci/migrate/tags.rb
@@ -4,45 +4,38 @@ module Ci
module Migrate
class Tags
def restore
- puts 'Migrating tags for Runners... '
- list_objects('Runner').each do |id|
- putc '.'
- runner = Ci::Runner.find_by_id(id)
- if runner
- tags = list_tags('Runner', id)
- runner.update_attributes(tag_list: tags)
+ puts 'Inserting tags...'
+ connection.select_all('SELECT ci_tags.name FROM ci_tags').each do |tag|
+ begin
+ connection.execute("INSERT INTO tags (name) VALUES(#{ActiveRecord::Base::sanitize(tag['name'])})")
+ rescue ActiveRecord::RecordNotUnique
end
end
- puts ''
- puts 'Migrating tags for Builds... '
- list_objects('Build').each do |id|
- putc '.'
- build = Ci::Build.find_by_id(id)
- if build
- tags = list_tags('Build', id)
- build.update_attributes(tag_list: tags)
- end
+ ActiveRecord::Base.transaction do
+ puts 'Deleting old taggings...'
+ connection.execute "DELETE FROM taggings WHERE context = 'tags' AND taggable_type LIKE 'Ci::%'"
+
+ puts 'Inserting taggings...'
+ connection.execute(
+ 'INSERT INTO taggings (taggable_type, taggable_id, tag_id, context) ' +
+ "SELECT CONCAT('Ci::', ci_taggings.taggable_type), ci_taggings.taggable_id, tags.id, 'tags' FROM ci_taggings " +
+ 'JOIN ci_tags ON ci_tags.id = ci_taggings.tag_id ' +
+ 'JOIN tags ON tags.name = ci_tags.name '
+ )
+
+ puts 'Resetting counters... '
+ connection.execute(
+ 'UPDATE tags SET ' +
+ 'taggings_count = (SELECT COUNT(*) FROM taggings WHERE tags.id = taggings.tag_id)'
+ )
end
- puts ''
end
protected
- def list_objects(type)
- ids = ActiveRecord::Base.connection.select_all(
- "select distinct taggable_id from ci_taggings where taggable_type = #{ActiveRecord::Base::sanitize(type)}"
- )
- ids.map { |id| id['taggable_id'] }
- end
-
- def list_tags(type, id)
- tags = ActiveRecord::Base.connection.select_all(
- 'select ci_tags.name from ci_tags ' +
- 'join ci_taggings on ci_tags.id = ci_taggings.tag_id ' +
- "where taggable_type = #{ActiveRecord::Base::sanitize(type)} and taggable_id = #{ActiveRecord::Base::sanitize(id)} and context = 'tags'"
- )
- tags.map { |tag| tag['name'] }
+ def connection
+ ActiveRecord::Base.connection
end
end
end
diff --git a/lib/tasks/ci/migrate.rake b/lib/tasks/ci/migrate.rake
index e7d41874a11..1de664c85e1 100644
--- a/lib/tasks/ci/migrate.rake
+++ b/lib/tasks/ci/migrate.rake
@@ -1,40 +1,56 @@
namespace :ci do
desc 'GitLab | Import and migrate CI database'
task migrate: :environment do
+ warn_user_is_not_gitlab
+ configure_cron_mode
+
unless ENV['force'] == 'yes'
- puts "This will truncate all CI tables and restore it from provided backup."
- puts "You will lose any previous CI data stored in the database."
+ puts 'This will remove all CI related data and restore it from the provided backup.'
ask_to_continue
- puts ""
+ puts ''
end
- Rake::Task["ci:migrate:db"].invoke
- Rake::Task["ci:migrate:autoincrements"].invoke
- Rake::Task["ci:migrate:tags"].invoke
- Rake::Task["ci:migrate:services"].invoke
+ # disable CI for time of migration
+ enable_ci(false)
+
+ # unpack archives
+ migrate = Ci::Migrate::Manager.new
+ migrate.unpack
+
+ Rake::Task['ci:migrate:db'].invoke
+ Rake::Task['ci:migrate:builds'].invoke
+ Rake::Task['ci:migrate:tags'].invoke
+ Rake::Task['ci:migrate:services'].invoke
+
+ # enable CI for time of migration
+ enable_ci(true)
+
+ migrate.cleanup
end
namespace :migrate do
desc 'GitLab | Import CI database'
task db: :environment do
- if ENV["CI_DUMP"].nil?
- puts "No CI SQL dump specified:"
- puts "rake gitlab:backup:restore CI_DUMP=ci_dump.sql"
- exit 1
- end
-
- ci_dump = ENV["CI_DUMP"]
- unless File.exists?(ci_dump)
- puts "The specified sql dump doesn't exist!"
- exit 1
- end
+ configure_cron_mode
+ $progress.puts 'Restoring database ... '.blue
+ Ci::Migrate::Database.new.restore
+ $progress.puts 'done'.green
+ end
- ::Ci::Migrate::Database.new.restore(ci_dump)
+ desc 'GitLab | Import CI builds'
+ task builds: :environment do
+ configure_cron_mode
+ $progress.puts 'Restoring builds ... '.blue
+ Ci::Migrate::Builds.new.restore
+ $progress.puts 'done'.green
end
desc 'GitLab | Migrate CI tags'
task tags: :environment do
+ configure_cron_mode
+ $progress.puts 'Migrating tags ... '.blue
::Ci::Migrate::Tags.new.restore
+ $progress.puts 'done'.green
end
desc 'GitLab | Migrate CI auto-increments'
@@ -56,8 +72,16 @@ namespace :ci do
desc 'GitLab | Migrate CI services'
task services: :environment do
+ $progress.puts 'Migrating services ... '.blue
c = ActiveRecord::Base.connection
c.execute("UPDATE ci_services SET type=CONCAT('Ci::', type) WHERE type NOT LIKE 'Ci::%'")
+ $progress.puts 'done'.green
end
end
+
+ def enable_ci(enabled)
+ settings = ApplicationSetting.current || ApplicationSetting.create_from_defaults
+ settings.ci_enabled = enabled
+ settings.save!
+ end
end
View Lorry Controller Status