Add traces and artifacts migrationmigrate-old-traces-and-artifacts

3 files changed, 159 insertions, 1 deletions

diff --git a/db/migrate/20170523083112_migrate_old_artifacts.rb b/db/migrate/20170523083112_migrate_old_artifacts.rb
new file mode 100644
index 00000000000..e030c400ac9
--- /dev/null
+++ b/db/migrate/20170523083112_migrate_old_artifacts.rb
@@ -0,0 +1,79 @@
+require 'work_queue'
+
+class MigrateOldArtifacts < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    builds_with_artifacts(min_id || 0).each do |build|
+      work_queue.enqueue_b do
+        migrate_artifacts(build)
+      end
+    end
+
+  ensure
+    work_queue.join
+  end
+
+  def down
+  end
+
+  private
+
+  def migrate_artifacts(build)
+    source = source_artifacts(build)
+    target = target_artifacts(build)
+    return unless File.exists?(source)
+
+    ensure_path(target)
+    FileUtils.move(source, target)
+  end
+
+  def ensure_path(path)
+    directory = File.dirname(path)
+    unless Dir.exist?(default_directory)
+      FileUtils.mkdir_p(directory)
+    end
+  end
+
+  def source_artifacts(build)
+    File.join(Gitlab.config.artifacts.path,
+      build['created_at'].utc.strftime('%Y_%m'),
+      build['ci_id'].to_s, 
+      build['id'].to_s)
+  end
+
+  def target_artifacts(build)
+    File.join(Gitlab.config.artifacts.path,
+      build['created_at'].utc.strftime('%Y_%m'),
+      build['project_id'].to_s, 
+      build['id'].to_s)
+  end
+
+  def min_id
+    select_value <<-SQL.strip_heredoc
+      SELECT min(ci_builds.id) FROM ci_builds
+        JOIN projects ON ci_builds.project_id = projects.id 
+         AND projects.ci_id IS NULL
+    SQL
+  end
+
+  def builds_with_artifacts(before_id)
+    select_all <<-SQL.strip_heredoc
+      SELECT ci_builds.id, ci_builds.project_id,
+             projects.ci_id, ci_builds.created_at
+        FROM ci_builds
+        JOIN projects ON ci_builds.project_id = projects.id 
+       WHERE ci_builds.id < #{before_id}
+         AND ci_builds.artifacts_file NOT IN (NULL, '')
+         AND projects.ci_id IS NOT NULL
+    SQL
+  end
+
+  def work_queue
+    @work_queue ||= WorkQueue.new(5)
+  end
+end
diff --git a/db/migrate/20170523085324_migrate_old_traces.rb b/db/migrate/20170523085324_migrate_old_traces.rb
new file mode 100644
index 00000000000..473a1bebf46
--- /dev/null
+++ b/db/migrate/20170523085324_migrate_old_traces.rb
@@ -0,0 +1,79 @@
+require 'work_queue'
+
+class MigrateOldTraces < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    builds_with_traces(min_id || 0).each do |build|
+      work_queue.enqueue_b do
+        migrate_trace(build)
+      end
+    end
+
+  ensure
+    work_queue.join
+  end
+
+  def down
+  end
+
+  private
+
+  def migrate_trace(build)
+    source = source_trace(build)
+    target = target_trace(build)
+    return unless File.exists?(source)
+
+    ensure_path(target)
+    FileUtils.move(source, target)
+  end
+
+  def ensure_path(path)
+    directory = File.dirname(path)
+    unless Dir.exist?(default_directory)
+      FileUtils.mkdir_p(directory)
+    end
+  end
+
+  def source_trace(build)
+    File.join(Gitlab.config.gitlab_ci.builds_path,
+      build['created_at'].utc.strftime('%Y_%m'),
+      build['ci_id'].to_s, 
+      build['id'].to_s + ".log")
+  end
+
+  def target_trace(build)
+    File.join(Gitlab.config.gitlab_ci.builds_path,
+      build['created_at'].utc.strftime('%Y_%m'),
+      build['project_id'].to_s, 
+      build['id'].to_s + ".log")
+  end
+
+  def min_id
+    select_value <<-SQL.strip_heredoc
+      SELECT min(ci_builds.id) as min_id FROM ci_builds
+        JOIN projects ON ci_builds.project_id = projects.id 
+         AND projects.ci_id IS NULL
+    SQL
+  end
+
+  def builds_with_traces(before_id)
+    select_all <<-SQL.strip_heredoc
+      SELECT ci_builds.id, ci_builds.project_id,
+             projects.ci_id, ci_builds.created_at
+        FROM ci_builds
+        JOIN projects ON ci_builds.project_id = projects.id 
+       WHERE ci_builds.id < #{before_id}
+         AND ci_builds.erased_at IS NULL
+         AND projects.ci_id IS NOT NULL
+    SQL
+  end
+
+  def work_queue
+    @work_queue ||= WorkQueue.new(10)
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index d14126401c9..3bcacd188ea 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20170518231126) do
+ActiveRecord::Schema.define(version: 20170523085324) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"