Backfill ProjectStatisics packages_size

2 files changed, 73 insertions, 0 deletions

diff --git a/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb b/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb
new file mode 100644
index 00000000000..994fb4dd04c
--- /dev/null
+++ b/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class PopulateProjectStatisticsPackagesSize < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    stats_ids = ProjectStatistics.joins(project: { packages: :package_files }).distinct.select(:id)
+
+    packages_size = Arel.sql(
+      '(SELECT SUM(size) FROM packages_package_files ' \
+      'JOIN packages_packages ON packages_packages.id = packages_package_files.package_id ' \
+      'WHERE packages_packages.project_id = project_statistics.project_id)'
+    )
+    update_column_in_batches(:project_statistics, :packages_size, packages_size) do |table, query|
+      query.where(table[:id].in(stats_ids))
+    end
+
+    storage_size = Arel.sql('(repository_size + lfs_objects_size + build_artifacts_size + COALESCE(packages_size, 0))')
+    update_column_in_batches(:project_statistics, :storage_size, storage_size) do |table, query|
+      query.where(table[:id].in(stats_ids))
+    end
+  end
+
+  def down
+    storage_size = Arel.sql('(repository_size + lfs_objects_size + build_artifacts_size)')
+    update_column_in_batches(:project_statistics, :storage_size, storage_size) do |table, query|
+      query.where(table[:packages_size].gt(0))
+    end
+
+    update_column_in_batches(:project_statistics, :packages_size, nil)
+  end
+end
diff --git a/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb b/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb
new file mode 100644
index 00000000000..9d9ce146585
--- /dev/null
+++ b/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('ee', 'db', 'post_migrate', '20190418132125_populate_project_statistics_packages_size.rb')
+
+describe PopulateProjectStatisticsPackagesSize, :migration do
+  let(:project_statistics) { table(:project_statistics) }
+  let(:namespaces)         { table(:namespaces) }
+  let(:projects)           { table(:projects) }
+  let(:packages)           { table(:packages_packages) }
+  let(:package_files)      { table(:packages_package_files) }
+
+  let(:file_size)      { 1.kilobyte }
+  let(:repo_size)      { 2.megabytes }
+  let(:lfs_size)       { 3.gigabytes }
+  let(:artifacts_size) { 4.terabytes }
+  let(:storage_size)   { repo_size + lfs_size + artifacts_size }
+
+  let(:namespace)  { namespaces.create(name: 'foo', path: 'foo') }
+  let(:package)    { packages.create!(project_id: project.id, name: 'a package', package_type: 1) }
+  let(:project)    { projects.create!(namespace_id: namespace.id) }
+
+  let!(:statistics) { project_statistics.create!(project_id: project.id, namespace_id: namespace.id, storage_size: storage_size, repository_size: repo_size, lfs_objects_size: lfs_size, build_artifacts_size: artifacts_size) }
+  let!(:package_file) { package_files.create!(package_id: package.id, file: 'a file.txt', file_name: 'a file.txt', size: file_size)}
+
+  it 'backfills ProjectStatistics packages_size' do
+    expect { migrate! }
+      .to change { statistics.reload.packages_size }
+      .from(nil).to(file_size)
+  end
+
+  it 'updates ProjectStatistics storage_size' do
+    expect { migrate! }
+      .to change { statistics.reload.storage_size }
+      .by(file_size)
+  end
+end