From a5520fa5d64abf96407fd473c5caea969687ebe9 Mon Sep 17 00:00:00 2001 From: Alessio Caiazza Date: Thu, 18 Apr 2019 16:16:51 +0200 Subject: Backfill ProjectStatisics packages_size --- ...25_populate_project_statistics_packages_size.rb | 36 +++++++++++++++++++++ ...pulate_project_statistics_packages_size_spec.rb | 37 ++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb create mode 100644 ee/spec/migrations/populate_project_statistics_packages_size_spec.rb diff --git a/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb b/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb new file mode 100644 index 00000000000..994fb4dd04c --- /dev/null +++ b/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class PopulateProjectStatisticsPackagesSize < ActiveRecord::Migration[5.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + stats_ids = ProjectStatistics.joins(project: { packages: :package_files }).distinct.select(:id) + + packages_size = Arel.sql( + '(SELECT SUM(size) FROM packages_package_files ' \ + 'JOIN packages_packages ON packages_packages.id = packages_package_files.package_id ' \ + 'WHERE packages_packages.project_id = project_statistics.project_id)' + ) + update_column_in_batches(:project_statistics, :packages_size, packages_size) do |table, query| + query.where(table[:id].in(stats_ids)) + end + + storage_size = Arel.sql('(repository_size + lfs_objects_size + build_artifacts_size + COALESCE(packages_size, 0))') + update_column_in_batches(:project_statistics, :storage_size, storage_size) do |table, query| + query.where(table[:id].in(stats_ids)) + end + end + + def down + storage_size = Arel.sql('(repository_size + lfs_objects_size + build_artifacts_size)') + update_column_in_batches(:project_statistics, :storage_size, storage_size) do |table, query| + query.where(table[:packages_size].gt(0)) + end + + update_column_in_batches(:project_statistics, :packages_size, nil) + end +end diff --git a/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb b/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb new file mode 100644 index 00000000000..9d9ce146585 --- /dev/null +++ b/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('ee', 'db', 'post_migrate', '20190418132125_populate_project_statistics_packages_size.rb') + +describe PopulateProjectStatisticsPackagesSize, :migration do + let(:project_statistics) { table(:project_statistics) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:packages) { table(:packages_packages) } + let(:package_files) { table(:packages_package_files) } + + let(:file_size) { 1.kilobyte } + let(:repo_size) { 2.megabytes } + let(:lfs_size) { 3.gigabytes } + let(:artifacts_size) { 4.terabytes } + let(:storage_size) { repo_size + lfs_size + artifacts_size } + + let(:namespace) { namespaces.create(name: 'foo', path: 'foo') } + let(:package) { packages.create!(project_id: project.id, name: 'a package', package_type: 1) } + let(:project) { projects.create!(namespace_id: namespace.id) } + + let!(:statistics) { project_statistics.create!(project_id: project.id, namespace_id: namespace.id, storage_size: storage_size, repository_size: repo_size, lfs_objects_size: lfs_size, build_artifacts_size: artifacts_size) } + let!(:package_file) { package_files.create!(package_id: package.id, file: 'a file.txt', file_name: 'a file.txt', size: file_size)} + + it 'backfills ProjectStatistics packages_size' do + expect { migrate! } + .to change { statistics.reload.packages_size } + .from(nil).to(file_size) + end + + it 'updates ProjectStatistics storage_size' do + expect { migrate! } + .to change { statistics.reload.storage_size } + .by(file_size) + end +end -- cgit v1.2.1