From cb72d9efd2db30cd2df1d3bad445ffb00e5b3c0f Mon Sep 17 00:00:00 2001 From: Alessio Caiazza Date: Mon, 15 Apr 2019 13:51:30 +0200 Subject: Add packages_size to ProjectStatistics This new field will allow to keep track of the storage used by the packages features, it provides also aggregation at namespace level. --- db/schema.rb | 1 + spec/models/project_statistics_spec.rb | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index e90a14e2c98..4ffeb32e6b7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -2463,6 +2463,7 @@ ActiveRecord::Schema.define(version: 20190426180107) do t.bigint "packages_size" t.bigint "shared_runners_seconds", default: 0, null: false t.datetime "shared_runners_seconds_last_reset" + t.bigint "packages_size" t.index ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree t.index ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree end diff --git a/spec/models/project_statistics_spec.rb b/spec/models/project_statistics_spec.rb index 738398a06f9..914f0c2394c 100644 --- a/spec/models/project_statistics_spec.rb +++ b/spec/models/project_statistics_spec.rb @@ -124,7 +124,7 @@ describe ProjectStatistics do end describe '.increment_statistic' do - shared_examples 'a statistic that increases storage_size' do + shared_examples 'a statistic that also increases storage_size' do it 'increases the statistic by that amount' do expect { described_class.increment_statistic(project.id, stat, 13) } .to change { statistics.reload.send(stat) || 0 } @@ -153,7 +153,7 @@ describe ProjectStatistics do context 'when the amount is 0' do it 'does not execute a query' do project - expect { described_class.increment_statistic(project.id, :build_artifacts_size, 0) } + expect { described_class.increment_statistic(project.id, stat, 0) } .not_to exceed_query_limit(0) end end -- cgit v1.2.1 From 174f45980bde366eb3cbda0862e037e64a2846a6 Mon Sep 17 00:00:00 2001 From: Alessio Caiazza Date: Mon, 15 Apr 2019 18:09:31 +0200 Subject: Track package_file size --- ee/app/models/packages/package_file.rb | 5 +++++ ee/changelogs/unreleased/ac-package-storage-stats-ee.yml | 5 +++++ ee/spec/models/packages/package_file_spec.rb | 4 ++++ 3 files changed, 14 insertions(+) create mode 100644 ee/changelogs/unreleased/ac-package-storage-stats-ee.yml diff --git a/ee/app/models/packages/package_file.rb b/ee/app/models/packages/package_file.rb index 74d5aad236d..299bb424a88 100644 --- a/ee/app/models/packages/package_file.rb +++ b/ee/app/models/packages/package_file.rb @@ -1,5 +1,10 @@ # frozen_string_literal: true class Packages::PackageFile < ApplicationRecord + include UpdateProjectStatistics + + delegate :project, :project_id, to: :package + update_project_statistics stat: :packages_size + belongs_to :package validates :package, presence: true diff --git a/ee/changelogs/unreleased/ac-package-storage-stats-ee.yml b/ee/changelogs/unreleased/ac-package-storage-stats-ee.yml new file mode 100644 index 00000000000..c91c965543b --- /dev/null +++ b/ee/changelogs/unreleased/ac-package-storage-stats-ee.yml @@ -0,0 +1,5 @@ +--- +title: Keep track of packages_file in ProjectStatistics +merge_request: 11020 +author: +type: added diff --git a/ee/spec/models/packages/package_file_spec.rb b/ee/spec/models/packages/package_file_spec.rb index a3167ef0b5e..79876338090 100644 --- a/ee/spec/models/packages/package_file_spec.rb +++ b/ee/spec/models/packages/package_file_spec.rb @@ -9,4 +9,8 @@ RSpec.describe Packages::PackageFile, type: :model do describe 'validations' do it { is_expected.to validate_presence_of(:package) } end + + it_behaves_like 'UpdateProjectStatistics' do + subject { build(:package_file, :jar, size: 42) } + end end -- cgit v1.2.1 From a5520fa5d64abf96407fd473c5caea969687ebe9 Mon Sep 17 00:00:00 2001 From: Alessio Caiazza Date: Thu, 18 Apr 2019 16:16:51 +0200 Subject: Backfill ProjectStatisics packages_size --- ...25_populate_project_statistics_packages_size.rb | 36 +++++++++++++++++++++ ...pulate_project_statistics_packages_size_spec.rb | 37 ++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb create mode 100644 ee/spec/migrations/populate_project_statistics_packages_size_spec.rb diff --git a/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb b/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb new file mode 100644 index 00000000000..994fb4dd04c --- /dev/null +++ b/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class PopulateProjectStatisticsPackagesSize < ActiveRecord::Migration[5.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + stats_ids = ProjectStatistics.joins(project: { packages: :package_files }).distinct.select(:id) + + packages_size = Arel.sql( + '(SELECT SUM(size) FROM packages_package_files ' \ + 'JOIN packages_packages ON packages_packages.id = packages_package_files.package_id ' \ + 'WHERE packages_packages.project_id = project_statistics.project_id)' + ) + update_column_in_batches(:project_statistics, :packages_size, packages_size) do |table, query| + query.where(table[:id].in(stats_ids)) + end + + storage_size = Arel.sql('(repository_size + lfs_objects_size + build_artifacts_size + COALESCE(packages_size, 0))') + update_column_in_batches(:project_statistics, :storage_size, storage_size) do |table, query| + query.where(table[:id].in(stats_ids)) + end + end + + def down + storage_size = Arel.sql('(repository_size + lfs_objects_size + build_artifacts_size)') + update_column_in_batches(:project_statistics, :storage_size, storage_size) do |table, query| + query.where(table[:packages_size].gt(0)) + end + + update_column_in_batches(:project_statistics, :packages_size, nil) + end +end diff --git a/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb b/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb new file mode 100644 index 00000000000..9d9ce146585 --- /dev/null +++ b/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('ee', 'db', 'post_migrate', '20190418132125_populate_project_statistics_packages_size.rb') + +describe PopulateProjectStatisticsPackagesSize, :migration do + let(:project_statistics) { table(:project_statistics) } + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:packages) { table(:packages_packages) } + let(:package_files) { table(:packages_package_files) } + + let(:file_size) { 1.kilobyte } + let(:repo_size) { 2.megabytes } + let(:lfs_size) { 3.gigabytes } + let(:artifacts_size) { 4.terabytes } + let(:storage_size) { repo_size + lfs_size + artifacts_size } + + let(:namespace) { namespaces.create(name: 'foo', path: 'foo') } + let(:package) { packages.create!(project_id: project.id, name: 'a package', package_type: 1) } + let(:project) { projects.create!(namespace_id: namespace.id) } + + let!(:statistics) { project_statistics.create!(project_id: project.id, namespace_id: namespace.id, storage_size: storage_size, repository_size: repo_size, lfs_objects_size: lfs_size, build_artifacts_size: artifacts_size) } + let!(:package_file) { package_files.create!(package_id: package.id, file: 'a file.txt', file_name: 'a file.txt', size: file_size)} + + it 'backfills ProjectStatistics packages_size' do + expect { migrate! } + .to change { statistics.reload.packages_size } + .from(nil).to(file_size) + end + + it 'updates ProjectStatistics storage_size' do + expect { migrate! } + .to change { statistics.reload.storage_size } + .by(file_size) + end +end -- cgit v1.2.1 From dd03e745572dcbfafbff198fd2ef813fb65f3420 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 2 May 2019 14:27:24 -0700 Subject: Fix typo caused by bad conflict resolution --- spec/models/project_statistics_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/project_statistics_spec.rb b/spec/models/project_statistics_spec.rb index 914f0c2394c..d25a417d72d 100644 --- a/spec/models/project_statistics_spec.rb +++ b/spec/models/project_statistics_spec.rb @@ -124,7 +124,7 @@ describe ProjectStatistics do end describe '.increment_statistic' do - shared_examples 'a statistic that also increases storage_size' do + shared_examples 'a statistic that increases storage_size' do it 'increases the statistic by that amount' do expect { described_class.increment_statistic(project.id, stat, 13) } .to change { statistics.reload.send(stat) || 0 } -- cgit v1.2.1 From 8e680d1746d197ce237cc86741e362fafb31c154 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 2 May 2019 17:13:47 -0700 Subject: Fix merge error in project_statistics_spec.rb --- spec/models/project_statistics_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/project_statistics_spec.rb b/spec/models/project_statistics_spec.rb index d25a417d72d..738398a06f9 100644 --- a/spec/models/project_statistics_spec.rb +++ b/spec/models/project_statistics_spec.rb @@ -153,7 +153,7 @@ describe ProjectStatistics do context 'when the amount is 0' do it 'does not execute a query' do project - expect { described_class.increment_statistic(project.id, stat, 0) } + expect { described_class.increment_statistic(project.id, :build_artifacts_size, 0) } .not_to exceed_query_limit(0) end end -- cgit v1.2.1 From a1e31c8be3fcf66ca995dcdeca6f90742ddeeb1f Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 2 May 2019 17:15:41 -0700 Subject: Remove redundant packages_size entry --- db/schema.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/db/schema.rb b/db/schema.rb index 4ffeb32e6b7..48a7f9f834a 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -2460,7 +2460,6 @@ ActiveRecord::Schema.define(version: 20190426180107) do t.bigint "repository_size", default: 0, null: false t.bigint "lfs_objects_size", default: 0, null: false t.bigint "build_artifacts_size", default: 0, null: false - t.bigint "packages_size" t.bigint "shared_runners_seconds", default: 0, null: false t.datetime "shared_runners_seconds_last_reset" t.bigint "packages_size" -- cgit v1.2.1