Merge branch 'ac-package-storage-stats-ee' into 'master'

track packages_file in ProjectStatistics

Closes gitlab-ce#59303 and #10575

See merge request gitlab-org/gitlab-ee!11020

6 files changed, 88 insertions, 1 deletions

diff --git a/db/schema.rb b/db/schema.rb
index e90a14e2c98..48a7f9f834a 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -2460,9 +2460,9 @@ ActiveRecord::Schema.define(version: 20190426180107) do
     t.bigint "repository_size", default: 0, null: false
     t.bigint "lfs_objects_size", default: 0, null: false
     t.bigint "build_artifacts_size", default: 0, null: false
-    t.bigint "packages_size"
     t.bigint "shared_runners_seconds", default: 0, null: false
     t.datetime "shared_runners_seconds_last_reset"
+    t.bigint "packages_size"
     t.index ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
     t.index ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
   end
diff --git a/ee/app/models/packages/package_file.rb b/ee/app/models/packages/package_file.rb
index 74d5aad236d..299bb424a88 100644
--- a/ee/app/models/packages/package_file.rb
+++ b/ee/app/models/packages/package_file.rb
@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 class Packages::PackageFile < ApplicationRecord
+  include UpdateProjectStatistics
+
+  delegate :project, :project_id, to: :package
+  update_project_statistics stat: :packages_size
+
   belongs_to :package
 
   validates :package, presence: true
diff --git a/ee/changelogs/unreleased/ac-package-storage-stats-ee.yml b/ee/changelogs/unreleased/ac-package-storage-stats-ee.yml
new file mode 100644
index 00000000000..c91c965543b
--- /dev/null
+++ b/ee/changelogs/unreleased/ac-package-storage-stats-ee.yml
@@ -0,0 +1,5 @@
+---
+title: Keep track of packages_file in ProjectStatistics
+merge_request: 11020
+author:
+type: added
diff --git a/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb b/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb
new file mode 100644
index 00000000000..994fb4dd04c
--- /dev/null
+++ b/ee/db/post_migrate/20190418132125_populate_project_statistics_packages_size.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class PopulateProjectStatisticsPackagesSize < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+
+  def up
+    stats_ids = ProjectStatistics.joins(project: { packages: :package_files }).distinct.select(:id)
+
+    packages_size = Arel.sql(
+      '(SELECT SUM(size) FROM packages_package_files ' \
+      'JOIN packages_packages ON packages_packages.id = packages_package_files.package_id ' \
+      'WHERE packages_packages.project_id = project_statistics.project_id)'
+    )
+    update_column_in_batches(:project_statistics, :packages_size, packages_size) do |table, query|
+      query.where(table[:id].in(stats_ids))
+    end
+
+    storage_size = Arel.sql('(repository_size + lfs_objects_size + build_artifacts_size + COALESCE(packages_size, 0))')
+    update_column_in_batches(:project_statistics, :storage_size, storage_size) do |table, query|
+      query.where(table[:id].in(stats_ids))
+    end
+  end
+
+  def down
+    storage_size = Arel.sql('(repository_size + lfs_objects_size + build_artifacts_size)')
+    update_column_in_batches(:project_statistics, :storage_size, storage_size) do |table, query|
+      query.where(table[:packages_size].gt(0))
+    end
+
+    update_column_in_batches(:project_statistics, :packages_size, nil)
+  end
+end
diff --git a/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb b/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb
new file mode 100644
index 00000000000..9d9ce146585
--- /dev/null
+++ b/ee/spec/migrations/populate_project_statistics_packages_size_spec.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('ee', 'db', 'post_migrate', '20190418132125_populate_project_statistics_packages_size.rb')
+
+describe PopulateProjectStatisticsPackagesSize, :migration do
+  let(:project_statistics) { table(:project_statistics) }
+  let(:namespaces)         { table(:namespaces) }
+  let(:projects)           { table(:projects) }
+  let(:packages)           { table(:packages_packages) }
+  let(:package_files)      { table(:packages_package_files) }
+
+  let(:file_size)      { 1.kilobyte }
+  let(:repo_size)      { 2.megabytes }
+  let(:lfs_size)       { 3.gigabytes }
+  let(:artifacts_size) { 4.terabytes }
+  let(:storage_size)   { repo_size + lfs_size + artifacts_size }
+
+  let(:namespace)  { namespaces.create(name: 'foo', path: 'foo') }
+  let(:package)    { packages.create!(project_id: project.id, name: 'a package', package_type: 1) }
+  let(:project)    { projects.create!(namespace_id: namespace.id) }
+
+  let!(:statistics) { project_statistics.create!(project_id: project.id, namespace_id: namespace.id, storage_size: storage_size, repository_size: repo_size, lfs_objects_size: lfs_size, build_artifacts_size: artifacts_size) }
+  let!(:package_file) { package_files.create!(package_id: package.id, file: 'a file.txt', file_name: 'a file.txt', size: file_size)}
+
+  it 'backfills ProjectStatistics packages_size' do
+    expect { migrate! }
+      .to change { statistics.reload.packages_size }
+      .from(nil).to(file_size)
+  end
+
+  it 'updates ProjectStatistics storage_size' do
+    expect { migrate! }
+      .to change { statistics.reload.storage_size }
+      .by(file_size)
+  end
+end
diff --git a/ee/spec/models/packages/package_file_spec.rb b/ee/spec/models/packages/package_file_spec.rb
index a3167ef0b5e..79876338090 100644
--- a/ee/spec/models/packages/package_file_spec.rb
+++ b/ee/spec/models/packages/package_file_spec.rb
@@ -9,4 +9,8 @@ RSpec.describe Packages::PackageFile, type: :model do
   describe 'validations' do
     it { is_expected.to validate_presence_of(:package) }
   end
+
+  it_behaves_like 'UpdateProjectStatistics' do
+    subject { build(:package_file, :jar, size: 42) }
+  end
 end