Merge branch 'feature/more-storage-statistics' into 'master'

Add more storage statistics

See merge request !7754

22 files changed, 164 insertions, 62 deletions

diff --git a/app/controllers/admin/groups_controller.rb b/app/controllers/admin/groups_controller.rb
index 1e3d194e9f9..61a3a03182a 100644
--- a/app/controllers/admin/groups_controller.rb
+++ b/app/controllers/admin/groups_controller.rb
@@ -1,17 +1,18 @@
 class Admin::GroupsController < Admin::ApplicationController
-  before_action :group, only: [:edit, :show, :update, :destroy, :project_update, :members_update]
+  before_action :group, only: [:edit, :update, :destroy, :project_update, :members_update]
 
   def index
-    @groups = Group.all
+    @groups = Group.with_statistics
     @groups = @groups.sort(@sort = params[:sort])
     @groups = @groups.search(params[:name]) if params[:name].present?
     @groups = @groups.page(params[:page])
   end
 
   def show
+    @group = Group.with_statistics.find_by_full_path(params[:id])
     @members = @group.members.order("access_level DESC").page(params[:members_page])
     @requesters = AccessRequestsFinder.new(@group).execute(current_user)
-    @projects = @group.projects.page(params[:projects_page])
+    @projects = @group.projects.with_statistics.page(params[:projects_page])
   end
 
   def new
diff --git a/app/controllers/admin/projects_controller.rb b/app/controllers/admin/projects_controller.rb
index 1d963bdf7d5..b09ae423096 100644
--- a/app/controllers/admin/projects_controller.rb
+++ b/app/controllers/admin/projects_controller.rb
@@ -3,7 +3,7 @@ class Admin::ProjectsController < Admin::ApplicationController
   before_action :group, only: [:show, :transfer]
 
   def index
-    @projects = Project.all
+    @projects = Project.with_statistics
     @projects = @projects.in_namespace(params[:namespace_id]) if params[:namespace_id].present?
     @projects = @projects.where(visibility_level: params[:visibility_level]) if params[:visibility_level].present?
     @projects = @projects.with_push if params[:with_push].present?
diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb
index efe9c001bcf..01c8fa2739f 100644
--- a/app/controllers/groups_controller.rb
+++ b/app/controllers/groups_controller.rb
@@ -75,7 +75,7 @@ class GroupsController < Groups::ApplicationController
   end
 
   def projects
-    @projects = @group.projects.page(params[:page])
+    @projects = @group.projects.with_statistics.page(params[:page])
   end
 
   def update
diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb
index 7445f3c113c..0f17f6d8b7e 100644
--- a/app/helpers/projects_helper.rb
+++ b/app/helpers/projects_helper.rb
@@ -246,11 +246,6 @@ module ProjectsHelper
     end
   end
 
-  def repository_size(project = @project)
-    size_in_bytes = project.repository_size * 1.megabyte
-    number_to_human_size(size_in_bytes, delimiter: ',', precision: 2)
-  end
-
   def default_url_to_repo(project = @project)
     case default_clone_protocol
     when 'ssh'
@@ -398,20 +393,6 @@ module ProjectsHelper
     [@project.path_with_namespace, sha, "readme"].join('-')
   end
 
-  def round_commit_count(project)
-    count = project.commit_count
-
-    if count > 10000
-      '10000+'
-    elsif count > 5000
-      '5000+'
-    elsif count > 1000
-      '1000+'
-    else
-      count
-    end
-  end
-
   def current_ref
     @ref || @repository.try(:root_ref)
   end
diff --git a/app/helpers/sorting_helper.rb b/app/helpers/sorting_helper.rb
index f03c4627050..ff787fb4131 100644
--- a/app/helpers/sorting_helper.rb
+++ b/app/helpers/sorting_helper.rb
@@ -11,6 +11,7 @@ module SortingHelper
       sort_value_due_date_soon => sort_title_due_date_soon,
       sort_value_due_date_later => sort_title_due_date_later,
       sort_value_largest_repo => sort_title_largest_repo,
+      sort_value_largest_group => sort_title_largest_group,
       sort_value_recently_signin => sort_title_recently_signin,
       sort_value_oldest_signin => sort_title_oldest_signin,
       sort_value_downvotes => sort_title_downvotes,
@@ -92,6 +93,10 @@ module SortingHelper
     'Largest repository'
   end
 
+  def sort_title_largest_group
+    'Largest group'
+  end
+
   def sort_title_recently_signin
     'Recent sign in'
   end
@@ -193,7 +198,11 @@ module SortingHelper
   end
 
   def sort_value_largest_repo
-    'repository_size_desc'
+    'storage_size_desc'
+  end
+
+  def sort_value_largest_group
+    'storage_size_desc'
   end
 
   def sort_value_recently_signin
diff --git a/app/helpers/storage_helper.rb b/app/helpers/storage_helper.rb
new file mode 100644
index 00000000000..e19c67a37ca
--- /dev/null
+++ b/app/helpers/storage_helper.rb
@@ -0,0 +1,7 @@
+module StorageHelper
+  def storage_counter(size_in_bytes)
+    precision = size_in_bytes < 1.megabyte ? 0 : 1
+
+    number_to_human_size(size_in_bytes, delimiter: ',', precision: precision, significant: false)
+  end
+end
diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index cb76cdf5981..27042798741 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -43,6 +43,8 @@ module Ci
     before_destroy { project }
 
     after_create :execute_hooks
+    after_save :update_project_statistics, if: :artifacts_size_changed?
+    after_destroy :update_project_statistics
 
     class << self
       def first_pending
@@ -584,5 +586,9 @@ module Ci
       Ci::MaskSecret.mask!(trace, token)
       trace
     end
+
+    def update_project_statistics
+      ProjectCacheWorker.perform_async(project_id, [], [:build_artifacts_size])
+    end
   end
 end
diff --git a/app/models/group.rb b/app/models/group.rb
index ac8a82c8c1e..85696ad9747 100644
--- a/app/models/group.rb
+++ b/app/models/group.rb
@@ -48,7 +48,13 @@ class Group < Namespace
     end
 
     def sort(method)
-      order_by(method)
+      if method == 'storage_size_desc'
+        # storage_size is a virtual column so we need to
+        # pass a string to avoid AR adding the table name
+        reorder('storage_size DESC, namespaces.id DESC')
+      else
+        order_by(method)
+      end
     end
 
     def reference_prefix
diff --git a/app/models/lfs_objects_project.rb b/app/models/lfs_objects_project.rb
index 0fd5f089db9..007eed5600a 100644
--- a/app/models/lfs_objects_project.rb
+++ b/app/models/lfs_objects_project.rb
@@ -5,4 +5,13 @@ class LfsObjectsProject < ActiveRecord::Base
   validates :lfs_object_id, presence: true
   validates :lfs_object_id, uniqueness: { scope: [:project_id], message: "already exists in project" }
   validates :project_id, presence: true
+
+  after_create :update_project_statistics
+  after_destroy :update_project_statistics
+
+  private
+
+  def update_project_statistics
+    ProjectCacheWorker.perform_async(project_id, [], [:lfs_objects_size])
+  end
 end
diff --git a/app/models/namespace.rb b/app/models/namespace.rb
index b52f08c7081..d41833de66f 100644
--- a/app/models/namespace.rb
+++ b/app/models/namespace.rb
@@ -9,6 +9,7 @@ class Namespace < ActiveRecord::Base
   cache_markdown_field :description, pipeline: :description
 
   has_many :projects, dependent: :destroy
+  has_many :project_statistics
   belongs_to :owner, class_name: "User"
 
   belongs_to :parent, class_name: "Namespace"
@@ -38,6 +39,18 @@ class Namespace < ActiveRecord::Base
 
   scope :root, -> { where('type IS NULL') }
 
+  scope :with_statistics, -> do
+    joins('LEFT JOIN project_statistics ps ON ps.namespace_id = namespaces.id')
+      .group('namespaces.id')
+      .select(
+        'namespaces.*',
+        'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
+        'COALESCE(SUM(ps.repository_size), 0) AS repository_size',
+        'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
+        'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
+      )
+  end
+
   class << self
     def by_path(path)
       find_by('lower(path) = :value', value: path.downcase)
diff --git a/app/models/project.rb b/app/models/project.rb
index 26fa20f856d..19bbe65b01d 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -44,6 +44,7 @@ class Project < ActiveRecord::Base
   after_create :ensure_dir_exist
   after_create :create_project_feature, unless: :project_feature
   after_save :ensure_dir_exist, if: :namespace_id_changed?
+  after_save :update_project_statistics, if: :namespace_id_changed?
 
   # set last_activity_at to the same as created_at
   after_create :set_last_activity_at
@@ -151,6 +152,7 @@ class Project < ActiveRecord::Base
 
   has_one :import_data, dependent: :destroy, class_name: "ProjectImportData"
   has_one :project_feature, dependent: :destroy
+  has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete
 
   has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id
   has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id
@@ -220,6 +222,7 @@ class Project < ActiveRecord::Base
   scope :with_push, -> { joins(:events).where('events.action = ?', Event::PUSHED) }
 
   scope :with_project_feature, -> { joins('LEFT JOIN project_features ON projects.id = project_features.project_id') }
+  scope :with_statistics, -> { includes(:statistics) }
 
   # "enabled" here means "not disabled". It includes private features!
   scope :with_feature_enabled, ->(feature) {
@@ -332,8 +335,10 @@ class Project < ActiveRecord::Base
     end
 
     def sort(method)
-      if method == 'repository_size_desc'
-        reorder(repository_size: :desc, id: :desc)
+      if method == 'storage_size_desc'
+        # storage_size is a joined column so we need to
+        # pass a string to avoid AR adding the table name
+        reorder('project_statistics.storage_size DESC, projects.id DESC')
       else
         order_by(method)
       end
@@ -1036,14 +1041,6 @@ class Project < ActiveRecord::Base
     forked? && project == forked_from_project
   end
 
-  def update_repository_size
-    update_attribute(:repository_size, repository.size)
-  end
-
-  def update_commit_count
-    update_attribute(:commit_count, repository.commit_count)
-  end
-
   def forks_count
     forks.count
   end
@@ -1322,4 +1319,9 @@ class Project < ActiveRecord::Base
   def full_path_changed?
     path_changed? || namespace_id_changed?
   end
+
+  def update_project_statistics
+    stats = statistics || build_statistics
+    stats.update(namespace_id: namespace_id)
+  end
 end
diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb
new file mode 100644
index 00000000000..2270ac75071
--- /dev/null
+++ b/app/models/project_statistics.rb
@@ -0,0 +1,43 @@
+class ProjectStatistics < ActiveRecord::Base
+  belongs_to :project
+  belongs_to :namespace
+
+  before_save :update_storage_size
+
+  STORAGE_COLUMNS = [:repository_size, :lfs_objects_size, :build_artifacts_size]
+  STATISTICS_COLUMNS = [:commit_count] + STORAGE_COLUMNS
+
+  def total_repository_size
+    repository_size + lfs_objects_size
+  end
+
+  def refresh!(only: nil)
+    STATISTICS_COLUMNS.each do |column, generator|
+      if only.blank? || only.include?(column)
+        public_send("update_#{column}")
+      end
+    end
+
+    save!
+  end
+
+  def update_commit_count
+    self.commit_count = project.repository.commit_count
+  end
+
+  def update_repository_size
+    self.repository_size = project.repository.size
+  end
+
+  def update_lfs_objects_size
+    self.lfs_objects_size = project.lfs_objects.sum(:size)
+  end
+
+  def update_build_artifacts_size
+    self.build_artifacts_size = project.builds.sum(:artifacts_size)
+  end
+
+  def update_storage_size
+    self.storage_size = STORAGE_COLUMNS.sum(&method(:read_attribute))
+  end
+end
diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb
index 6bbc3a9d9ff..dbe2fda27b5 100644
--- a/app/services/git_push_service.rb
+++ b/app/services/git_push_service.rb
@@ -77,7 +77,7 @@ class GitPushService < BaseService
       types = []
     end
 
-    ProjectCacheWorker.perform_async(@project.id, types)
+    ProjectCacheWorker.perform_async(@project.id, types, [:commit_count, :repository_size])
   end
 
   # Schedules processing of commit messages.
diff --git a/app/services/git_tag_push_service.rb b/app/services/git_tag_push_service.rb
index 20a4445bddf..96432837481 100644
--- a/app/services/git_tag_push_service.rb
+++ b/app/services/git_tag_push_service.rb
@@ -12,7 +12,7 @@ class GitTagPushService < BaseService
     project.execute_hooks(@push_data.dup, :tag_push_hooks)
     project.execute_services(@push_data.dup, :tag_push_hooks)
     Ci::CreatePipelineService.new(project, current_user, @push_data).execute
-    ProjectCacheWorker.perform_async(project.id)
+    ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size])
 
     true
   end
diff --git a/app/views/admin/groups/_group.html.haml b/app/views/admin/groups/_group.html.haml
index cf28f92853e..6fc212119c4 100644
--- a/app/views/admin/groups/_group.html.haml
+++ b/app/views/admin/groups/_group.html.haml
@@ -5,6 +5,9 @@
     = link_to 'Edit', admin_group_edit_path(group), id: "edit_#{dom_id(group)}", class: 'btn'
     = link_to 'Delete', [:admin, group], data: { confirm: "Are you sure you want to remove #{group.name}?" }, method: :delete, class: 'btn btn-remove'
   .stats
+    %span.badge
+      = storage_counter(group.storage_size)
+
     %span
       = icon('bookmark')
       = number_with_delimiter(group.projects.count)
diff --git a/app/views/admin/groups/index.html.haml b/app/views/admin/groups/index.html.haml
index 794f910a61f..07775247cfd 100644
--- a/app/views/admin/groups/index.html.haml
+++ b/app/views/admin/groups/index.html.haml
@@ -27,6 +27,8 @@
                   = sort_title_recently_updated
                 = link_to admin_groups_path(sort: sort_value_oldest_updated, name: project_name) do
                   = sort_title_oldest_updated
+                = link_to admin_groups_path(sort: sort_value_largest_group, name: project_name) do
+                  = sort_title_largest_group
           = link_to new_admin_group_path, class: "btn btn-new" do
             New Group
   %ul.content-list
diff --git a/app/views/admin/groups/show.html.haml b/app/views/admin/groups/show.html.haml
index 7b0175af214..ab9c79f6add 100644
--- a/app/views/admin/groups/show.html.haml
+++ b/app/views/admin/groups/show.html.haml
@@ -39,6 +39,18 @@
             = @group.created_at.to_s(:medium)
 
         %li
+          %span.light Storage:
+          %strong= storage_counter(@group.storage_size)
+          (
+          = storage_counter(@group.repository_size)
+          repositories,
+          = storage_counter(@group.build_artifacts_size)
+          build artifacts,
+          = storage_counter(@group.lfs_objects_size)
+          LFS
+          )
+
+        %li
           %span.light Group Git LFS status:
           %strong
             = group_lfs_status(@group)
@@ -55,8 +67,8 @@
           %li
             %strong
               = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
-              %span.label.label-gray
-                = repository_size(project)
+              %span.badge
+                = storage_counter(project.statistics.storage_size)
             %span.pull-right.light
               %span.monospace= project.path_with_namespace + ".git"
       .panel-footer
@@ -73,8 +85,8 @@
             %li
               %strong
                 = link_to project.name_with_namespace, [:admin, project.namespace.becomes(Namespace), project]
-                %span.label.label-gray
-                  = repository_size(project)
+                %span.badge
+                  = storage_counter(project.statistics.storage_size)
               %span.pull-right.light
                 %span.monospace= project.path_with_namespace + ".git"
 
diff --git a/app/views/admin/projects/index.html.haml b/app/views/admin/projects/index.html.haml
index 8bc7dc7dd51..2e6f03fcde0 100644
--- a/app/views/admin/projects/index.html.haml
+++ b/app/views/admin/projects/index.html.haml
@@ -69,8 +69,8 @@
             .controls
               - if project.archived
                 %span.label.label-warning archived
-              %span.label.label-gray
-                = repository_size(project)
+              %span.badge
+                = storage_counter(project.statistics.storage_size)
               = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn"
               = link_to 'Delete', [project.namespace.becomes(Namespace), project], data: { confirm: remove_project_message(project) }, method: :delete, class: "btn btn-remove"
             .title
diff --git a/app/views/admin/projects/show.html.haml b/app/views/admin/projects/show.html.haml
index 6c7c3c48604..2967da6e692 100644
--- a/app/views/admin/projects/show.html.haml
+++ b/app/views/admin/projects/show.html.haml
@@ -65,9 +65,16 @@
               = @project.repository.path_to_repo
 
           %li
-            %span.light Size
-            %strong
-              = repository_size(@project)
+            %span.light Storage:
+            %strong= storage_counter(@project.statistics.storage_size)
+            (
+            = storage_counter(@project.statistics.repository_size)
+            repository,
+            = storage_counter(@project.statistics.build_artifacts_size)
+            build artifacts,
+            = storage_counter(@project.statistics.lfs_objects_size)
+            LFS
+            )
 
           %li
             %span.light last commit:
diff --git a/app/views/groups/projects.html.haml b/app/views/groups/projects.html.haml
index 33fee334d93..2e7e5e5c309 100644
--- a/app/views/groups/projects.html.haml
+++ b/app/views/groups/projects.html.haml
@@ -18,8 +18,8 @@
         .pull-right
           - if project.archived
             %span.label.label-warning archived
-          %span.label.label-gray
-            = repository_size(project)
+          %span.badge
+            = storage_counter(project.statistics.storage_size)
           = link_to 'Members', namespace_project_project_members_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
           = link_to 'Edit', edit_namespace_project_path(project.namespace, project), id: "edit_#{dom_id(project)}", class: "btn btn-sm"
           = link_to 'Remove', project, data: { confirm: remove_project_message(project)}, method: :delete, class: "btn btn-sm btn-remove"
diff --git a/app/views/projects/show.html.haml b/app/views/projects/show.html.haml
index a915c159cb4..e33182c357f 100644
--- a/app/views/projects/show.html.haml
+++ b/app/views/projects/show.html.haml
@@ -17,10 +17,10 @@
     %ul.nav
       %li
         = link_to project_files_path(@project) do
-          Files (#{repository_size})
+          Files (#{storage_counter(@project.statistics.total_repository_size)})
       %li
         = link_to namespace_project_commits_path(@project.namespace, @project, current_ref) do
-          #{'Commit'.pluralize(@project.commit_count)} (#{number_with_delimiter(@project.commit_count)})
+          #{'Commit'.pluralize(@project.statistics.commit_count)} (#{number_with_delimiter(@project.statistics.commit_count)})
       %li
         = link_to namespace_project_branches_path(@project.namespace, @project) do
           #{'Branch'.pluralize(@repository.branch_count)} (#{number_with_delimiter(@repository.branch_count)})
diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb
index 27d7e652721..8ff9d07860f 100644
--- a/app/workers/project_cache_worker.rb
+++ b/app/workers/project_cache_worker.rb
@@ -6,26 +6,27 @@ class ProjectCacheWorker
   LEASE_TIMEOUT = 15.minutes.to_i
 
   # project_id - The ID of the project for which to flush the cache.
-  # refresh - An Array containing extra types of data to refresh such as
-  #           `:readme` to flush the README and `:changelog` to flush the
-  #           CHANGELOG.
-  def perform(project_id, refresh = [])
+  # files - An Array containing extra types of files to refresh such as
+  #         `:readme` to flush the README and `:changelog` to flush the
+  #         CHANGELOG.
+  # statistics - An Array containing columns from ProjectStatistics to
+  #              refresh, if empty all columns will be refreshed
+  def perform(project_id, files = [], statistics = [])
     project = Project.find_by(id: project_id)
 
     return unless project && project.repository.exists?
 
-    update_repository_size(project)
-    project.update_commit_count
+    update_statistics(project, statistics.map(&:to_sym))
 
-    project.repository.refresh_method_caches(refresh.map(&:to_sym))
+    project.repository.refresh_method_caches(files.map(&:to_sym))
   end
 
-  def update_repository_size(project)
-    return unless try_obtain_lease_for(project.id, :update_repository_size)
+  def update_statistics(project, statistics = [])
+    return unless try_obtain_lease_for(project.id, :update_statistics)
 
-    Rails.logger.info("Updating repository size for project #{project.id}")
+    Rails.logger.info("Updating statistics for project #{project.id}")
 
-    project.update_repository_size
+    project.statistics.refresh!(only: statistics)
   end
 
   private