Add more storage statistics

This adds counters for build artifacts and LFS objects, and moves
the preexisting repository_size and commit_count from the projects
table into a new project_statistics table.

The counters are displayed in the administration area for projects
and groups, and also available through the API for admins (on */all)
and normal users (on */owned)

The statistics are updated through ProjectCacheWorker, which can now
do more granular updates with the new :statistics argument.

1 files changed, 43 insertions, 37 deletions

diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index f5609d878f8..3be14e8eb76 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -40,6 +40,15 @@ module API
 
     resource :projects do
       helpers do
+        params :collection_params do
+          use :sort_params
+          use :filter_params
+          use :pagination
+
+          optional :simple, type: Boolean, default: false,
+                            desc: 'Return only the ID, URL, name, and path of each project'
+        end
+
         params :sort_params do
           optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
                               default: 'created_at', desc: 'Return projects ordered by field'
@@ -52,97 +61,94 @@ module API
           optional :visibility, type: String, values: %w[public internal private],
                                 desc: 'Limit by visibility'
           optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
-          use :sort_params
+        end
+
+        params :statistics_params do
+          optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
         end
 
         params :create_params do
           optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
           optional :import_url, type: String, desc: 'URL from which the project is imported'
         end
+
+        def present_projects(projects, options = {})
+          options = options.reverse_merge(
+            with: Entities::Project,
+            current_user: current_user,
+            simple: params[:simple],
+          )
+
+          projects = filter_projects(projects)
+          projects = projects.with_statistics if options[:statistics]
+          options[:with] = Entities::BasicProjectDetails if options[:simple]
+
+          present paginate(projects), options
+        end
       end
 
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/visible' do
-        projects = ProjectsFinder.new.execute(current_user)
-        projects = filter_projects(projects)
-        entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, current_user: current_user
+        entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
+        present_projects ProjectsFinder.new.execute(current_user), with: entity
       end
 
       desc 'Get a projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get do
         authenticate!
 
-        projects = current_user.authorized_projects
-        projects = filter_projects(projects)
-        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, current_user: current_user
+        present_projects current_user.authorized_projects,
+          with: Entities::ProjectWithAccess
       end
 
       desc 'Get an owned projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/owned' do
         authenticate!
 
-        projects = current_user.owned_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user
+        present_projects current_user.owned_projects,
+          with: Entities::ProjectWithAccess,
+          statistics: params[:statistics]
       end
 
       desc 'Gets starred project for the authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/starred' do
         authenticate!
 
-        projects = current_user.viewable_starred_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::Project, current_user: current_user
+        present_projects current_user.viewable_starred_projects
       end
 
       desc 'Get all projects for admin user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/all' do
         authenticated_as_admin!
 
-        projects = Project.all
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, current_user: current_user
+        present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics]
       end
 
       desc 'Search for projects the current user has access to' do