Merge branch 'feature/more-storage-statistics' into 'master'

Add more storage statistics

See merge request !7754

6 files changed, 98 insertions, 79 deletions

diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index dfbb3ab86dd..9f15c08f472 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -78,21 +78,21 @@ module API
       expose :container_registry_enabled
 
       # Expose old field names with the new permissions methods to keep API compatible
-      expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) }
-      expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) }
-      expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) }
-      expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) }
-      expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) }
+      expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) }
+      expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) }
+      expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) }
+      expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) }
+      expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) }
 
       expose :created_at, :last_activity_at
       expose :shared_runners_enabled
       expose :lfs_enabled?, as: :lfs_enabled
       expose :creator_id
-      expose :namespace
+      expose :namespace, using: 'API::Entities::Namespace'
       expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? }
       expose :avatar_url
       expose :star_count, :forks_count
-      expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? }
+      expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? }
       expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] }
       expose :public_builds
       expose :shared_with_groups do |project, options|
@@ -101,6 +101,16 @@ module API
       expose :only_allow_merge_if_build_succeeds
       expose :request_access_enabled
       expose :only_allow_merge_if_all_discussions_are_resolved
+
+      expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics
+    end
+
+    class ProjectStatistics < Grape::Entity
+      expose :commit_count
+      expose :storage_size
+      expose :repository_size
+      expose :lfs_objects_size
+      expose :build_artifacts_size
     end
 
     class Member < UserBasic
@@ -127,6 +137,15 @@ module API
       expose :avatar_url
       expose :web_url
       expose :request_access_enabled
+
+      expose :statistics, if: :statistics do
+        with_options format_with: -> (value) { value.to_i } do
+          expose :storage_size
+          expose :repository_size
+          expose :lfs_objects_size
+          expose :build_artifacts_size
+        end
+      end
     end
 
     class GroupDetail < Group
@@ -391,7 +410,7 @@ module API
     end
 
     class Namespace < Grape::Entity
-      expose :id, :path, :kind
+      expose :id, :name, :path, :kind
     end
 
     class MemberAccess < Grape::Entity
@@ -440,12 +459,12 @@ module API
     class ProjectWithAccess < Project
       expose :permissions do
         expose :project_access, using: Entities::ProjectAccess do |project, options|
-          project.project_members.find_by(user_id: options[:user].id)
+          project.project_members.find_by(user_id: options[:current_user].id)
         end
 
         expose :group_access, using: Entities::GroupAccess do |project, options|
           if project.group
-            project.group.group_members.find_by(user_id: options[:user].id)
+            project.group.group_members.find_by(user_id: options[:current_user].id)
           end
         end
       end
diff --git a/lib/api/groups.rb b/lib/api/groups.rb
index 9b9d3df7435..e04d2e40fb6 100644
--- a/lib/api/groups.rb
+++ b/lib/api/groups.rb
@@ -11,6 +11,20 @@ module API
         optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group'
         optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access'
       end
+
+      params :statistics_params do
+        optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
+      end
+
+      def present_groups(groups, options = {})
+        options = options.reverse_merge(
+          with: Entities::Group,
+          current_user: current_user,
+        )
+
+        groups = groups.with_statistics if options[:statistics]
+        present paginate(groups), options
+      end
     end
 
     resource :groups do
@@ -18,6 +32,7 @@ module API
         success Entities::Group
       end
       params do
+        use :statistics_params
         optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list'
         optional :all_available, type: Boolean, desc: 'Show all group that you have access to'
         optional :search, type: String, desc: 'Search for a specific group'
@@ -38,7 +53,7 @@ module API
         groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present?
         groups = groups.reorder(params[:order_by] => params[:sort])
 
-        present paginate(groups), with: Entities::Group
+        present_groups groups, statistics: params[:statistics] && current_user.is_admin?
       end
 
       desc 'Get list of owned groups for authenticated user' do
@@ -46,10 +61,10 @@ module API
       end
       params do
         use :pagination
+        use :statistics_params
       end
       get '/owned' do
-        groups = current_user.owned_groups
-        present paginate(groups), with: Entities::Group, user: current_user
+        present_groups current_user.owned_groups, statistics: params[:statistics]
       end
 
       desc 'Create a group. Available only for users who can create groups.' do
@@ -66,7 +81,7 @@ module API
         group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute
 
         if group.persisted?
-          present group, with: Entities::Group
+          present group, with: Entities::Group, current_user: current_user
         else
           render_api_error!("Failed to save group #{group.errors.messages}", 400)
         end
@@ -92,7 +107,7 @@ module API
         authorize! :admin_group, group
 
         if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute
-          present group, with: Entities::GroupDetail
+          present group, with: Entities::GroupDetail, current_user: current_user
         else
           render_validation_error!(group)
         end
@@ -103,7 +118,7 @@ module API
       end
       get ":id" do
         group = find_group!(params[:id])
-        present group, with: Entities::GroupDetail
+        present group, with: Entities::GroupDetail, current_user: current_user
       end
 
       desc 'Remove a group.'
@@ -134,7 +149,7 @@ module API
         projects = GroupProjectsFinder.new(group).execute(current_user)
         projects = filter_projects(projects)
         entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project
-        present paginate(projects), with: entity, user: current_user
+        present paginate(projects), with: entity, current_user: current_user
       end
 
       desc 'Transfer a project to the group namespace. Available only for admin.' do
@@ -150,7 +165,7 @@ module API
         result = ::Projects::TransferService.new(project, current_user).execute(group)
 
         if result
-          present group, with: Entities::GroupDetail
+          present group, with: Entities::GroupDetail, current_user: current_user
         else
           render_api_error!("Failed to transfer project #{project.errors.messages}", 400)
         end
diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb
index 4be659fc20b..fe00c83bff3 100644
--- a/lib/api/helpers.rb
+++ b/lib/api/helpers.rb
@@ -248,7 +248,7 @@ module API
       rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500)
     end
 
-    # Projects helpers
+    # project helpers
 
     def filter_projects(projects)
       if params[:search].present?
diff --git a/lib/api/projects.rb b/lib/api/projects.rb
index 2929d2157dc..3be14e8eb76 100644
--- a/lib/api/projects.rb
+++ b/lib/api/projects.rb
@@ -40,6 +40,15 @@ module API
 
     resource :projects do
       helpers do
+        params :collection_params do
+          use :sort_params
+          use :filter_params
+          use :pagination
+
+          optional :simple, type: Boolean, default: false,
+                            desc: 'Return only the ID, URL, name, and path of each project'
+        end
+
         params :sort_params do
           optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at],
                               default: 'created_at', desc: 'Return projects ordered by field'
@@ -52,97 +61,94 @@ module API
           optional :visibility, type: String, values: %w[public internal private],
                                 desc: 'Limit by visibility'
           optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria'
-          use :sort_params
+        end
+
+        params :statistics_params do
+          optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'
         end
 
         params :create_params do
           optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.'
           optional :import_url, type: String, desc: 'URL from which the project is imported'
         end
+
+        def present_projects(projects, options = {})
+          options = options.reverse_merge(
+            with: Entities::Project,
+            current_user: current_user,
+            simple: params[:simple],
+          )
+
+          projects = filter_projects(projects)
+          projects = projects.with_statistics if options[:statistics]
+          options[:with] = Entities::BasicProjectDetails if options[:simple]
+
+          present paginate(projects), options
+        end
       end
 
       desc 'Get a list of visible projects for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/visible' do
-        projects = ProjectsFinder.new.execute(current_user)
-        projects = filter_projects(projects)
-        entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, user: current_user
+        entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
+        present_projects ProjectsFinder.new.execute(current_user), with: entity
       end
 
       desc 'Get a projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        optional :simple, type: Boolean, default: false,
-                          desc: 'Return only the ID, URL, name, and path of each project'
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get do
         authenticate!
 
-        projects = current_user.authorized_projects
-        projects = filter_projects(projects)
-        entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess
-
-        present paginate(projects), with: entity, user: current_user
+        present_projects current_user.authorized_projects,
+          with: Entities::ProjectWithAccess
       end
 
       desc 'Get an owned projects list for authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/owned' do
         authenticate!
 
-        projects = current_user.owned_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, user: current_user
+        present_projects current_user.owned_projects,
+          with: Entities::ProjectWithAccess,
+          statistics: params[:statistics]
       end
 
       desc 'Gets starred project for the authenticated user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
       end
       get '/starred' do
         authenticate!
 
-        projects = current_user.viewable_starred_projects
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::Project, user: current_user
+        present_projects current_user.viewable_starred_projects
       end
 
       desc 'Get all projects for admin user' do
         success Entities::BasicProjectDetails
       end
       params do
-        use :filter_params
-        use :pagination
+        use :collection_params
+        use :statistics_params
       end
       get '/all' do
         authenticated_as_admin!
 
-        projects = Project.all
-        projects = filter_projects(projects)
-
-        present paginate(projects), with: Entities::ProjectWithAccess, user: current_user
+        present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics]
       end
 
       desc 'Search for projects the current user has access to' do
@@ -221,7 +227,7 @@ module API
       end
       get ":id" do
         entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails
-        present user_project, with: entity, user: current_user,
+        present user_project, with: entity, current_user: current_user,
                               user_can_admin_project: can?(current_user, :admin_project, user_project)
       end
 
diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake
index dbdd4e977e8..a2eca74a3c8 100644
--- a/lib/tasks/gitlab/import.rake
+++ b/lib/tasks/gitlab/import.rake
@@ -63,8 +63,7 @@ namespace :gitlab do
 
             if project.persisted?
               puts " * Created #{project.name} (#{repo_path})".color(:green)
-              project.update_repository_size
-              project.update_commit_count
+              ProjectCacheWorker.perform(project.id)
             else
               puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red)
               puts "   Errors: #{project.errors.messages}".color(:red)
diff --git a/lib/tasks/gitlab/update_commit_count.rake b/lib/tasks/gitlab/update_commit_count.rake
deleted file mode 100644
index 3bd10b0208b..00000000000
--- a/lib/tasks/gitlab/update_commit_count.rake
+++ /dev/null
@@ -1,20 +0,0 @@
-namespace :gitlab do
-  desc "GitLab | Update commit count for projects"
-  task update_commit_count: :environment do
-    projects = Project.where(commit_count: 0)
-    puts "#{projects.size} projects need to be updated. This might take a while."
-    ask_to_continue unless ENV['force'] == 'yes'
-
-    projects.find_each(batch_size: 100) do |project|
-      print "#{project.name_with_namespace.color(:yellow)} ... "
-
-      unless project.repo_exists?
-        puts "skipping, because the repo is empty".color(:magenta)
-        next
-      end
-
-      project.update_commit_count
-      puts project.commit_count.to_s.color(:green)
-    end
-  end
-end