Merge branch '42086-project-fetch-statistics-api-http-only' into 'master'57125-heatmap-chart-type

Resolve "Project fetch statistics API (HTTP only)"

Closes #42086

See merge request gitlab-org/gitlab-ce!23596

25 files changed, 393 insertions, 3 deletions

diff --git a/app/controllers/projects/git_http_controller.rb b/app/controllers/projects/git_http_controller.rb
index 30e436365de..0c5328fc941 100644
--- a/app/controllers/projects/git_http_controller.rb
+++ b/app/controllers/projects/git_http_controller.rb
@@ -20,6 +20,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController
 
   # POST /foo/bar.git/git-upload-pack (git pull)
   def git_upload_pack
+    enqueue_fetch_statistics_update
+
     render_ok
   end
 
@@ -67,6 +69,13 @@ class Projects::GitHttpController < Projects::GitHttpClientController
     render plain: exception.message, status: :service_unavailable
   end
 
+  def enqueue_fetch_statistics_update
+    return if wiki?
+    return unless project.daily_statistics_enabled?
+
+    ProjectDailyStatisticsWorker.perform_async(project.id)
+  end
+
   def access
     @access ||= access_klass.new(access_actor, project,
       'http', authentication_abilities: authentication_abilities,
diff --git a/app/finders/projects/daily_statistics_finder.rb b/app/finders/projects/daily_statistics_finder.rb
new file mode 100644
index 00000000000..912c23107bc
--- /dev/null
+++ b/app/finders/projects/daily_statistics_finder.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Projects
+  class DailyStatisticsFinder
+    attr_reader :project
+
+    def initialize(project)
+      @project = project
+    end
+
+    def fetches
+      ProjectDailyStatistic.of_project(project)
+        .of_last_30_days
+        .sorted_by_date_desc
+    end
+
+    def total_fetch_count
+      fetches.sum_fetch_count
+    end
+  end
+end
diff --git a/app/models/project.rb b/app/models/project.rb
index b016a65f0bb..47baf899222 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -639,6 +639,10 @@ class Project < ActiveRecord::Base
     auto_devops&.enabled.nil? && !(Gitlab::CurrentSettings.auto_devops_enabled? || Feature.enabled?(:force_autodevops_on_by_default, self))
   end
 
+  def daily_statistics_enabled?
+    Feature.enabled?(:project_daily_statistics, self, default_enabled: true)
+  end
+
   def empty_repo?
     repository.empty?
   end
diff --git a/app/models/project_daily_statistic.rb b/app/models/project_daily_statistic.rb
new file mode 100644
index 00000000000..ff115dd010f
--- /dev/null
+++ b/app/models/project_daily_statistic.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class ProjectDailyStatistic < ActiveRecord::Base
+  belongs_to :project
+
+  scope :of_project, -> (project) { where(project: project) }
+  scope :of_last_30_days, -> { where('date >= ?', 29.days.ago.utc.to_date) }
+  scope :sorted_by_date_desc, -> { order(project_id: :desc, date: :desc) }
+  scope :sum_fetch_count, -> { sum(:fetch_count) }
+end
diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb
index 95dd8b2795e..533782104ac 100644
--- a/app/policies/project_policy.rb
+++ b/app/policies/project_policy.rb
@@ -278,6 +278,7 @@ class ProjectPolicy < BasePolicy
     enable :admin_cluster
     enable :create_environment_terminal
     enable :destroy_release
+    enable :daily_statistics
   end
 
   rule { (mirror_available & can?(:admin_project)) | admin }.enable :admin_remote_mirror
diff --git a/app/services/projects/fetch_statistics_increment_service.rb b/app/services/projects/fetch_statistics_increment_service.rb
new file mode 100644
index 00000000000..8644e6bf313
--- /dev/null
+++ b/app/services/projects/fetch_statistics_increment_service.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Projects
+  class FetchStatisticsIncrementService
+    attr_reader :project
+
+    def initialize(project)
+      @project = project
+    end
+
+    def execute
+      increment_fetch_count_sql = <<~SQL
+        INSERT INTO #{table_name} (project_id, date, fetch_count)
+        VALUES (#{project.id}, '#{Date.today}', 1)
+      SQL
+
+      increment_fetch_count_sql += if Gitlab::Database.postgresql?
+                                     "ON CONFLICT (project_id, date) DO UPDATE SET fetch_count = #{table_name}.fetch_count + 1"
+                                   else
+                                     "ON DUPLICATE KEY UPDATE fetch_count = #{table_name}.fetch_count + 1"
+                                   end
+
+      ActiveRecord::Base.connection.execute(increment_fetch_count_sql)
+    end
+
+    private
+
+    def table_name
+      ProjectDailyStatistic.table_name
+    end
+  end
+end
diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml
index d0fc130b04f..337f39b2091 100644
--- a/app/workers/all_queues.yml
+++ b/app/workers/all_queues.yml
@@ -146,3 +146,4 @@
 - repository_cleanup
 - delete_stored_files
 - import_issues_csv
+- project_daily_statistics
diff --git a/app/workers/project_daily_statistics_worker.rb b/app/workers/project_daily_statistics_worker.rb
new file mode 100644
index 00000000000..101f5c28459
--- /dev/null
+++ b/app/workers/project_daily_statistics_worker.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class ProjectDailyStatisticsWorker
+  include ApplicationWorker
+
+  def perform(project_id)
+    project = Project.find_by_id(project_id)
+
+    return unless project&.repository&.exists?
+
+    Projects::FetchStatisticsIncrementService.new(project).execute
+  end
+end
diff --git a/changelogs/unreleased/42086-project-fetch-statistics-api-http-only.yml b/changelogs/unreleased/42086-project-fetch-statistics-api-http-only.yml
new file mode 100644
index 00000000000..f2c4f88b746
--- /dev/null
+++ b/changelogs/unreleased/42086-project-fetch-statistics-api-http-only.yml
@@ -0,0 +1,5 @@
+---
+title: Add project fetch statistics
+merge_request: 23596
+author: Jacopo Beschi @jacopo-beschi
+type: added
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 90cd787d5ac..1f40363e126 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -85,5 +85,6 @@
   - [repository_cleanup, 1]
   - [delete_stored_files, 1]
   - [remote_mirror_notification, 2]
+  - [project_daily_statistics, 1]
   - [import_issues_csv, 2]
   - [chat_notification, 2]
diff --git a/db/migrate/20181205171941_create_project_daily_statistics.rb b/db/migrate/20181205171941_create_project_daily_statistics.rb
new file mode 100644
index 00000000000..c9e2a1e1aa7
--- /dev/null
+++ b/db/migrate/20181205171941_create_project_daily_statistics.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class CreateProjectDailyStatistics < ActiveRecord::Migration[5.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  def change
+    create_table :project_daily_statistics, id: :bigserial do |t|
+      t.integer :project_id, null: false
+      t.integer :fetch_count, null: false
+      t.date :date
+
+      t.index [:project_id, :date], unique: true, order: { date: :desc }
+      t.foreign_key :projects, on_delete: :cascade
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index d835e48938d..0e850f412ed 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -1586,6 +1586,13 @@ ActiveRecord::Schema.define(version: 20190220150130) do
     t.index ["project_id", "key"], name: "index_project_custom_attributes_on_project_id_and_key", unique: true, using: :btree
   end
 
+  create_table "project_daily_statistics", id: :bigserial, force: :cascade do |t|
+    t.integer "project_id", null: false
+    t.integer "fetch_count", null: false
+    t.date "date"
+    t.index ["project_id", "date"], name: "index_project_daily_statistics_on_project_id_and_date", unique: true, order: { date: :desc }, using: :btree
+  end
+
   create_table "project_deploy_tokens", force: :cascade do |t|
     t.integer "project_id", null: false
     t.integer "deploy_token_id", null: false
@@ -2469,6 +2476,7 @@ ActiveRecord::Schema.define(version: 20190220150130) do
   add_foreign_key "project_auto_devops", "projects", on_delete: :cascade
   add_foreign_key "project_ci_cd_settings", "projects", name: "fk_24c15d2f2e", on_delete: :cascade
   add_foreign_key "project_custom_attributes", "projects", on_delete: :cascade
+  add_foreign_key "project_daily_statistics", "projects", on_delete: :cascade
   add_foreign_key "project_deploy_tokens", "deploy_tokens", on_delete: :cascade
   add_foreign_key "project_deploy_tokens", "projects", on_delete: :cascade
   add_foreign_key "project_error_tracking_settings", "projects", on_delete: :cascade
diff --git a/doc/api/project_statistics.md b/doc/api/project_statistics.md
new file mode 100644
index 00000000000..34d73abfcbf
--- /dev/null
+++ b/doc/api/project_statistics.md
@@ -0,0 +1,49 @@
+# Project statistics API
+
+Every API call to [project](../user/project/index.md) statistics must be authenticated.
+
+## Get the statistics of the last 30 days
+
+Retrieving the statistics requires write access to the repository.
+Currently only HTTP fetches statistics are returned.
+Fetches statistics includes both clones and pulls count and are HTTP only, SSH fetches are not included.
+
+```
+GET /projects/:id/statistics
+```
+
+| Attribute  | Type   | Required | Description |
+| ---------- | ------ | -------- | ----------- |
+| `id `      | integer / string | yes      | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) |
+
+Example response:
+
+```json
+{
+  "fetches": {
+    "total": 50,
+    "days": [
+      {
+        "count": 10,
+        "date": "2018-01-10"
+      },
+      {
+        "count": 10,
+        "date": "2018-01-09"
+      },
+      {
+        "count": 10,
+        "date": "2018-01-08"
+      },
+      {
+        "count": 10,
+        "date": "2018-01-07"
+      },
+      {
+        "count": 10,
+        "date": "2018-01-06"
+      }
+    ]
+  }
+}
+```
diff --git a/doc/user/permissions.md b/doc/user/permissions.md
index dff77acd71b..ea3e9d0cce9 100644
--- a/doc/user/permissions.md
+++ b/doc/user/permissions.md
@@ -110,6 +110,7 @@ The following table depicts the various user permission levels in a project.
 | Force push to protected branches [^4] |         |            |             |          |        |
 | Remove protected branches [^4]        |         |            |             |          |        |
 | View project Audit Events             |         |            |             | ✓        | ✓      |
+| View project statistics               |         |            |             | ✓        | ✓      |
 
 ## Project features permissions
 
diff --git a/doc/user/project/index.md b/doc/user/project/index.md
index c62de41c539..0dc50d28cb0 100644
--- a/doc/user/project/index.md
+++ b/doc/user/project/index.md
@@ -8,7 +8,7 @@ Your projects can be [available](../../public_access/public_access.md)
 publicly, internally, or privately, at your choice. GitLab does not limit
 the number of private projects you create.
 
-## Project's features
+## Project features
 
 When you create a project in GitLab, you'll have access to a large number of
 [features](https://about.gitlab.com/features/):
@@ -82,7 +82,7 @@ When you create a project in GitLab, you'll have access to a large number of
   the source, build output, and other metadata or artifacts
   associated with a released version of your code.
 
-### Project's integrations
+### Project integrations
 
 [Integrate your project](integrations/index.md) with Jira, Mattermost,
 Kubernetes, Slack, and a lot more.
@@ -116,7 +116,7 @@ Read through the documentation on [project settings](settings/index.md).
 - [Export a project from GitLab](settings/import_export.md#exporting-a-project-and-its-data)
 - [Importing and exporting projects between GitLab instances](settings/import_export.md)
 
-## Project's members
+## Project members
 
 Learn how to [add members to your projects](members/index.md).
 
@@ -170,3 +170,23 @@ password <personal_access_token>
 
 To quickly access a project from the GitLab UI using the project ID,
 visit the `/projects/:id` URL in your browser or other tool accessing the project.
+
+## Project APIs
+
+There are numerous [APIs](../../api/README.md) to use with your projects:
+
+- [Badges](../../api/project_badges.md)
+- [Clusters](../../api/project_clusters.md)
+- [Discussions](../../api/discussions.md)
+- [General](../../api/projects.md)
+- [Import/export](../../api/project_import_export.md)
+- [Issue Board](../../api/boards.md)
+- [Labels](../../api/labels.md)
+- [Markdown](../../api/markdown.md)
+- [Merge Requests](../../api/merge_requests.md)
+- [Milestones](../../api/milestones.md)
+- [Services](../../api/services.md)
+- [Snippets](../../api/project_snippets.md)
+- [Templates](../../api/project_templates.md)
+- [Traffic](../../api/project_statistics.md)
+- [Variables](../../api/project_level_variables.md)
diff --git a/lib/api/api.rb b/lib/api/api.rb
index 4dd1b459554..bf8ddba6f0d 100644
--- a/lib/api/api.rb
+++ b/lib/api/api.rb
@@ -141,6 +141,7 @@ module API
     mount ::API::Projects
     mount ::API::ProjectSnapshots
     mount ::API::ProjectSnippets
+    mount ::API::ProjectStatistics
     mount ::API::ProjectTemplates
     mount ::API::ProtectedBranches
     mount ::API::ProtectedTags
diff --git a/lib/api/entities.rb b/lib/api/entities.rb
index 26ca90c02cb..18f15632f2b 100644
--- a/lib/api/entities.rb
+++ b/lib/api/entities.rb
@@ -300,6 +300,18 @@ module API
       expose :build_artifacts_size, as: :job_artifacts_size
     end
 
+    class ProjectDailyFetches < Grape::Entity
+      expose :fetch_count, as: :count
+      expose :date
+    end
+
+    class ProjectDailyStatistics < Grape::Entity
+      expose :fetches do
+        expose :total_fetch_count, as: :total
+        expose :fetches, as: :days, using: ProjectDailyFetches
+      end
+    end
+
     class Member < Grape::Entity
       expose :user, merge: true, using: UserBasic
       expose :access_level
diff --git a/lib/api/project_statistics.rb b/lib/api/project_statistics.rb
new file mode 100644
index 00000000000..2f73785f72d
--- /dev/null
+++ b/lib/api/project_statistics.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module API
+  class ProjectStatistics < Grape::API
+    before do
+      authenticate!
+      not_found! unless user_project.daily_statistics_enabled?
+      authorize! :daily_statistics, user_project
+    end
+
+    params do
+      requires :id, type: String, desc: 'The ID of a project'
+    end
+    resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+      desc 'Get the list of project fetch statistics for the last 30 days'
+      get ":id/statistics" do
+        statistic_finder = ::Projects::DailyStatisticsFinder.new(user_project)
+
+        present statistic_finder, with: Entities::ProjectDailyStatistics
+      end
+    end
+  end
+end
diff --git a/spec/factories/project_daily_statistics.rb b/spec/factories/project_daily_statistics.rb
new file mode 100644
index 00000000000..7e4142fa401
--- /dev/null
+++ b/spec/factories/project_daily_statistics.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :project_daily_statistic do
+    project
+    fetch_count 1
+  end
+end
diff --git a/spec/models/project_daily_statistic_spec.rb b/spec/models/project_daily_statistic_spec.rb
new file mode 100644
index 00000000000..86210af15d8
--- /dev/null
+++ b/spec/models/project_daily_statistic_spec.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe ProjectDailyStatistic do
+  it { is_expected.to belong_to(:project) }
+end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 9cc9894003d..9fb0d04ca9e 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -2361,6 +2361,18 @@ describe Project do
     end
   end
 
+  describe '#daily_statistics_enabled?' do
+    it { is_expected.to be_daily_statistics_enabled }
+
+    context 'when :project_daily_statistics is disabled for the project' do
+      before do
+        stub_feature_flags(project_daily_statistics: { thing: subject, enabled: false })
+      end
+
+      it { is_expected.not_to be_daily_statistics_enabled }
+    end
+  end
+
   describe '#change_head' do
     let(:project) { create(:project, :repository) }
 
diff --git a/spec/policies/project_policy_spec.rb b/spec/policies/project_policy_spec.rb
index 93a468f585b..997bdc82af6 100644
--- a/spec/policies/project_policy_spec.rb
+++ b/spec/policies/project_policy_spec.rb
@@ -49,6 +49,7 @@ describe ProjectPolicy do
       admin_project_member admin_note admin_wiki admin_project
       admin_commit_status admin_build admin_container_image
       admin_pipeline admin_environment admin_deployment destroy_release add_cluster
+      daily_statistics
     ]
   end
 
diff --git a/spec/requests/api/project_statistics_spec.rb b/spec/requests/api/project_statistics_spec.rb
new file mode 100644
index 00000000000..184d0a72c37
--- /dev/null
+++ b/spec/requests/api/project_statistics_spec.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe API::ProjectStatistics do
+  let(:maintainer) { create(:user) }
+  let(:public_project) { create(:project, :public) }
+
+  before do
+    public_project.add_maintainer(maintainer)
+  end
+
+  describe 'GET /projects/:id/statistics' do
+    let!(:fetch_statistics1) { create(:project_daily_statistic, project: public_project, fetch_count: 30, date: 29.days.ago) }
+    let!(:fetch_statistics2) { create(:project_daily_statistic, project: public_project, fetch_count: 4, date: 3.days.ago) }
+    let!(:fetch_statistics3) { create(:project_daily_statistic, project: public_project, fetch_count: 3, date: 2.days.ago) }
+    let!(:fetch_statistics4) { create(:project_daily_statistic, project: public_project, fetch_count: 2, date: 1.day.ago) }
+    let!(:fetch_statistics5) { create(:project_daily_statistic, project: public_project, fetch_count: 1, date: Date.today) }
+    let!(:fetch_statistics_other_project) { create(:project_daily_statistic, project: create(:project), fetch_count: 29, date: 29.days.ago) }
+
+    it 'returns the fetch statistics of the last 30 days' do
+      get api("/projects/#{public_project.id}/statistics", maintainer)
+
+      expect(response).to have_gitlab_http_status(200)
+      fetches = json_response['fetches']
+      expect(fetches['total']).to eq(40)
+      expect(fetches['days'].length).to eq(5)
+      expect(fetches['days'].first).to eq({ 'count' => fetch_statistics5.fetch_count, 'date' => fetch_statistics5.date.to_s })
+      expect(fetches['days'].last).to eq({ 'count' => fetch_statistics1.fetch_count, 'date' => fetch_statistics1.date.to_s })
+    end
+
+    it 'excludes the fetch statistics older than 30 days' do
+      create(:project_daily_statistic, fetch_count: 31, project: public_project, date: 30.days.ago)
+
+      get api("/projects/#{public_project.id}/statistics", maintainer)
+
+      expect(response).to have_gitlab_http_status(200)
+      fetches = json_response['fetches']
+      expect(fetches['total']).to eq(40)
+      expect(fetches['days'].length).to eq(5)
+      expect(fetches['days'].last).to eq({ 'count' => fetch_statistics1.fetch_count, 'date' => fetch_statistics1.date.to_s })
+    end
+
+    it 'responds with 403 when the user is not a maintainer of the repository' do
+      developer = create(:user)
+      public_project.add_developer(developer)
+
+      get api("/projects/#{public_project.id}/statistics", developer)
+
+      expect(response).to have_gitlab_http_status(403)
+      expect(json_response['message']).to eq('403 Forbidden')
+    end
+
+    it 'responds with 404 when daily_statistics_enabled? is false' do
+      stub_feature_flags(project_daily_statistics: { thing: public_project, enabled: false })
+
+      get api("/projects/#{public_project.id}/statistics", maintainer)
+
+      expect(response).to have_gitlab_http_status(404)
+    end
+  end
+end
diff --git a/spec/services/projects/fetch_statistics_increment_service_spec.rb b/spec/services/projects/fetch_statistics_increment_service_spec.rb
new file mode 100644
index 00000000000..fcfb138aad6
--- /dev/null
+++ b/spec/services/projects/fetch_statistics_increment_service_spec.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+module Projects
+  describe FetchStatisticsIncrementService do
+    let(:project) { create(:project) }
+
+    describe '#execute' do
+      subject { described_class.new(project).execute }
+
+      it 'creates a new record for today with count == 1' do
+        expect { subject }.to change { ProjectDailyStatistic.count }.by(1)
+        created_stat = ProjectDailyStatistic.last
+
+        expect(created_stat.fetch_count).to eq(1)
+        expect(created_stat.project).to eq(project)
+        expect(created_stat.date).to eq(Date.today)
+      end
+
+      it "doesn't increment previous days statistics" do
+        yesterday_stat = create(:project_daily_statistic, fetch_count: 5, project: project, date: 1.day.ago)
+
+        expect { subject }.not_to change { yesterday_stat.reload.fetch_count }
+      end
+
+      context 'when the record already exists for today' do
+        let!(:project_daily_stat) { create(:project_daily_statistic, fetch_count: 5, project: project, date: Date.today) }
+
+        it 'increments the today record count by 1' do
+          expect { subject }.to change { project_daily_stat.reload.fetch_count }.to(6)
+        end
+      end
+    end
+  end
+end
diff --git a/spec/workers/project_daily_statistics_worker_spec.rb b/spec/workers/project_daily_statistics_worker_spec.rb
new file mode 100644
index 00000000000..8640add99e5
--- /dev/null
+++ b/spec/workers/project_daily_statistics_worker_spec.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+describe ProjectDailyStatisticsWorker, '#perform' do
+  let(:worker) { described_class.new }
+  let(:project) { create(:project) }
+
+  describe '#perform' do
+    context 'with a non-existing project' do
+      it 'does nothing' do
+        expect(Projects::FetchStatisticsIncrementService).not_to receive(:new)
+
+        worker.perform(-1)
+      end
+    end
+
+    context 'with an existing project without a repository' do
+      it 'does nothing' do
+        expect(Projects::FetchStatisticsIncrementService).not_to receive(:new)
+
+        worker.perform(project.id)
+      end
+    end
+
+    it 'calls daily_statistics_service with the given project' do
+      project = create(:project, :repository)
+
+      expect_next_instance_of(Projects::FetchStatisticsIncrementService, project) do |service|
+        expect(service).to receive(:execute)
+      end
+
+      worker.perform(project.id)
+    end
+  end
+end