diff options
Diffstat (limited to 'app')
23 files changed, 354 insertions, 2 deletions
diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index d88b7cd0e17..0faf757eaab 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -367,6 +367,9 @@ const ShortcutsBlob = require('./shortcuts_blob'); case 'admin': new Admin(); switch (path[1]) { + case 'cohorts': + new gl.UsagePing(); + break; case 'groups': new UsersSelect(); break; diff --git a/app/assets/javascripts/main.js b/app/assets/javascripts/main.js index c50ec24c818..36616d02bf6 100644 --- a/app/assets/javascripts/main.js +++ b/app/assets/javascripts/main.js @@ -165,6 +165,7 @@ import './syntax_highlight'; import './task_list'; import './todos'; import './tree'; +import './usage_ping'; import './user'; import './user_tabs'; import './username_validator'; diff --git a/app/assets/javascripts/usage_ping.js b/app/assets/javascripts/usage_ping.js new file mode 100644 index 00000000000..fd3af7d7ab6 --- /dev/null +++ b/app/assets/javascripts/usage_ping.js @@ -0,0 +1,15 @@ +function UsagePing() { + const usageDataUrl = $('.usage-data').data('endpoint'); + + $.ajax({ + type: 'GET', + url: usageDataUrl, + dataType: 'html', + success(html) { + $('.usage-data').html(html); + }, + }); +} + +window.gl = window.gl || {}; +window.gl.UsagePing = UsagePing; diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index 515d8e1523b..643993d035e 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -17,6 +17,18 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController end end + def usage_data + respond_to do |format| + format.html do + usage_data = Gitlab::UsageData.data + usage_data_json = params[:pretty] ? JSON.pretty_generate(usage_data) : usage_data.to_json + + render html: Gitlab::Highlight.highlight('payload.json', usage_data_json) + end + format.json { render json: Gitlab::UsageData.to_json } + end + end + def reset_runners_token @application_setting.reset_runners_registration_token! flash[:notice] = 'New runners registration token has been generated!' @@ -135,6 +147,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController :version_check_enabled, :terminal_max_session_time, :polling_interval_multiplier, + :usage_ping_enabled, disabled_oauth_sign_in_sources: [], import_sources: [], diff --git a/app/controllers/admin/cohorts_controller.rb b/app/controllers/admin/cohorts_controller.rb new file mode 100644 index 00000000000..9b77c554908 --- /dev/null +++ b/app/controllers/admin/cohorts_controller.rb @@ -0,0 +1,11 @@ +class Admin::CohortsController < Admin::ApplicationController + def index + if current_application_settings.usage_ping_enabled + cohorts_results = Rails.cache.fetch('cohorts', expires_in: 1.day) do + CohortsService.new.execute + end + + @cohorts = CohortsSerializer.new.represent(cohorts_results) + end + end +end diff --git a/app/controllers/projects/git_http_controller.rb b/app/controllers/projects/git_http_controller.rb index 37f6f637ff0..10adddb4636 100644 --- a/app/controllers/projects/git_http_controller.rb +++ b/app/controllers/projects/git_http_controller.rb @@ -5,6 +5,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController # GET /foo/bar.git/info/refs?service=git-receive-pack (git push) def info_refs if upload_pack? && upload_pack_allowed? + log_user_activity + render_ok elsif receive_pack? && receive_pack_allowed? render_ok @@ -106,4 +108,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController def access_klass @access_klass ||= wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess end + + def log_user_activity + Users::ActivityService.new(user, 'pull').execute + end end diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index d3091a4f8e9..8c6ba4915cd 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -35,6 +35,7 @@ class SessionsController < Devise::SessionsController # hide the signed-in notification flash[:notice] = nil log_audit_event(current_user, with: authentication_method) + log_user_activity(current_user) end end @@ -123,6 +124,10 @@ class SessionsController < Devise::SessionsController for_authentication.security_event end + def log_user_activity(user) + Users::ActivityService.new(user, 'login').execute + end + def load_recaptcha Gitlab::Recaptcha.load_configurations! end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 2961e16f5e0..dd1a6922968 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -238,7 +238,8 @@ class ApplicationSetting < ActiveRecord::Base terminal_max_session_time: 0, two_factor_grace_period: 48, user_default_external: false, - polling_interval_multiplier: 1 + polling_interval_multiplier: 1, + usage_ping_enabled: true } end diff --git a/app/serializers/cohort_activity_month_entity.rb b/app/serializers/cohort_activity_month_entity.rb new file mode 100644 index 00000000000..e6788a8b596 --- /dev/null +++ b/app/serializers/cohort_activity_month_entity.rb @@ -0,0 +1,11 @@ +class CohortActivityMonthEntity < Grape::Entity + include ActionView::Helpers::NumberHelper + + expose :total do |cohort_activity_month| + number_with_delimiter(cohort_activity_month[:total]) + end + + expose :percentage do |cohort_activity_month| + number_to_percentage(cohort_activity_month[:percentage], precision: 0) + end +end diff --git a/app/serializers/cohort_entity.rb b/app/serializers/cohort_entity.rb new file mode 100644 index 00000000000..7cdba5b0484 --- /dev/null +++ b/app/serializers/cohort_entity.rb @@ -0,0 +1,17 @@ +class CohortEntity < Grape::Entity + include ActionView::Helpers::NumberHelper + + expose :registration_month do |cohort| + cohort[:registration_month].strftime('%b %Y') + end + + expose :total do |cohort| + number_with_delimiter(cohort[:total]) + end + + expose :inactive do |cohort| + number_with_delimiter(cohort[:inactive]) + end + + expose :activity_months, using: CohortActivityMonthEntity +end diff --git a/app/serializers/cohorts_entity.rb b/app/serializers/cohorts_entity.rb new file mode 100644 index 00000000000..98f5995ba6f --- /dev/null +++ b/app/serializers/cohorts_entity.rb @@ -0,0 +1,4 @@ +class CohortsEntity < Grape::Entity + expose :months_included + expose :cohorts, using: CohortEntity +end diff --git a/app/serializers/cohorts_serializer.rb b/app/serializers/cohorts_serializer.rb new file mode 100644 index 00000000000..fe9367b13d8 --- /dev/null +++ b/app/serializers/cohorts_serializer.rb @@ -0,0 +1,3 @@ +class CohortsSerializer < AnalyticsGenericSerializer + entity CohortsEntity +end diff --git a/app/services/cohorts_service.rb b/app/services/cohorts_service.rb new file mode 100644 index 00000000000..6781533af28 --- /dev/null +++ b/app/services/cohorts_service.rb @@ -0,0 +1,100 @@ +class CohortsService + MONTHS_INCLUDED = 12 + + def execute + { + months_included: MONTHS_INCLUDED, + cohorts: cohorts + } + end + + # Get an array of hashes that looks like: + # + # [ + # { + # registration_month: Date.new(2017, 3), + # activity_months: [3, 2, 1], + # total: 3 + # inactive: 0 + # }, + # etc. + # + # The `months` array is always from oldest to newest, so it's always + # non-strictly decreasing from left to right. + def cohorts + months = Array.new(MONTHS_INCLUDED) { |i| i.months.ago.beginning_of_month.to_date } + + Array.new(MONTHS_INCLUDED) do + registration_month = months.last + activity_months = running_totals(months, registration_month) + + # Even if no users registered in this month, we always want to have a + # value to fill in the table. + inactive = counts_by_month[[registration_month, nil]].to_i + + months.pop + + { + registration_month: registration_month, + activity_months: activity_months, + total: activity_months.first[:total], + inactive: inactive + } + end + end + + private + + # Calculate a running sum of active users, so users active in later months + # count as active in this month, too. Start with the most recent month first, + # for calculating the running totals, and then reverse for displaying in the + # table. + # + # Each month has a total, and a percentage of the overall total, as keys. + def running_totals(all_months, registration_month) + month_totals = + all_months + .map { |activity_month| counts_by_month[[registration_month, activity_month]] } + .reduce([]) { |result, total| result << result.last.to_i + total.to_i } + .reverse + + overall_total = month_totals.first + + month_totals.map do |total| + { total: total, percentage: total.zero? ? 0 : 100 * total / overall_total } + end + end + + # Get a hash that looks like: + # + # { + # [created_at_month, last_activity_on_month] => count, + # [created_at_month, last_activity_on_month_2] => count_2, + # # etc. + # } + # + # created_at_month can never be nil, but last_activity_on_month can (when a + # user has never logged in, just been created). This covers the last + # MONTHS_INCLUDED months. + def counts_by_month + @counts_by_month ||= + begin + created_at_month = column_to_date('created_at') + last_activity_on_month = column_to_date('last_activity_on') + + User + .where('created_at > ?', MONTHS_INCLUDED.months.ago.end_of_month) + .group(created_at_month, last_activity_on_month) + .reorder("#{created_at_month} ASC", "#{last_activity_on_month} ASC") + .count + end + end + + def column_to_date(column) + if Gitlab::Database.postgresql? + "CAST(DATE_TRUNC('month', #{column}) AS date)" + else + "STR_TO_DATE(DATE_FORMAT(#{column}, '%Y-%m-01'), '%Y-%m-%d')" + end + end +end diff --git a/app/services/event_create_service.rb b/app/services/event_create_service.rb index e24cc66e0fe..0f3a485a3fd 100644 --- a/app/services/event_create_service.rb +++ b/app/services/event_create_service.rb @@ -72,6 +72,8 @@ class EventCreateService def push(project, current_user, push_data) create_event(project, current_user, Event::PUSHED, data: push_data) + + Users::ActivityService.new(current_user, 'push').execute end private diff --git a/app/services/users/activity_service.rb b/app/services/users/activity_service.rb new file mode 100644 index 00000000000..facf21a7f5c --- /dev/null +++ b/app/services/users/activity_service.rb @@ -0,0 +1,22 @@ +module Users + class ActivityService + def initialize(author, activity) + @author = author.respond_to?(:user) ? author.user : author + @activity = activity + end + + def execute + return unless @author && @author.is_a?(User) + + record_activity + end + + private + + def record_activity + Gitlab::UserActivities.record(@author.id) + + Rails.logger.debug("Recorded activity: #{@activity} for User ID: #{@author.id} (username: #{@author.username}") + end + end +end diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index f4ba44096d3..0dc1103eece 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -477,7 +477,7 @@ diagrams in Asciidoc documents using an external PlantUML service. %fieldset - %legend Usage statistics + %legend#usage-statistics Usage statistics .form-group .col-sm-offset-2.col-sm-10 .checkbox @@ -486,6 +486,19 @@ Version check enabled .help-block Let GitLab inform you when an update is available. + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :usage_ping_enabled do + = f.check_box :usage_ping_enabled + Usage ping enabled + = link_to icon('question-circle'), help_page_path("user/admin_area/settings/usage_statistics", anchor: "usage-data") + .help-block + Every week GitLab will report license usage back to GitLab, Inc. + Disable this option if you do not want this to occur. To see the + JSON payload that will be sent, visit the + = succeed '.' do + = link_to "Cohorts page", admin_cohorts_path(anchor: 'usage-ping') %fieldset %legend Email diff --git a/app/views/admin/cohorts/_cohorts_table.html.haml b/app/views/admin/cohorts/_cohorts_table.html.haml new file mode 100644 index 00000000000..701a4e62b39 --- /dev/null +++ b/app/views/admin/cohorts/_cohorts_table.html.haml @@ -0,0 +1,28 @@ +.bs-callout.clearfix + %p + User cohorts are shown for the last #{@cohorts[:months_included]} + months. Only users with activity are counted in the cohort total; inactive + users are counted separately. + = link_to icon('question-circle'), help_page_path('user/admin_area/user_cohorts', anchor: 'cohorts'), title: 'About this feature', target: '_blank' + +.table-holder + %table.table + %thead + %tr + %th Registration month + %th Inactive users + %th Cohort total + - @cohorts[:months_included].times do |i| + %th Month #{i} + %tbody + - @cohorts[:cohorts].each do |cohort| + %tr + %td= cohort[:registration_month] + %td= cohort[:inactive] + %td= cohort[:total] + - cohort[:activity_months].each do |activity_month| + %td + - next if cohort[:total] == '0' + = activity_month[:percentage] + %br + = activity_month[:total] diff --git a/app/views/admin/cohorts/_usage_ping.html.haml b/app/views/admin/cohorts/_usage_ping.html.haml new file mode 100644 index 00000000000..73aa95d84f1 --- /dev/null +++ b/app/views/admin/cohorts/_usage_ping.html.haml @@ -0,0 +1,10 @@ +%h2#usage-ping Usage ping + +.bs-callout.clearfix + %p + User cohorts are shown because the usage ping is enabled. The data sent with + this is shown below. To disable this, visit + = succeed '.' do + = link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics') + +%pre.usage-data.js-syntax-highlight.code.highlight{ data: { endpoint: usage_data_admin_application_settings_path(format: :html, pretty: true) } } diff --git a/app/views/admin/cohorts/index.html.haml b/app/views/admin/cohorts/index.html.haml new file mode 100644 index 00000000000..46fe12a5a99 --- /dev/null +++ b/app/views/admin/cohorts/index.html.haml @@ -0,0 +1,16 @@ +- @no_container = true += render "admin/dashboard/head" + +%div{ class: container_class } + - if @cohorts + = render 'cohorts_table' + = render 'usage_ping' + - else + .bs-callout.bs-callout-warning.clearfix + %p + User cohorts are only shown when the + = link_to 'usage ping', help_page_path('user/admin_area/usage_statistics'), target: '_blank' + is enabled. To enable it and see user cohorts, + visit + = succeed '.' do + = link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics') diff --git a/app/views/admin/dashboard/_head.html.haml b/app/views/admin/dashboard/_head.html.haml index 7893c1dee97..163bd5662b0 100644 --- a/app/views/admin/dashboard/_head.html.haml +++ b/app/views/admin/dashboard/_head.html.haml @@ -27,3 +27,7 @@ = link_to admin_runners_path, title: 'Runners' do %span Runners + = nav_link path: 'cohorts#index' do + = link_to admin_cohorts_path, title: 'Cohorts' do + %span + Cohorts diff --git a/app/workers/gitlab_usage_ping_worker.rb b/app/workers/gitlab_usage_ping_worker.rb new file mode 100644 index 00000000000..2f02235b0ac --- /dev/null +++ b/app/workers/gitlab_usage_ping_worker.rb @@ -0,0 +1,31 @@ +class GitlabUsagePingWorker + LEASE_TIMEOUT = 86400 + + include Sidekiq::Worker + include CronjobQueue + include HTTParty + + def perform + return unless current_application_settings.usage_ping_enabled + + # Multiple Sidekiq workers could run this. We should only do this at most once a day. + return unless try_obtain_lease + + begin + HTTParty.post(url, + body: Gitlab::UsageData.to_json(force_refresh: true), + headers: { 'Content-type' => 'application/json' } + ) + rescue HTTParty::Error => e + Rails.logger.info "Unable to contact GitLab, Inc.: #{e}" + end + end + + def try_obtain_lease + Gitlab::ExclusiveLease.new('gitlab_usage_ping_worker:ping', timeout: LEASE_TIMEOUT).try_obtain + end + + def url + 'https://version.gitlab.com/usage_data' + end +end diff --git a/app/workers/schedule_update_user_activity_worker.rb b/app/workers/schedule_update_user_activity_worker.rb new file mode 100644 index 00000000000..6c2c3e437f3 --- /dev/null +++ b/app/workers/schedule_update_user_activity_worker.rb @@ -0,0 +1,10 @@ +class ScheduleUpdateUserActivityWorker + include Sidekiq::Worker + include CronjobQueue + + def perform(batch_size = 500) + Gitlab::UserActivities.new.each_slice(batch_size) do |batch| + UpdateUserActivityWorker.perform_async(Hash[batch]) + end + end +end diff --git a/app/workers/update_user_activity_worker.rb b/app/workers/update_user_activity_worker.rb new file mode 100644 index 00000000000..b3c2f13aa33 --- /dev/null +++ b/app/workers/update_user_activity_worker.rb @@ -0,0 +1,26 @@ +class UpdateUserActivityWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + def perform(pairs) + pairs = cast_data(pairs) + ids = pairs.keys + conditions = 'WHEN id = ? THEN ? ' * ids.length + + User.where(id: ids). + update_all([ + "last_activity_on = CASE #{conditions} ELSE last_activity_on END", + *pairs.to_a.flatten + ]) + + Gitlab::UserActivities.new.delete(*ids) + end + + private + + def cast_data(pairs) + pairs.each_with_object({}) do |(key, value), new_pairs| + new_pairs[key.to_i] = Time.at(value.to_i).to_s(:db) + end + end +end |