diff options
| author | Sean McGivern <sean@mcgivern.me.uk> | 2017-04-19 14:48:31 +0000 |
|---|---|---|
| committer | Sean McGivern <sean@mcgivern.me.uk> | 2017-04-19 14:48:31 +0000 |
| commit | 40a972057da58dd5c15cf0f0b2952f264dba0076 (patch) | |
| tree | c454dd64d0e8af557ad9bf71337ad5c4014efbef | |
| parent | a9da37434af6d44c5f851affd4bd69b370760e8e (diff) | |
| parent | 5f570690d4cf5635eb41ec43f21bdf21f79c5dda (diff) | |
| download | gitlab-ce-40a972057da58dd5c15cf0f0b2952f264dba0076.tar.gz | |
Merge branch 'usage-ping-port' into 'master'
Usage ping port
Closes #27750
See merge request !10481
62 files changed, 1498 insertions, 50 deletions
diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index d88b7cd0e17..0faf757eaab 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -367,6 +367,9 @@ const ShortcutsBlob = require('./shortcuts_blob'); case 'admin': new Admin(); switch (path[1]) { + case 'cohorts': + new gl.UsagePing(); + break; case 'groups': new UsersSelect(); break; diff --git a/app/assets/javascripts/main.js b/app/assets/javascripts/main.js index c50ec24c818..36616d02bf6 100644 --- a/app/assets/javascripts/main.js +++ b/app/assets/javascripts/main.js @@ -165,6 +165,7 @@ import './syntax_highlight'; import './task_list'; import './todos'; import './tree'; +import './usage_ping'; import './user'; import './user_tabs'; import './username_validator'; diff --git a/app/assets/javascripts/usage_ping.js b/app/assets/javascripts/usage_ping.js new file mode 100644 index 00000000000..fd3af7d7ab6 --- /dev/null +++ b/app/assets/javascripts/usage_ping.js @@ -0,0 +1,15 @@ +function UsagePing() { + const usageDataUrl = $('.usage-data').data('endpoint'); + + $.ajax({ + type: 'GET', + url: usageDataUrl, + dataType: 'html', + success(html) { + $('.usage-data').html(html); + }, + }); +} + +window.gl = window.gl || {}; +window.gl.UsagePing = UsagePing; diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index 515d8e1523b..643993d035e 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -17,6 +17,18 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController end end + def usage_data + respond_to do |format| + format.html do + usage_data = Gitlab::UsageData.data + usage_data_json = params[:pretty] ? JSON.pretty_generate(usage_data) : usage_data.to_json + + render html: Gitlab::Highlight.highlight('payload.json', usage_data_json) + end + format.json { render json: Gitlab::UsageData.to_json } + end + end + def reset_runners_token @application_setting.reset_runners_registration_token! flash[:notice] = 'New runners registration token has been generated!' @@ -135,6 +147,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController :version_check_enabled, :terminal_max_session_time, :polling_interval_multiplier, + :usage_ping_enabled, disabled_oauth_sign_in_sources: [], import_sources: [], diff --git a/app/controllers/admin/cohorts_controller.rb b/app/controllers/admin/cohorts_controller.rb new file mode 100644 index 00000000000..9b77c554908 --- /dev/null +++ b/app/controllers/admin/cohorts_controller.rb @@ -0,0 +1,11 @@ +class Admin::CohortsController < Admin::ApplicationController + def index + if current_application_settings.usage_ping_enabled + cohorts_results = Rails.cache.fetch('cohorts', expires_in: 1.day) do + CohortsService.new.execute + end + + @cohorts = CohortsSerializer.new.represent(cohorts_results) + end + end +end diff --git a/app/controllers/projects/git_http_controller.rb b/app/controllers/projects/git_http_controller.rb index 37f6f637ff0..10adddb4636 100644 --- a/app/controllers/projects/git_http_controller.rb +++ b/app/controllers/projects/git_http_controller.rb @@ -5,6 +5,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController # GET /foo/bar.git/info/refs?service=git-receive-pack (git push) def info_refs if upload_pack? && upload_pack_allowed? + log_user_activity + render_ok elsif receive_pack? && receive_pack_allowed? render_ok @@ -106,4 +108,8 @@ class Projects::GitHttpController < Projects::GitHttpClientController def access_klass @access_klass ||= wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess end + + def log_user_activity + Users::ActivityService.new(user, 'pull').execute + end end diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index d3091a4f8e9..8c6ba4915cd 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -35,6 +35,7 @@ class SessionsController < Devise::SessionsController # hide the signed-in notification flash[:notice] = nil log_audit_event(current_user, with: authentication_method) + log_user_activity(current_user) end end @@ -123,6 +124,10 @@ class SessionsController < Devise::SessionsController for_authentication.security_event end + def log_user_activity(user) + Users::ActivityService.new(user, 'login').execute + end + def load_recaptcha Gitlab::Recaptcha.load_configurations! end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 2961e16f5e0..dd1a6922968 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -238,7 +238,8 @@ class ApplicationSetting < ActiveRecord::Base terminal_max_session_time: 0, two_factor_grace_period: 48, user_default_external: false, - polling_interval_multiplier: 1 + polling_interval_multiplier: 1, + usage_ping_enabled: true } end diff --git a/app/serializers/cohort_activity_month_entity.rb b/app/serializers/cohort_activity_month_entity.rb new file mode 100644 index 00000000000..e6788a8b596 --- /dev/null +++ b/app/serializers/cohort_activity_month_entity.rb @@ -0,0 +1,11 @@ +class CohortActivityMonthEntity < Grape::Entity + include ActionView::Helpers::NumberHelper + + expose :total do |cohort_activity_month| + number_with_delimiter(cohort_activity_month[:total]) + end + + expose :percentage do |cohort_activity_month| + number_to_percentage(cohort_activity_month[:percentage], precision: 0) + end +end diff --git a/app/serializers/cohort_entity.rb b/app/serializers/cohort_entity.rb new file mode 100644 index 00000000000..7cdba5b0484 --- /dev/null +++ b/app/serializers/cohort_entity.rb @@ -0,0 +1,17 @@ +class CohortEntity < Grape::Entity + include ActionView::Helpers::NumberHelper + + expose :registration_month do |cohort| + cohort[:registration_month].strftime('%b %Y') + end + + expose :total do |cohort| + number_with_delimiter(cohort[:total]) + end + + expose :inactive do |cohort| + number_with_delimiter(cohort[:inactive]) + end + + expose :activity_months, using: CohortActivityMonthEntity +end diff --git a/app/serializers/cohorts_entity.rb b/app/serializers/cohorts_entity.rb new file mode 100644 index 00000000000..98f5995ba6f --- /dev/null +++ b/app/serializers/cohorts_entity.rb @@ -0,0 +1,4 @@ +class CohortsEntity < Grape::Entity + expose :months_included + expose :cohorts, using: CohortEntity +end diff --git a/app/serializers/cohorts_serializer.rb b/app/serializers/cohorts_serializer.rb new file mode 100644 index 00000000000..fe9367b13d8 --- /dev/null +++ b/app/serializers/cohorts_serializer.rb @@ -0,0 +1,3 @@ +class CohortsSerializer < AnalyticsGenericSerializer + entity CohortsEntity +end diff --git a/app/services/cohorts_service.rb b/app/services/cohorts_service.rb new file mode 100644 index 00000000000..6781533af28 --- /dev/null +++ b/app/services/cohorts_service.rb @@ -0,0 +1,100 @@ +class CohortsService + MONTHS_INCLUDED = 12 + + def execute + { + months_included: MONTHS_INCLUDED, + cohorts: cohorts + } + end + + # Get an array of hashes that looks like: + # + # [ + # { + # registration_month: Date.new(2017, 3), + # activity_months: [3, 2, 1], + # total: 3 + # inactive: 0 + # }, + # etc. + # + # The `months` array is always from oldest to newest, so it's always + # non-strictly decreasing from left to right. + def cohorts + months = Array.new(MONTHS_INCLUDED) { |i| i.months.ago.beginning_of_month.to_date } + + Array.new(MONTHS_INCLUDED) do + registration_month = months.last + activity_months = running_totals(months, registration_month) + + # Even if no users registered in this month, we always want to have a + # value to fill in the table. + inactive = counts_by_month[[registration_month, nil]].to_i + + months.pop + + { + registration_month: registration_month, + activity_months: activity_months, + total: activity_months.first[:total], + inactive: inactive + } + end + end + + private + + # Calculate a running sum of active users, so users active in later months + # count as active in this month, too. Start with the most recent month first, + # for calculating the running totals, and then reverse for displaying in the + # table. + # + # Each month has a total, and a percentage of the overall total, as keys. + def running_totals(all_months, registration_month) + month_totals = + all_months + .map { |activity_month| counts_by_month[[registration_month, activity_month]] } + .reduce([]) { |result, total| result << result.last.to_i + total.to_i } + .reverse + + overall_total = month_totals.first + + month_totals.map do |total| + { total: total, percentage: total.zero? ? 0 : 100 * total / overall_total } + end + end + + # Get a hash that looks like: + # + # { + # [created_at_month, last_activity_on_month] => count, + # [created_at_month, last_activity_on_month_2] => count_2, + # # etc. + # } + # + # created_at_month can never be nil, but last_activity_on_month can (when a + # user has never logged in, just been created). This covers the last + # MONTHS_INCLUDED months. + def counts_by_month + @counts_by_month ||= + begin + created_at_month = column_to_date('created_at') + last_activity_on_month = column_to_date('last_activity_on') + + User + .where('created_at > ?', MONTHS_INCLUDED.months.ago.end_of_month) + .group(created_at_month, last_activity_on_month) + .reorder("#{created_at_month} ASC", "#{last_activity_on_month} ASC") + .count + end + end + + def column_to_date(column) + if Gitlab::Database.postgresql? + "CAST(DATE_TRUNC('month', #{column}) AS date)" + else + "STR_TO_DATE(DATE_FORMAT(#{column}, '%Y-%m-01'), '%Y-%m-%d')" + end + end +end diff --git a/app/services/event_create_service.rb b/app/services/event_create_service.rb index e24cc66e0fe..0f3a485a3fd 100644 --- a/app/services/event_create_service.rb +++ b/app/services/event_create_service.rb @@ -72,6 +72,8 @@ class EventCreateService def push(project, current_user, push_data) create_event(project, current_user, Event::PUSHED, data: push_data) + + Users::ActivityService.new(current_user, 'push').execute end private diff --git a/app/services/users/activity_service.rb b/app/services/users/activity_service.rb new file mode 100644 index 00000000000..facf21a7f5c --- /dev/null +++ b/app/services/users/activity_service.rb @@ -0,0 +1,22 @@ +module Users + class ActivityService + def initialize(author, activity) + @author = author.respond_to?(:user) ? author.user : author + @activity = activity + end + + def execute + return unless @author && @author.is_a?(User) + + record_activity + end + + private + + def record_activity + Gitlab::UserActivities.record(@author.id) + + Rails.logger.debug("Recorded activity: #{@activity} for User ID: #{@author.id} (username: #{@author.username}") + end + end +end diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index f4ba44096d3..0dc1103eece 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -477,7 +477,7 @@ diagrams in Asciidoc documents using an external PlantUML service. %fieldset - %legend Usage statistics + %legend#usage-statistics Usage statistics .form-group .col-sm-offset-2.col-sm-10 .checkbox @@ -486,6 +486,19 @@ Version check enabled .help-block Let GitLab inform you when an update is available. + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.label :usage_ping_enabled do + = f.check_box :usage_ping_enabled + Usage ping enabled + = link_to icon('question-circle'), help_page_path("user/admin_area/settings/usage_statistics", anchor: "usage-data") + .help-block + Every week GitLab will report license usage back to GitLab, Inc. + Disable this option if you do not want this to occur. To see the + JSON payload that will be sent, visit the + = succeed '.' do + = link_to "Cohorts page", admin_cohorts_path(anchor: 'usage-ping') %fieldset %legend Email diff --git a/app/views/admin/cohorts/_cohorts_table.html.haml b/app/views/admin/cohorts/_cohorts_table.html.haml new file mode 100644 index 00000000000..701a4e62b39 --- /dev/null +++ b/app/views/admin/cohorts/_cohorts_table.html.haml @@ -0,0 +1,28 @@ +.bs-callout.clearfix + %p + User cohorts are shown for the last #{@cohorts[:months_included]} + months. Only users with activity are counted in the cohort total; inactive + users are counted separately. + = link_to icon('question-circle'), help_page_path('user/admin_area/user_cohorts', anchor: 'cohorts'), title: 'About this feature', target: '_blank' + +.table-holder + %table.table + %thead + %tr + %th Registration month + %th Inactive users + %th Cohort total + - @cohorts[:months_included].times do |i| + %th Month #{i} + %tbody + - @cohorts[:cohorts].each do |cohort| + %tr + %td= cohort[:registration_month] + %td= cohort[:inactive] + %td= cohort[:total] + - cohort[:activity_months].each do |activity_month| + %td + - next if cohort[:total] == '0' + = activity_month[:percentage] + %br + = activity_month[:total] diff --git a/app/views/admin/cohorts/_usage_ping.html.haml b/app/views/admin/cohorts/_usage_ping.html.haml new file mode 100644 index 00000000000..73aa95d84f1 --- /dev/null +++ b/app/views/admin/cohorts/_usage_ping.html.haml @@ -0,0 +1,10 @@ +%h2#usage-ping Usage ping + +.bs-callout.clearfix + %p + User cohorts are shown because the usage ping is enabled. The data sent with + this is shown below. To disable this, visit + = succeed '.' do + = link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics') + +%pre.usage-data.js-syntax-highlight.code.highlight{ data: { endpoint: usage_data_admin_application_settings_path(format: :html, pretty: true) } } diff --git a/app/views/admin/cohorts/index.html.haml b/app/views/admin/cohorts/index.html.haml new file mode 100644 index 00000000000..46fe12a5a99 --- /dev/null +++ b/app/views/admin/cohorts/index.html.haml @@ -0,0 +1,16 @@ +- @no_container = true += render "admin/dashboard/head" + +%div{ class: container_class } + - if @cohorts + = render 'cohorts_table' + = render 'usage_ping' + - else + .bs-callout.bs-callout-warning.clearfix + %p + User cohorts are only shown when the + = link_to 'usage ping', help_page_path('user/admin_area/usage_statistics'), target: '_blank' + is enabled. To enable it and see user cohorts, + visit + = succeed '.' do + = link_to 'application settings', admin_application_settings_path(anchor: 'usage-statistics') diff --git a/app/views/admin/dashboard/_head.html.haml b/app/views/admin/dashboard/_head.html.haml index 7893c1dee97..163bd5662b0 100644 --- a/app/views/admin/dashboard/_head.html.haml +++ b/app/views/admin/dashboard/_head.html.haml @@ -27,3 +27,7 @@ = link_to admin_runners_path, title: 'Runners' do %span Runners + = nav_link path: 'cohorts#index' do + = link_to admin_cohorts_path, title: 'Cohorts' do + %span + Cohorts diff --git a/app/workers/gitlab_usage_ping_worker.rb b/app/workers/gitlab_usage_ping_worker.rb new file mode 100644 index 00000000000..2f02235b0ac --- /dev/null +++ b/app/workers/gitlab_usage_ping_worker.rb @@ -0,0 +1,31 @@ +class GitlabUsagePingWorker + LEASE_TIMEOUT = 86400 + + include Sidekiq::Worker + include CronjobQueue + include HTTParty + + def perform + return unless current_application_settings.usage_ping_enabled + + # Multiple Sidekiq workers could run this. We should only do this at most once a day. + return unless try_obtain_lease + + begin + HTTParty.post(url, + body: Gitlab::UsageData.to_json(force_refresh: true), + headers: { 'Content-type' => 'application/json' } + ) + rescue HTTParty::Error => e + Rails.logger.info "Unable to contact GitLab, Inc.: #{e}" + end + end + + def try_obtain_lease + Gitlab::ExclusiveLease.new('gitlab_usage_ping_worker:ping', timeout: LEASE_TIMEOUT).try_obtain + end + + def url + 'https://version.gitlab.com/usage_data' + end +end diff --git a/app/workers/schedule_update_user_activity_worker.rb b/app/workers/schedule_update_user_activity_worker.rb new file mode 100644 index 00000000000..6c2c3e437f3 --- /dev/null +++ b/app/workers/schedule_update_user_activity_worker.rb @@ -0,0 +1,10 @@ +class ScheduleUpdateUserActivityWorker + include Sidekiq::Worker + include CronjobQueue + + def perform(batch_size = 500) + Gitlab::UserActivities.new.each_slice(batch_size) do |batch| + UpdateUserActivityWorker.perform_async(Hash[batch]) + end + end +end diff --git a/app/workers/update_user_activity_worker.rb b/app/workers/update_user_activity_worker.rb new file mode 100644 index 00000000000..b3c2f13aa33 --- /dev/null +++ b/app/workers/update_user_activity_worker.rb @@ -0,0 +1,26 @@ +class UpdateUserActivityWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + def perform(pairs) + pairs = cast_data(pairs) + ids = pairs.keys + conditions = 'WHEN id = ? THEN ? ' * ids.length + + User.where(id: ids). + update_all([ + "last_activity_on = CASE #{conditions} ELSE last_activity_on END", + *pairs.to_a.flatten + ]) + + Gitlab::UserActivities.new.delete(*ids) + end + + private + + def cast_data(pairs) + pairs.each_with_object({}) do |(key, value), new_pairs| + new_pairs[key.to_i] = Time.at(value.to_i).to_s(:db) + end + end +end diff --git a/changelogs/unreleased/usage-ping-port.yml b/changelogs/unreleased/usage-ping-port.yml new file mode 100644 index 00000000000..4f135100fce --- /dev/null +++ b/changelogs/unreleased/usage-ping-port.yml @@ -0,0 +1,4 @@ +--- +title: Add usage ping to CE +merge_request: +author: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 4c9d829aa9f..87bf48a3dcd 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -110,6 +110,14 @@ class Settings < Settingslogic URI.parse(url_without_path).host end + + # Random cron time every Sunday to load balance usage pings + def cron_random_weekly_time + hour = rand(24) + minute = rand(60) + + "#{minute} #{hour} * * 0" + end end end @@ -204,8 +212,8 @@ Settings.gitlab['email_from'] ||= ENV['GITLAB_EMAIL_FROM'] || "gitlab@#{Settings Settings.gitlab['email_display_name'] ||= ENV['GITLAB_EMAIL_DISPLAY_NAME'] || 'GitLab' Settings.gitlab['email_reply_to'] ||= ENV['GITLAB_EMAIL_REPLY_TO'] || "noreply@#{Settings.gitlab.host}" Settings.gitlab['email_subject_suffix'] ||= ENV['GITLAB_EMAIL_SUBJECT_SUFFIX'] || "" -Settings.gitlab['base_url'] ||= Settings.send(:build_base_gitlab_url) -Settings.gitlab['url'] ||= Settings.send(:build_gitlab_url) +Settings.gitlab['base_url'] ||= Settings.__send__(:build_base_gitlab_url) +Settings.gitlab['url'] ||= Settings.__send__(:build_gitlab_url) Settings.gitlab['user'] ||= 'git' Settings.gitlab['user_home'] ||= begin Etc.getpwnam(Settings.gitlab['user']).dir @@ -215,7 +223,7 @@ end Settings.gitlab['time_zone'] ||= nil Settings.gitlab['signup_enabled'] ||= true if Settings.gitlab['signup_enabled'].nil? Settings.gitlab['signin_enabled'] ||= true if Settings.gitlab['signin_enabled'].nil? -Settings.gitlab['restricted_visibility_levels'] = Settings.send(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], []) +Settings.gitlab['restricted_visibility_levels'] = Settings.__send__(:verify_constant_array, Gitlab::VisibilityLevel, Settings.gitlab['restricted_visibility_levels'], []) Settings.gitlab['username_changing_enabled'] = true if Settings.gitlab['username_changing_enabled'].nil? Settings.gitlab['issue_closing_pattern'] = '((?:[Cc]los(?:e[sd]?|ing)|[Ff]ix(?:e[sd]|ing)?|[Rr]esolv(?:e[sd]?|ing))(:?) +(?:(?:issues? +)?%{issue_ref}(?:(?:, *| +and +)?)|([A-Z][A-Z0-9_]+-\d+))+)' if Settings.gitlab['issue_closing_pattern'].nil? Settings.gitlab['default_projects_features'] ||= {} @@ -228,7 +236,7 @@ Settings.gitlab.default_projects_features['wiki'] = true if Settin Settings.gitlab.default_projects_features['snippets'] = true if Settings.gitlab.default_projects_features['snippets'].nil? Settings.gitlab.default_projects_features['builds'] = true if Settings.gitlab.default_projects_features['builds'].nil? Settings.gitlab.default_projects_features['container_registry'] = true if Settings.gitlab.default_projects_features['container_registry'].nil? -Settings.gitlab.default_projects_features['visibility_level'] = Settings.send(:verify_constant, Gitlab::VisibilityLevel, Settings.gitlab.default_projects_features['visibility_level'], Gitlab::VisibilityLevel::PRIVATE) +Settings.gitlab.default_projects_features['visibility_level'] = Settings.__send__(:verify_constant, Gitlab::VisibilityLevel, Settings.gitlab.default_projects_features['visibility_level'], Gitlab::VisibilityLevel::PRIVATE) Settings.gitlab['domain_whitelist'] ||= [] Settings.gitlab['import_sources'] ||= %w[github bitbucket gitlab google_code fogbugz git gitlab_project gitea] Settings.gitlab['trusted_proxies'] ||= [] @@ -242,7 +250,7 @@ Settings.gitlab_ci['shared_runners_enabled'] = true if Settings.gitlab_ci['share Settings.gitlab_ci['all_broken_builds'] = true if Settings.gitlab_ci['all_broken_builds'].nil? Settings.gitlab_ci['add_pusher'] = false if Settings.gitlab_ci['add_pusher'].nil? Settings.gitlab_ci['builds_path'] = Settings.absolute(Settings.gitlab_ci['builds_path'] || "builds/") -Settings.gitlab_ci['url'] ||= Settings.send(:build_gitlab_ci_url) +Settings.gitlab_ci['url'] ||= Settings.__send__(:build_gitlab_ci_url) # # Reply by email @@ -281,7 +289,7 @@ Settings.pages['https'] = false if Settings.pages['https'].nil? Settings.pages['host'] ||= "example.com" Settings.pages['port'] ||= Settings.pages.https ? 443 : 80 Settings.pages['protocol'] ||= Settings.pages.https ? "https" : "http" -Settings.pages['url'] ||= Settings.send(:build_pages_url) +Settings.pages['url'] ||= Settings.__send__(:build_pages_url) Settings.pages['external_http'] ||= false unless Settings.pages['external_http'].present? Settings.pages['external_https'] ||= false unless Settings.pages['external_https'].present? @@ -355,6 +363,14 @@ Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['job_class'] = 'Rem Settings.cron_jobs['stuck_import_jobs_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_import_jobs_worker']['cron'] ||= '15 * * * *' Settings.cron_jobs['stuck_import_jobs_worker']['job_class'] = 'StuckImportJobsWorker' +Settings.cron_jobs['gitlab_usage_ping_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['gitlab_usage_ping_worker']['cron'] ||= Settings.__send__(:cron_random_weekly_time) +Settings.cron_jobs['gitlab_usage_ping_worker']['job_class'] = 'GitlabUsagePingWorker' + +# Every day at 00:30 +Settings.cron_jobs['schedule_update_user_activity_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['schedule_update_user_activity_worker']['cron'] ||= '30 0 * * *' +Settings.cron_jobs['schedule_update_user_activity_worker']['job_class'] = 'ScheduleUpdateUserActivityWorker' # # GitLab Shell @@ -369,7 +385,7 @@ Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host Settings.gitlab_shell['ssh_port'] ||= 22 Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user -Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix) +Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.__send__(:build_gitlab_shell_ssh_path_prefix) # # Repositories diff --git a/config/routes/admin.rb b/config/routes/admin.rb index 486ce3c5c87..52ba10604d4 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -91,6 +91,7 @@ namespace :admin do resource :application_settings, only: [:show, :update] do resources :services, only: [:index, :edit, :update] + get :usage_data put :reset_runners_token put :reset_health_check_token put :clear_repository_check_states @@ -105,6 +106,8 @@ namespace :admin do end end + resources :cohorts, only: :index + resources :builds, only: :index do collection do post :cancel_all diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 9d2066a6490..bf8964d7f68 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -53,3 +53,4 @@ - [default, 1] - [pages, 1] - [system_hook_push, 1] + - [update_user_activity, 1] diff --git a/db/migrate/20160713222618_add_usage_ping_to_application_settings.rb b/db/migrate/20160713222618_add_usage_ping_to_application_settings.rb new file mode 100644 index 00000000000..a7f76cc626e --- /dev/null +++ b/db/migrate/20160713222618_add_usage_ping_to_application_settings.rb @@ -0,0 +1,9 @@ +class AddUsagePingToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :application_settings, :usage_ping_enabled, :boolean, default: true, null: false + end +end diff --git a/db/migrate/20161007073613_create_user_activities.rb b/db/migrate/20161007073613_create_user_activities.rb new file mode 100644 index 00000000000..1d694e777a1 --- /dev/null +++ b/db/migrate/20161007073613_create_user_activities.rb @@ -0,0 +1,7 @@ +class CreateUserActivities < ActiveRecord::Migration + DOWNTIME = false + + # This migration is a no-op. It just exists to match EE. + def change + end +end diff --git a/db/migrate/20170307125949_add_last_activity_on_to_users.rb b/db/migrate/20170307125949_add_last_activity_on_to_users.rb new file mode 100644 index 00000000000..0100836b473 --- /dev/null +++ b/db/migrate/20170307125949_add_last_activity_on_to_users.rb @@ -0,0 +1,9 @@ +class AddLastActivityOnToUsers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :users, :last_activity_on, :date + end +end diff --git a/db/migrate/20170328010804_add_uuid_to_application_settings.rb b/db/migrate/20170328010804_add_uuid_to_application_settings.rb new file mode 100644 index 00000000000..5dfcc751c7b --- /dev/null +++ b/db/migrate/20170328010804_add_uuid_to_application_settings.rb @@ -0,0 +1,16 @@ +class AddUuidToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column :application_settings, :uuid, :string + execute("UPDATE application_settings SET uuid = #{quote(SecureRandom.uuid)}") + end + + def down + remove_column :application_settings, :uuid + end +end diff --git a/db/post_migrate/20161128170531_drop_user_activities_table.rb b/db/post_migrate/20161128170531_drop_user_activities_table.rb new file mode 100644 index 00000000000..00bc0c73015 --- /dev/null +++ b/db/post_migrate/20161128170531_drop_user_activities_table.rb @@ -0,0 +1,9 @@ +class DropUserActivitiesTable < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + # This migration is a no-op. It just exists to match EE. + def change + end +end diff --git a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb new file mode 100644 index 00000000000..9ad36482c8a --- /dev/null +++ b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb @@ -0,0 +1,87 @@ +class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + DOWNTIME = false + USER_ACTIVITY_SET_KEY = 'user/activities'.freeze + ACTIVITIES_PER_PAGE = 100 + TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED = Time.utc(2016, 12, 1) + + def up + return if activities_count(TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED, Time.now).zero? + + day = Time.at(activities(TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED, Time.now).first.second) + + transaction do + while day <= Time.now.utc.tomorrow + persist_last_activity_on(day: day) + day = day.tomorrow + end + end + end + + def down + # This ensures we don't lock all users for the duration of the migration. + update_column_in_batches(:users, :last_activity_on, nil) do |table, query| + query.where(table[:last_activity_on].not_eq(nil)) + end + end + + private + + def persist_last_activity_on(day:, page: 1) + activities_count = activities_count(day.at_beginning_of_day, day.at_end_of_day) + + return if activities_count.zero? + + activities = activities(day.at_beginning_of_day, day.at_end_of_day, page: page) + + update_sql = + Arel::UpdateManager.new(ActiveRecord::Base). + table(users_table). + set(users_table[:last_activity_on] => day.to_date). + where(users_table[:username].in(activities.map(&:first))). + to_sql + + connection.exec_update(update_sql, self.class.name, []) + + unless last_page?(page, activities_count) + persist_last_activity_on(day: day, page: page + 1) + end + end + + def users_table + @users_table ||= Arel::Table.new(:users) + end + + def activities(from, to, page: 1) + Gitlab::Redis.with do |redis| + redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i, + with_scores: true, + limit: limit(page)) + end + end + + def activities_count(from, to) + Gitlab::Redis.with do |redis| + redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i) + end + end + + def limit(page) + [offset(page), ACTIVITIES_PER_PAGE] + end + + def total_pages(count) + (count.to_f / ACTIVITIES_PER_PAGE).ceil + end + + def last_page?(page, count) + page >= total_pages(count) + end + + def offset(page) + (page - 1) * ACTIVITIES_PER_PAGE + end +end diff --git a/db/schema.rb b/db/schema.rb index 26ec6367bd0..d46b7564958 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -116,6 +116,8 @@ ActiveRecord::Schema.define(version: 20170418103908) do t.integer "unique_ips_limit_time_window" t.boolean "unique_ips_limit_enabled", default: false, null: false t.decimal "polling_interval_multiplier", default: 1.0, null: false + t.boolean "usage_ping_enabled", default: true, null: false + t.string "uuid" end create_table "audit_events", force: :cascade do |t| @@ -1301,6 +1303,7 @@ ActiveRecord::Schema.define(version: 20170418103908) do t.string "organization" t.boolean "authorized_projects_populated" t.boolean "ghost" + t.date "last_activity_on" t.boolean "notified_of_own_activity" t.boolean "require_two_factor_authentication_from_group", default: false, null: false t.integer "two_factor_grace_period", default: 48, null: false diff --git a/doc/README.md b/doc/README.md index 7703c64e152..9e6a5b4ed44 100644 --- a/doc/README.md +++ b/doc/README.md @@ -69,6 +69,7 @@ All technical content published by GitLab lives in the documentation, including: - [Sidekiq Troubleshooting](administration/troubleshooting/sidekiq.md) Debug when Sidekiq appears hung and is not processing jobs. - [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed. - [Update](update/README.md) Update guides to upgrade your installation. +- [User cohorts](user/admin_area/user_cohorts.md) View user activity over time. - [Web terminals](administration/integration/terminal.md) Provide terminal access to environments from within GitLab. - [Welcome message](customization/welcome_message.md) Add a custom welcome message to the sign-in page. diff --git a/doc/api/users.md b/doc/api/users.md index 2ada4d09c84..a79d31d19fa 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -72,6 +72,7 @@ GET /users "organization": "", "last_sign_in_at": "2012-06-01T11:41:01Z", "confirmed_at": "2012-05-23T09:05:22Z", + "last_activity_on": "2012-05-23", "color_scheme_id": 2, "projects_limit": 100, "current_sign_in_at": "2012-06-02T06:36:55Z", @@ -104,6 +105,7 @@ GET /users "organization": "", "last_sign_in_at": null, "confirmed_at": "2012-05-30T16:53:06.148Z", + "last_activity_on": "2012-05-23", "color_scheme_id": 3, "projects_limit": 100, "current_sign_in_at": "2014-03-19T17:54:13Z", @@ -196,6 +198,7 @@ Parameters: "organization": "", "last_sign_in_at": "2012-06-01T11:41:01Z", "confirmed_at": "2012-05-23T09:05:22Z", + "last_activity_on": "2012-05-23", "color_scheme_id": 2, "projects_limit": 100, "current_sign_in_at": "2012-06-02T06:36:55Z", @@ -320,6 +323,7 @@ GET /user "organization": "", "last_sign_in_at": "2012-06-01T11:41:01Z", "confirmed_at": "2012-05-23T09:05:22Z", + "last_activity_on": "2012-05-23", "color_scheme_id": 2, "projects_limit": 100, "current_sign_in_at": "2012-06-02T06:36:55Z", @@ -365,6 +369,7 @@ GET /user "organization": "", "last_sign_in_at": "2012-06-01T11:41:01Z", "confirmed_at": "2012-05-23T09:05:22Z", + "last_activity_on": "2012-05-23", "color_scheme_id": 2, "projects_limit": 100, "current_sign_in_at": "2012-06-02T06:36:55Z", @@ -986,3 +991,55 @@ Parameters: | --------- | ---- | -------- | ----------- | | `user_id` | integer | yes | The ID of the user | | `impersonation_token_id` | integer | yes | The ID of the impersonation token | + +### Get user activities (admin only) + +>**Note:** This API endpoint is only available on 8.15 (EE) and 9.1 (CE) and above. + +Get the last activity date for all users, sorted from oldest to newest. + +The activities that update the timestamp are: + + - Git HTTP/SSH activities (such as clone, push) + - User logging in into GitLab + +By default, it shows the activity for all users in the last 6 months, but this can be +amended by using the `from` parameter. + +``` +GET /user/activities +``` + +Parameters: + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `from` | string | no | Date string in the format YEAR-MONTH-DAY, e.g. `2016-03-11`. Defaults to 6 months ago. | + +```bash +curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/user/activities +``` + +Example response: + +```json +[ + { + "username": "user1", + "last_activity_on": "2015-12-14", + "last_activity_at": "2015-12-14" + }, + { + "username": "user2", + "last_activity_on": "2015-12-15", + "last_activity_at": "2015-12-15" + }, + { + "username": "user3", + "last_activity_on": "2015-12-16", + "last_activity_at": "2015-12-16" + } +] +``` + +Please note that `last_activity_at` is deprecated, please use `last_activity_on`. diff --git a/doc/user/admin_area/img/cohorts.png b/doc/user/admin_area/img/cohorts.png Binary files differnew file mode 100644 index 00000000000..8bae7faff07 --- /dev/null +++ b/doc/user/admin_area/img/cohorts.png diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md new file mode 100644 index 00000000000..c3f3179d99e --- /dev/null +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -0,0 +1,102 @@ +# Usage statistics + +GitLab Inc. will periodically collect information about your instance in order +to perform various actions. + +All statistics are opt-in and you can always disable them from the admin panel. + +## Version check + +GitLab can inform you when an update is available and the importance of it. + +No information other than the GitLab version and the instance's domain name +are collected. + +In the **Overview** tab you can see if your GitLab version is up to date. There +are three cases: 1) you are up to date (green), 2) there is an update available +(yellow) and 3) your version is vulnerable and a security fix is released (red). + +In any case, you will see a message informing you of the state and the +importance of the update. + +If enabled, the version status will also be shown in the help page (`/help`) +for all signed in users. + +## Usage ping + +> [Introduced][ee-557] in GitLab Enterprise Edition 8.10. More statistics +[were added][ee-735] in GitLab Enterprise Edition +8.12. [Moved to GitLab Community Edition][ce-23361] in 9.1. + +GitLab Inc. can collect non-sensitive information about how GitLab users +use their GitLab instance upon the activation of a ping feature +located in the admin panel (`/admin/application_settings`). + +You can see the **exact** JSON payload that your instance sends to GitLab +in the "Usage statistics" section of the admin panel. + +Nothing qualitative is collected. Only quantitative. That means no project +names, author names, comment bodies, names of labels, etc. + +The usage ping is sent in order for GitLab Inc. to have a better understanding +of how our users use our product, and to be more data-driven when creating or +changing features. + +The total number of the following is sent back to GitLab Inc.: + +- Comments +- Groups +- Users +- Projects +- Issues +- Labels +- CI builds +- Snippets +- Milestones +- Todos +- Pushes +- Merge requests +- Environments +- Triggers +- Deploy keys +- Pages +- Project Services +- Projects using the Prometheus service +- Issue Boards +- CI Runners +- Deployments +- Geo Nodes +- LDAP Groups +- LDAP Keys +- LDAP Users +- LFS objects +- Protected branches +- Releases +- Remote mirrors +- Uploads +- Web hooks + +Also, we track if you've installed Mattermost with GitLab. +For example: `"mattermost_enabled":true"`. + +More data will be added over time. The goal of this ping is to be as light as +possible, so it won't have any performance impact on your installation when +the calculation is made. + +### Deactivate the usage ping + +By default, usage ping is opt-out. If you want to deactivate this feature, go to +the Settings page of your administration panel and uncheck the Usage ping +checkbox. + +## Privacy policy + +GitLab Inc. does **not** collect any sensitive information, like project names +or the content of the comments. GitLab Inc. does not disclose or otherwise make +available any of the data collected on a customer specific basis. + +Read more about this in the [Privacy policy](https://about.gitlab.com/privacy). + +[ee-557]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/557 +[ee-735]: https://gitlab.com/gitlab-org/gitlab-ee/merge_requests/735 +[ce-23361]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23361 diff --git a/doc/user/admin_area/user_cohorts.md b/doc/user/admin_area/user_cohorts.md new file mode 100644 index 00000000000..1671487bc8c --- /dev/null +++ b/doc/user/admin_area/user_cohorts.md @@ -0,0 +1,30 @@ +# Cohorts + +> **Notes:** +- [Introduced][ce-23361] in GitLab 9.1. + +As a benefit of having the [usage ping active](settings/usage_statistics.md), +GitLab lets you analyze the users' activities of your GitLab installation. +Under `/admin/cohorts`, when the usage ping is active, GitLab will show the +monthly cohorts of new users and their activities over time. + +How do we read the user cohorts table? Let's take an example with the following +user cohorts. + + + +For the cohort of June 2016, 163 users have been created on this server. One +month later, in July 2016, 155 users (or 95% of the June cohort) are still +active. Two months later, 139 users (or 85%) are still active. 9 months later, +we can see that only 6% of this cohort are still active. + +How do we measure the activity of users? GitLab considers a user active if: +* the user signs in +* the user has Git activity (whether push or pull). + +### Setup + +1. Activate the usage ping as defined [in the documentation](settings/usage_statistics.md) +2. Go to `/admin/cohorts` to see the user cohorts of the server + +[ce-23361]: https://gitlab.com/gitlab-org/gitlab-ce/issues/23361 diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9919762cd82..64ab6f01eb5 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -18,6 +18,12 @@ module API expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end + class UserActivity < Grape::Entity + expose :username + expose :last_activity_on + expose :last_activity_on, as: :last_activity_at # Back-compat + end + class Identity < Grape::Entity expose :provider, :extern_uid end @@ -25,6 +31,7 @@ module API class UserPublic < User expose :last_sign_in_at expose :confirmed_at + expose :last_activity_on expose :email expose :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 810e5063996..718f936a1fc 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -60,6 +60,12 @@ module API rescue JSON::ParserError {} end + + def log_user_activity(actor) + commands = Gitlab::GitAccess::DOWNLOAD_COMMANDS + + ::Users::ActivityService.new(actor, 'Git SSH').execute if commands.include?(params[:action]) + end end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 215bc03d0e9..5b48ee8665f 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -40,6 +40,8 @@ module API response = { status: access_status.status, message: access_status.message } if access_status.status + log_user_activity(actor) + # Return the repository full path so that gitlab-shell has it when # handling ssh commands response[:repository_path] = diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 05423c17449..244725bb292 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -35,7 +35,7 @@ module API optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' optional :labels, type: String, desc: 'Comma-separated list of label names' - optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :due_date, type: String, desc: 'Date string in the format YEAR-MONTH-DAY' optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' end diff --git a/lib/api/users.rb b/lib/api/users.rb index eedc59f8636..9e0faff6c05 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -534,6 +534,21 @@ module API email.destroy current_user.update_secondary_emails! end + + desc 'Get a list of user activities' + params do + optional :from, type: DateTime, default: 6.months.ago, desc: 'Date string in the format YEAR-MONTH-DAY' + use :pagination + end + get "activities" do + authenticated_as_admin! + + activities = User. + where(User.arel_table[:last_activity_on].gteq(params[:from])). + reorder(last_activity_on: :asc) + + present paginate(activities), with: Entities::UserActivity + end end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb new file mode 100644 index 00000000000..6aca6db3123 --- /dev/null +++ b/lib/gitlab/usage_data.rb @@ -0,0 +1,65 @@ +module Gitlab + class UsageData + include Gitlab::CurrentSettings + + class << self + def data(force_refresh: false) + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + end + + def uncached_data + license_usage_data.merge(system_usage_data) + end + + def to_json(force_refresh: false) + data(force_refresh: force_refresh).to_json + end + + def system_usage_data + { + counts: { + boards: Board.count, + ci_builds: ::Ci::Build.count, + ci_pipelines: ::Ci::Pipeline.count, + ci_runners: ::Ci::Runner.count, + ci_triggers: ::Ci::Trigger.count, + deploy_keys: DeployKey.count, + deployments: Deployment.count, + environments: Environment.count, + groups: Group.count, + issues: Issue.count, + keys: Key.count, + labels: Label.count, + lfs_objects: LfsObject.count, + merge_requests: MergeRequest.count, + milestones: Milestone.count, + notes: Note.count, + pages_domains: PagesDomain.count, + projects: Project.count, + projects_prometheus_active: PrometheusService.active.count, + protected_branches: ProtectedBranch.count, + releases: Release.count, + services: Service.where(active: true).count, + snippets: Snippet.count, + todos: Todo.count, + uploads: Upload.count, + web_hooks: WebHook.count + } + } + end + + def license_usage_data + usage_data = { + uuid: current_application_settings.uuid, + version: Gitlab::VERSION, + active_user_count: User.active.count, + recorded_at: Time.now, + mattermost_enabled: Gitlab.config.mattermost.enabled, + edition: 'CE' + } + + usage_data + end + end + end +end diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb new file mode 100644 index 00000000000..eb36ab9fded --- /dev/null +++ b/lib/gitlab/user_activities.rb @@ -0,0 +1,34 @@ +module Gitlab + class UserActivities + include Enumerable + + KEY = 'users:activities'.freeze + BATCH_SIZE = 500 + + def self.record(key, time = Time.now) + Gitlab::Redis.with do |redis| + redis.hset(KEY, key, time.to_i) + end + end + + def delete(*keys) + Gitlab::Redis.with do |redis| + redis.hdel(KEY, keys) + end + end + + def each + cursor = 0 + loop do + cursor, pairs = + Gitlab::Redis.with do |redis| + redis.hscan(KEY, cursor, count: BATCH_SIZE) + end + + Hash[pairs].each { |pair| yield pair } + + break if cursor == '0' + end + end + end +end diff --git a/spec/controllers/admin/application_settings_controller_spec.rb b/spec/controllers/admin/application_settings_controller_spec.rb index 5dd8f66343f..2565622f8df 100644 --- a/spec/controllers/admin/application_settings_controller_spec.rb +++ b/spec/controllers/admin/application_settings_controller_spec.rb @@ -3,12 +3,49 @@ require 'spec_helper' describe Admin::ApplicationSettingsController do include StubENV + let(:group) { create(:group) } + let(:project) { create(:project, namespace: group) } let(:admin) { create(:admin) } + let(:user) { create(:user)} before do stub_env('IN_MEMORY_APPLICATION_SETTINGS', 'false') end + describe 'GET #usage_data with no access' do + before do + sign_in(user) + end + + it 'returns 404' do + get :usage_data, format: :html + + expect(response.status).to eq(404) + end + end + + describe 'GET #usage_data' do + before do + sign_in(admin) + end + + it 'returns HTML data' do + get :usage_data, format: :html + + expect(response.body).to start_with('<span') + expect(response.status).to eq(200) + end + + it 'returns JSON data' do + get :usage_data, format: :json + + body = JSON.parse(response.body) + expect(body["version"]).to eq(Gitlab::VERSION) + expect(body).to include('counts') + expect(response.status).to eq(200) + end + end + describe 'PUT #update' do before do sign_in(admin) diff --git a/spec/controllers/sessions_controller_spec.rb b/spec/controllers/sessions_controller_spec.rb index 9c16a7bc08b..038132cffe0 100644 --- a/spec/controllers/sessions_controller_spec.rb +++ b/spec/controllers/sessions_controller_spec.rb @@ -16,7 +16,9 @@ describe SessionsController do end end - context 'when using valid password' do + context 'when using valid password', :redis do + include UserActivitiesHelpers + let(:user) { create(:user) } it 'authenticates user correctly' do @@ -37,6 +39,12 @@ describe SessionsController do subject.sign_out user end end + + it 'updates the user activity' do + expect do + post(:create, user: { login: user.username, password: user.password }) + end.to change { user_activity(user) } + end end end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb new file mode 100644 index 00000000000..7f21288cf88 --- /dev/null +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -0,0 +1,70 @@ +require 'spec_helper' + +describe Gitlab::UsageData do + let!(:project) { create(:empty_project) } + let!(:project2) { create(:empty_project) } + let!(:board) { create(:board, project: project) } + + describe '#data' do + subject { Gitlab::UsageData.data } + + it "gathers usage data" do + expect(subject.keys).to match_array(%i( + active_user_count + counts + recorded_at + mattermost_enabled + edition + version + uuid + )) + end + + it "gathers usage counts" do + count_data = subject[:counts] + + expect(count_data[:boards]).to eq(1) + expect(count_data[:projects]).to eq(2) + + expect(count_data.keys).to match_array(%i( + boards + ci_builds + ci_pipelines + ci_runners + ci_triggers + deploy_keys + deployments + environments + groups + issues + keys + labels + lfs_objects + merge_requests + milestones + notes + projects + projects_prometheus_active + pages_domains + protected_branches + releases + services + snippets + todos + uploads + web_hooks + )) + end + end + + describe '#license_usage_data' do + subject { Gitlab::UsageData.license_usage_data } + + it "gathers license data" do + expect(subject[:uuid]).to eq(current_application_settings.uuid) + expect(subject[:version]).to eq(Gitlab::VERSION) + expect(subject[:active_user_count]).to eq(User.active.count) + expect(subject[:recorded_at]).to be_a(Time) + end + end +end diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb new file mode 100644 index 00000000000..187d88c8c58 --- /dev/null +++ b/spec/lib/gitlab/user_activities_spec.rb @@ -0,0 +1,127 @@ +require 'spec_helper' + +describe Gitlab::UserActivities, :redis, lib: true do + let(:now) { Time.now } + + describe '.record' do + context 'with no time given' do + it 'uses Time.now and records an activity in Redis' do + Timecop.freeze do + now # eager-load now + described_class.record(42) + end + + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + end + end + end + + context 'with a time given' do + it 'uses the given time and records an activity in Redis' do + described_class.record(42, now) + + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + end + end + end + end + + describe '.delete' do + context 'with a single key' do + context 'and key exists' do + it 'removes the pair from Redis' do + described_class.record(42, now) + + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + end + + subject.delete(42) + + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + end + end + end + + context 'and key does not exist' do + it 'removes the pair from Redis' do + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + end + + subject.delete(42) + + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + end + end + end + end + + context 'with multiple keys' do + context 'and all keys exist' do + it 'removes the pair from Redis' do + described_class.record(41, now) + described_class.record(42, now) + + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) + end + + subject.delete(41, 42) + + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + end + end + end + + context 'and some keys does not exist' do + it 'removes the existing pair from Redis' do + described_class.record(42, now) + + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) + end + + subject.delete(41, 42) + + Gitlab::Redis.with do |redis| + expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) + end + end + end + end + end + + describe 'Enumerable' do + before do + described_class.record(40, now) + described_class.record(41, now) + described_class.record(42, now) + end + + it 'allows to read the activities sequentially' do + expected = { '40' => now.to_i.to_s, '41' => now.to_i.to_s, '42' => now.to_i.to_s } + + actual = described_class.new.each_with_object({}) do |(key, time), actual| + actual[key] = time + end + + expect(actual).to eq(expected) + end + + context 'with many records' do + before do + 1_000.times { |i| described_class.record(i, now) } + end + + it 'is possible to loop through all the records' do + expect(described_class.new.count).to eq(1_000) + end + end + end +end diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb new file mode 100644 index 00000000000..1db9bc002ae --- /dev/null +++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb @@ -0,0 +1,49 @@ +# encoding: utf-8 + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb') + +describe MigrateUserActivitiesToUsersLastActivityOn, :redis do + let(:migration) { described_class.new } + let!(:user_active_1) { create(:user) } + let!(:user_active_2) { create(:user) } + + def record_activity(user, time) + Gitlab::Redis.with do |redis| + redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) + end + end + + around do |example| + Timecop.freeze { example.run } + end + + before do + record_activity(user_active_1, described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 2.months) + record_activity(user_active_2, described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 3.months) + mute_stdout { migration.up } + end + + describe '#up' do + it 'fills last_activity_on from the legacy Redis Sorted Set' do + expect(user_active_1.reload.last_activity_on).to eq((described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 2.months).to_date) + expect(user_active_2.reload.last_activity_on).to eq((described_class::TIME_WHEN_ACTIVITY_SET_WAS_INTRODUCED + 3.months).to_date) + end + end + + describe '#down' do + it 'sets last_activity_on to NULL for all users' do + mute_stdout { migration.down } + + expect(user_active_1.reload.last_activity_on).to be_nil + expect(user_active_2.reload.last_activity_on).to be_nil + end + end + + def mute_stdout + orig_stdout = $stdout + $stdout = StringIO.new + yield + $stdout = orig_stdout + end +end diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index 4be67df5a00..3d6010ede73 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -147,10 +147,15 @@ describe API::Internal, api: true do end end - describe "POST /internal/allowed" do + describe "POST /internal/allowed", :redis do context "access granted" do before do project.team << [user, :developer] + Timecop.freeze + end + + after do + Timecop.return end context 'with env passed as a JSON' do @@ -176,6 +181,7 @@ describe API::Internal, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_truthy expect(json_response["repository_path"]).to eq(project.wiki.repository.path_to_repo) + expect(user).not_to have_an_activity_record end end @@ -186,6 +192,7 @@ describe API::Internal, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_truthy expect(json_response["repository_path"]).to eq(project.wiki.repository.path_to_repo) + expect(user).to have_an_activity_record end end @@ -196,6 +203,7 @@ describe API::Internal, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_truthy expect(json_response["repository_path"]).to eq(project.repository.path_to_repo) + expect(user).to have_an_activity_record end end @@ -206,6 +214,7 @@ describe API::Internal, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_truthy expect(json_response["repository_path"]).to eq(project.repository.path_to_repo) + expect(user).not_to have_an_activity_record end context 'project as /namespace/project' do @@ -241,6 +250,7 @@ describe API::Internal, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_falsey + expect(user).not_to have_an_activity_record end end @@ -250,6 +260,7 @@ describe API::Internal, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_falsey + expect(user).not_to have_an_activity_record end end end @@ -267,6 +278,7 @@ describe API::Internal, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_falsey + expect(user).not_to have_an_activity_record end end @@ -276,6 +288,7 @@ describe API::Internal, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_falsey + expect(user).not_to have_an_activity_record end end end diff --git a/spec/requests/api/users_spec.rb b/spec/requests/api/users_spec.rb index f793c0db2f3..ea9b886e995 100644 --- a/spec/requests/api/users_spec.rb +++ b/spec/requests/api/users_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper' -describe API::Users, api: true do +describe API::Users, api: true do include ApiHelpers - let(:user) { create(:user) } + let(:user) { create(:user) } let(:admin) { create(:admin) } - let(:key) { create(:key, user: user) } - let(:email) { create(:email, user: user) } + let(:key) { create(:key, user: user) } + let(:email) { create(:email, user: user) } let(:omniauth_user) { create(:omniauth_user) } let(:ldap_user) { create(:omniauth_user, provider: 'ldapmain') } let(:ldap_blocked_user) { create(:omniauth_user, provider: 'ldapmain', state: 'ldap_blocked') } @@ -129,7 +129,7 @@ describe API::Users, api: true do end describe "POST /users" do - before{ admin } + before { admin } it "creates user" do expect do @@ -214,9 +214,9 @@ describe API::Users, api: true do it "does not create user with invalid email" do post api('/users', admin), - email: 'invalid email', - password: 'password', - name: 'test' + email: 'invalid email', + password: 'password', + name: 'test' expect(response).to have_http_status(400) end @@ -242,12 +242,12 @@ describe API::Users, api: true do it 'returns 400 error if user does not validate' do post api('/users', admin), - password: 'pass', - email: 'test@example.com', - username: 'test!', - name: 'test', - bio: 'g' * 256, - projects_limit: -1 + password: 'pass', + email: 'test@example.com', + username: 'test!', + name: 'test', + bio: 'g' * 256, + projects_limit: -1 expect(response).to have_http_status(400) expect(json_response['message']['password']). to eq(['is too short (minimum is 8 characters)']) @@ -267,19 +267,19 @@ describe API::Users, api: true do context 'with existing user' do before do post api('/users', admin), - email: 'test@example.com', - password: 'password', - username: 'test', - name: 'foo' + email: 'test@example.com', + password: 'password', + username: 'test', + name: 'foo' end it 'returns 409 conflict error if user with same email exists' do expect do post api('/users', admin), - name: 'foo', - email: 'test@example.com', - password: 'password', - username: 'foo' + name: 'foo', + email: 'test@example.com', + password: 'password', + username: 'foo' end.to change { User.count }.by(0) expect(response).to have_http_status(409) expect(json_response['message']).to eq('Email has already been taken') @@ -288,10 +288,10 @@ describe API::Users, api: true do it 'returns 409 conflict error if same username exists' do expect do post api('/users', admin), - name: 'foo', - email: 'foo@example.com', - password: 'password', - username: 'test' + name: 'foo', + email: 'foo@example.com', + password: 'password', + username: 'test' end.to change { User.count }.by(0) expect(response).to have_http_status(409) expect(json_response['message']).to eq('Username has already been taken') @@ -416,12 +416,12 @@ describe API::Users, api: true do it 'returns 400 error if user does not validate' do put api("/users/#{user.id}", admin), - password: 'pass', - email: 'test@example.com', - username: 'test!', - name: 'test', - bio: 'g' * 256, - projects_limit: -1 + password: 'pass', + email: 'test@example.com', + username: 'test!', + name: 'test', + bio: 'g' * 256, + projects_limit: -1 expect(response).to have_http_status(400) expect(json_response['message']['password']). to eq(['is too short (minimum is 8 characters)']) @@ -488,7 +488,7 @@ describe API::Users, api: true do key_attrs = attributes_for :key expect do post api("/users/#{user.id}/keys", admin), key_attrs - end.to change{ user.keys.count }.by(1) + end.to change { user.keys.count }.by(1) end it "returns 400 for invalid ID" do @@ -580,7 +580,7 @@ describe API::Users, api: true do email_attrs = attributes_for :email expect do post api("/users/#{user.id}/emails", admin), email_attrs - end.to change{ user.emails.count }.by(1) + end.to change { user.emails.count }.by(1) end it "returns a 400 for invalid ID" do @@ -842,7 +842,7 @@ describe API::Users, api: true do key_attrs = attributes_for :key expect do post api("/user/keys", user), key_attrs - end.to change{ user.keys.count }.by(1) + end.to change { user.keys.count }.by(1) expect(response).to have_http_status(201) end @@ -880,7 +880,7 @@ describe API::Users, api: true do delete api("/user/keys/#{key.id}", user) expect(response).to have_http_status(204) - end.to change{user.keys.count}.by(-1) + end.to change { user.keys.count}.by(-1) end it "returns 404 if key ID not found" do @@ -963,7 +963,7 @@ describe API::Users, api: true do email_attrs = attributes_for :email expect do post api("/user/emails", user), email_attrs - end.to change{ user.emails.count }.by(1) + end.to change { user.emails.count }.by(1) expect(response).to have_http_status(201) end @@ -989,7 +989,7 @@ describe API::Users, api: true do delete api("/user/emails/#{email.id}", user) expect(response).to have_http_status(204) - end.to change{user.emails.count}.by(-1) + end.to change { user.emails.count}.by(-1) end it "returns 404 if email ID not found" do @@ -1158,6 +1158,49 @@ describe API::Users, api: true do end end + context "user activities", :redis do + let!(:old_active_user) { create(:user, last_activity_on: Time.utc(2000, 1, 1)) } + let!(:newly_active_user) { create(:user, last_activity_on: 2.days.ago.midday) } + + context 'last activity as normal user' do + it 'has no permission' do + get api("/user/activities", user) + + expect(response).to have_http_status(403) + end + end + + context 'as admin' do + it 'returns the activities from the last 6 months' do + get api("/user/activities", admin) + + expect(response).to include_pagination_headers + expect(json_response.size).to eq(1) + + activity = json_response.last + + expect(activity['username']).to eq(newly_active_user.username) + expect(activity['last_activity_on']).to eq(2.days.ago.to_date.to_s) + expect(activity['last_activity_at']).to eq(2.days.ago.to_date.to_s) + end + + context 'passing a :from parameter' do + it 'returns the activities from the given date' do + get api("/user/activities?from=2000-1-1", admin) + + expect(response).to include_pagination_headers + expect(json_response.size).to eq(2) + + activity = json_response.first + + expect(activity['username']).to eq(old_active_user.username) + expect(activity['last_activity_on']).to eq(Time.utc(2000, 1, 1).to_date.to_s) + expect(activity['last_activity_at']).to eq(Time.utc(2000, 1, 1).to_date.to_s) + end + end + end + end + describe 'GET /users/:user_id/impersonation_tokens' do let!(:active_personal_access_token) { create(:personal_access_token, user: user) } let!(:revoked_personal_access_token) { create(:personal_access_token, :revoked, user: user) } diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index 006d6a6af1c..316742ff076 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -3,6 +3,7 @@ require "spec_helper" describe 'Git HTTP requests', lib: true do include GitHttpHelpers include WorkhorseHelpers + include UserActivitiesHelpers it "gives WWW-Authenticate hints" do clone_get('doesnt/exist.git') @@ -255,6 +256,14 @@ describe 'Git HTTP requests', lib: true do expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) end end + + it 'updates the user last activity', :redis do + expect(user_activity(user)).to be_nil + + download(path, env) do |response| + expect(user_activity(user)).to be_present + end + end end context "when an oauth token is provided" do diff --git a/spec/services/cohorts_service_spec.rb b/spec/services/cohorts_service_spec.rb new file mode 100644 index 00000000000..1e99442fdcb --- /dev/null +++ b/spec/services/cohorts_service_spec.rb @@ -0,0 +1,99 @@ +require 'spec_helper' + +describe CohortsService do + describe '#execute' do + def month_start(months_ago) + months_ago.months.ago.beginning_of_month.to_date + end + + # In the interests of speed and clarity, this example has minimal data. + it 'returns a list of user cohorts' do + 6.times do |months_ago| + months_ago_time = (months_ago * 2).months.ago + + create(:user, created_at: months_ago_time, last_activity_on: Time.now) + create(:user, created_at: months_ago_time, last_activity_on: months_ago_time) + end + + create(:user) # this user is inactive and belongs to the current month + + expected_cohorts = [ + { + registration_month: month_start(11), + activity_months: Array.new(12) { { total: 0, percentage: 0 } }, + total: 0, + inactive: 0 + }, + { + registration_month: month_start(10), + activity_months: [{ total: 2, percentage: 100 }] + Array.new(10) { { total: 1, percentage: 50 } }, + total: 2, + inactive: 0 + }, + { + registration_month: month_start(9), + activity_months: Array.new(10) { { total: 0, percentage: 0 } }, + total: 0, + inactive: 0 + }, + { + registration_month: month_start(8), + activity_months: [{ total: 2, percentage: 100 }] + Array.new(8) { { total: 1, percentage: 50 } }, + total: 2, + inactive: 0 + }, + { + registration_month: month_start(7), + activity_months: Array.new(8) { { total: 0, percentage: 0 } }, + total: 0, + inactive: 0 + }, + { + registration_month: month_start(6), + activity_months: [{ total: 2, percentage: 100 }] + Array.new(6) { { total: 1, percentage: 50 } }, + total: 2, + inactive: 0 + }, + { + registration_month: month_start(5), + activity_months: Array.new(6) { { total: 0, percentage: 0 } }, + total: 0, + inactive: 0 + }, + { + registration_month: month_start(4), + activity_months: [{ total: 2, percentage: 100 }] + Array.new(4) { { total: 1, percentage: 50 } }, + total: 2, + inactive: 0 + }, + { + registration_month: month_start(3), + activity_months: Array.new(4) { { total: 0, percentage: 0 } }, + total: 0, + inactive: 0 + }, + { + registration_month: month_start(2), + activity_months: [{ total: 2, percentage: 100 }] + Array.new(2) { { total: 1, percentage: 50 } }, + total: 2, + inactive: 0 + }, + { + registration_month: month_start(1), + activity_months: Array.new(2) { { total: 0, percentage: 0 } }, + total: 0, + inactive: 0 + }, + { + registration_month: month_start(0), + activity_months: [{ total: 2, percentage: 100 }], + total: 2, + inactive: 1 + }, + ] + + expect(described_class.new.execute).to eq(months_included: 12, + cohorts: expected_cohorts) + end + end +end diff --git a/spec/services/event_create_service_spec.rb b/spec/services/event_create_service_spec.rb index f2c2009bcbf..b06cefe071d 100644 --- a/spec/services/event_create_service_spec.rb +++ b/spec/services/event_create_service_spec.rb @@ -1,6 +1,8 @@ require 'spec_helper' describe EventCreateService, services: true do + include UserActivitiesHelpers + let(:service) { EventCreateService.new } describe 'Issues' do @@ -111,6 +113,19 @@ describe EventCreateService, services: true do end end + describe '#push', :redis do + let(:project) { create(:empty_project) } + let(:user) { create(:user) } + + it 'creates a new event' do + expect { service.push(project, user, {}) }.to change { Event.count } + end + + it 'updates user last activity' do + expect { service.push(project, user, {}) }.to change { user_activity(user) } + end + end + describe 'Project' do let(:user) { create :user } let(:project) { create(:empty_project) } diff --git a/spec/services/users/activity_service_spec.rb b/spec/services/users/activity_service_spec.rb new file mode 100644 index 00000000000..8d67ebe3231 --- /dev/null +++ b/spec/services/users/activity_service_spec.rb @@ -0,0 +1,48 @@ +require 'spec_helper' + +describe Users::ActivityService, services: true do + include UserActivitiesHelpers + + let(:user) { create(:user) } + + subject(:service) { described_class.new(user, 'type') } + + describe '#execute', :redis do + context 'when last activity is nil' do + before do + service.execute + end + + it 'sets the last activity timestamp for the user' do + expect(last_hour_user_ids).to eq([user.id]) + end + + it 'updates the same user' do + service.execute + + expect(last_hour_user_ids).to eq([user.id]) + end + + it 'updates the timestamp of an existing user' do + Timecop.freeze(Date.tomorrow) do + expect { service.execute }.to change { user_activity(user) }.to(Time.now.to_i.to_s) + end + end + + describe 'other user' do + it 'updates other user' do + other_user = create(:user) + described_class.new(other_user, 'type').execute + + expect(last_hour_user_ids).to match_array([user.id, other_user.id]) + end + end + end + end + + def last_hour_user_ids + Gitlab::UserActivities.new. + select { |k, v| v >= 1.hour.ago.to_i.to_s }. + map { |k, _| k.to_i } + end +end diff --git a/spec/support/matchers/user_activity_matchers.rb b/spec/support/matchers/user_activity_matchers.rb new file mode 100644 index 00000000000..ce3b683b6d2 --- /dev/null +++ b/spec/support/matchers/user_activity_matchers.rb @@ -0,0 +1,5 @@ +RSpec::Matchers.define :have_an_activity_record do |expected| + match do |user| + expect(Gitlab::UserActivities.new.find { |k, _| k == user.id.to_s }).to be_present + end +end diff --git a/spec/support/user_activities_helpers.rb b/spec/support/user_activities_helpers.rb new file mode 100644 index 00000000000..f7ca9a31edd --- /dev/null +++ b/spec/support/user_activities_helpers.rb @@ -0,0 +1,7 @@ +module UserActivitiesHelpers + def user_activity(user) + Gitlab::UserActivities.new. + find { |k, _| k == user.id.to_s }&. + second + end +end diff --git a/spec/workers/gitlab_usage_ping_worker_spec.rb b/spec/workers/gitlab_usage_ping_worker_spec.rb new file mode 100644 index 00000000000..b6c080f36f4 --- /dev/null +++ b/spec/workers/gitlab_usage_ping_worker_spec.rb @@ -0,0 +1,23 @@ +require 'spec_helper' + +describe GitlabUsagePingWorker do + subject { GitlabUsagePingWorker.new } + + it "sends POST request" do + stub_application_setting(usage_ping_enabled: true) + + stub_request(:post, "https://version.gitlab.com/usage_data"). + to_return(status: 200, body: '', headers: {}) + expect(Gitlab::UsageData).to receive(:to_json).with({ force_refresh: true }).and_call_original + expect(subject).to receive(:try_obtain_lease).and_return(true) + + expect(subject.perform.response.code.to_i).to eq(200) + end + + it "does not run if usage ping is disabled" do + stub_application_setting(usage_ping_enabled: false) + + expect(subject).not_to receive(:try_obtain_lease) + expect(subject).not_to receive(:perform) + end +end diff --git a/spec/workers/schedule_update_user_activity_worker_spec.rb b/spec/workers/schedule_update_user_activity_worker_spec.rb new file mode 100644 index 00000000000..e583c3203aa --- /dev/null +++ b/spec/workers/schedule_update_user_activity_worker_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +describe ScheduleUpdateUserActivityWorker, :redis do + let(:now) { Time.now } + + before do + Gitlab::UserActivities.record('1', now) + Gitlab::UserActivities.record('2', now) + end + + it 'schedules UpdateUserActivityWorker once' do + expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '1' => now.to_i.to_s, '2' => now.to_i.to_s }) + + subject.perform + end + + context 'when specifying a batch size' do + it 'schedules UpdateUserActivityWorker twice' do + expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '1' => now.to_i.to_s }) + expect(UpdateUserActivityWorker).to receive(:perform_async).with({ '2' => now.to_i.to_s }) + + subject.perform(1) + end + end +end diff --git a/spec/workers/update_user_activity_worker_spec.rb b/spec/workers/update_user_activity_worker_spec.rb new file mode 100644 index 00000000000..43e9511f116 --- /dev/null +++ b/spec/workers/update_user_activity_worker_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +describe UpdateUserActivityWorker, :redis do + let(:user_active_2_days_ago) { create(:user, current_sign_in_at: 10.months.ago) } + let(:user_active_yesterday_1) { create(:user) } + let(:user_active_yesterday_2) { create(:user) } + let(:user_active_today) { create(:user) } + let(:data) do + { + user_active_2_days_ago.id.to_s => 2.days.ago.at_midday.to_i.to_s, + user_active_yesterday_1.id.to_s => 1.day.ago.at_midday.to_i.to_s, + user_active_yesterday_2.id.to_s => 1.day.ago.at_midday.to_i.to_s, + user_active_today.id.to_s => Time.now.to_i.to_s + } + end + + it 'updates users.last_activity_on' do + subject.perform(data) + + aggregate_failures do + expect(user_active_2_days_ago.reload.last_activity_on).to eq(2.days.ago.to_date) + expect(user_active_yesterday_1.reload.last_activity_on).to eq(1.day.ago.to_date) + expect(user_active_yesterday_2.reload.last_activity_on).to eq(1.day.ago.to_date) + expect(user_active_today.reload.reload.last_activity_on).to eq(Date.today) + end + end + + it 'deletes the pairs from Redis' do + data.each { |id, time| Gitlab::UserActivities.record(id, time) } + + subject.perform(data) + + expect(Gitlab::UserActivities.new.to_a).to be_empty + end +end |
