From 1b102f5d119009575bda43bfc9b0ae8365f67c50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C5=82gorzata=20Ksionek?= Date: Sat, 6 Jul 2019 22:15:13 +0200 Subject: Add basic project extraction To allow project filtering Prepare summary for accepting multiple groups Modify deploys group summary class Add filtering by project name in issues summary Fix rubocop offences Add changelog entry Change name to id in project filtering Fix rebase problem Add project extraction --- app/models/cycle_analytics/group_level.rb | 3 +- ...roup-level-analytics-to-accept-multiple-ids.yml | 5 + db/schema.rb | 106 +++++++++++---------- lib/gitlab/cycle_analytics/base_data_extraction.rb | 27 ++++++ lib/gitlab/cycle_analytics/base_event_fetcher.rb | 13 +-- lib/gitlab/cycle_analytics/base_stage.rb | 13 +-- lib/gitlab/cycle_analytics/group_stage_summary.rb | 7 +- lib/gitlab/cycle_analytics/summary/group/base.rb | 3 +- lib/gitlab/cycle_analytics/summary/group/deploy.rb | 14 ++- lib/gitlab/cycle_analytics/summary/group/issue.rb | 13 ++- .../cycle_analytics/group_stage_summary_spec.rb | 29 +++++- .../lib/gitlab/cycle_analytics/issue_stage_spec.rb | 23 +++++ 12 files changed, 167 insertions(+), 89 deletions(-) create mode 100644 changelogs/unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml create mode 100644 lib/gitlab/cycle_analytics/base_data_extraction.rb diff --git a/app/models/cycle_analytics/group_level.rb b/app/models/cycle_analytics/group_level.rb index 508cde0ca00..b351ad82f95 100644 --- a/app/models/cycle_analytics/group_level.rb +++ b/app/models/cycle_analytics/group_level.rb @@ -13,7 +13,8 @@ module CycleAnalytics def summary @summary ||= ::Gitlab::CycleAnalytics::GroupStageSummary.new(group, from: options[:from], - current_user: options[:current_user]).data + current_user: options[:current_user], + options: options).data end def permissions(*) diff --git a/changelogs/unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml b/changelogs/unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml new file mode 100644 index 00000000000..a90e73a1410 --- /dev/null +++ b/changelogs/unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml @@ -0,0 +1,5 @@ +--- +title: Adjust group level analytics to accept multiple ids +merge_request: 30468 +author: +type: added diff --git a/db/schema.rb b/db/schema.rb index 79cd1a3a797..a001b9ba6ac 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -63,7 +63,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.datetime "updated_at" t.string "home_page_url" t.integer "default_branch_protection", default: 2 - t.text "help_text" t.text "restricted_visibility_levels" t.boolean "version_check_enabled", default: true t.integer "max_attachment_size", default: 10, null: false @@ -105,8 +104,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.integer "container_registry_token_expire_delay", default: 5 t.text "after_sign_up_text" t.boolean "user_default_external", default: false, null: false - t.boolean "elasticsearch_indexing", default: false, null: false - t.boolean "elasticsearch_search", default: false, null: false t.string "repository_storages", default: "default" t.string "enabled_git_access_protocol" t.boolean "domain_blacklist_enabled", default: false @@ -128,37 +125,18 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.boolean "html_emails_enabled", default: true t.string "plantuml_url" t.boolean "plantuml_enabled" - t.integer "shared_runners_minutes", default: 0, null: false - t.bigint "repository_size_limit", default: 0 t.integer "terminal_max_session_time", default: 0, null: false t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" t.boolean "unique_ips_limit_enabled", default: false, null: false t.string "default_artifacts_expire_in", default: "0", null: false - t.string "elasticsearch_url", default: "http://localhost:9200" - t.boolean "elasticsearch_aws", default: false, null: false - t.string "elasticsearch_aws_region", default: "us-east-1" - t.string "elasticsearch_aws_access_key" - t.string "elasticsearch_aws_secret_access_key" - t.integer "geo_status_timeout", default: 10 t.string "uuid" t.decimal "polling_interval_multiplier", default: "1.0", null: false - t.boolean "elasticsearch_experimental_indexer" t.integer "cached_markdown_version" - t.boolean "check_namespace_plan", default: false, null: false - t.integer "mirror_max_delay", default: 300, null: false - t.integer "mirror_max_capacity", default: 100, null: false - t.integer "mirror_capacity_threshold", default: 50, null: false t.boolean "prometheus_metrics_enabled", default: true, null: false - t.boolean "authorized_keys_enabled", default: true, null: false t.boolean "help_page_hide_commercial_content", default: false t.string "help_page_support_url" - t.boolean "slack_app_enabled", default: false - t.string "slack_app_id" - t.string "slack_app_secret" - t.string "slack_app_verification_token" t.integer "performance_bar_allowed_group_id" - t.boolean "allow_group_owners_to_manage_ldap", default: true, null: false t.boolean "hashed_storage_enabled", default: true, null: false t.boolean "project_export_enabled", default: true, null: false t.boolean "auto_devops_enabled", default: true, null: false @@ -171,38 +149,22 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.boolean "throttle_authenticated_web_enabled", default: false, null: false t.integer "throttle_authenticated_web_requests_per_period", default: 7200, null: false t.integer "throttle_authenticated_web_period_in_seconds", default: 3600, null: false + t.boolean "password_authentication_enabled_for_web" + t.boolean "password_authentication_enabled_for_git", default: true, null: false t.integer "gitaly_timeout_default", default: 55, null: false t.integer "gitaly_timeout_medium", default: 30, null: false t.integer "gitaly_timeout_fast", default: 10, null: false - t.boolean "mirror_available", default: true, null: false - t.boolean "password_authentication_enabled_for_web" - t.boolean "password_authentication_enabled_for_git", default: true, null: false + t.boolean "authorized_keys_enabled", default: true, null: false t.string "auto_devops_domain" - t.boolean "external_authorization_service_enabled", default: false, null: false - t.string "external_authorization_service_url" - t.string "external_authorization_service_default_label" t.boolean "pages_domain_verification_enabled", default: true, null: false t.string "user_default_internal_regex" t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false - t.float "external_authorization_service_timeout", default: 0.5 - t.text "external_auth_client_cert" - t.text "encrypted_external_auth_client_key" - t.string "encrypted_external_auth_client_key_iv" - t.string "encrypted_external_auth_client_key_pass" - t.string "encrypted_external_auth_client_key_pass_iv" - t.string "email_additional_text" t.boolean "enforce_terms", default: false - t.integer "file_template_project_id" - t.boolean "pseudonymizer_enabled", default: false, null: false + t.boolean "mirror_available", default: true, null: false t.boolean "hide_third_party_offers", default: false, null: false - t.boolean "snowplow_enabled", default: false, null: false - t.string "snowplow_collector_uri" - t.string "snowplow_site_id" - t.string "snowplow_cookie_domain" t.boolean "instance_statistics_visibility_private", default: false, null: false t.boolean "web_ide_clientside_preview_enabled", default: false, null: false t.boolean "user_show_add_ssh_key_message", default: true, null: false - t.integer "custom_project_templates_group_id" t.integer "usage_stats_set_by_user_id" t.integer "receive_max_input_size" t.integer "diff_max_patch_bytes", default: 102400, null: false @@ -212,21 +174,59 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.string "runners_registration_token_encrypted" t.integer "local_markdown_version", default: 0, null: false t.integer "first_day_of_week", default: 0, null: false - t.boolean "elasticsearch_limit_indexing", default: false, null: false t.integer "default_project_creation", default: 2, null: false + t.boolean "external_authorization_service_enabled", default: false, null: false + t.string "external_authorization_service_url" + t.string "external_authorization_service_default_label" + t.float "external_authorization_service_timeout", default: 0.5 + t.text "external_auth_client_cert" + t.text "encrypted_external_auth_client_key" + t.string "encrypted_external_auth_client_key_iv" + t.string "encrypted_external_auth_client_key_pass" + t.string "encrypted_external_auth_client_key_pass_iv" t.string "lets_encrypt_notification_email" t.boolean "lets_encrypt_terms_of_service_accepted", default: false, null: false - t.string "geo_node_allowed_ips", default: "0.0.0.0/0, ::/0" t.integer "elasticsearch_shards", default: 5, null: false t.integer "elasticsearch_replicas", default: 1, null: false t.text "encrypted_lets_encrypt_private_key" t.text "encrypted_lets_encrypt_private_key_iv" - t.string "required_instance_ci_template" t.boolean "dns_rebinding_protection_enabled", default: true, null: false t.boolean "default_project_deletion_protection", default: false, null: false - t.boolean "grafana_enabled", default: false, null: false + t.text "help_text" + t.boolean "elasticsearch_indexing", default: false, null: false + t.boolean "elasticsearch_search", default: false, null: false + t.integer "shared_runners_minutes", default: 0, null: false + t.bigint "repository_size_limit", default: 0 + t.string "elasticsearch_url", default: "http://localhost:9200" + t.boolean "elasticsearch_aws", default: false, null: false + t.string "elasticsearch_aws_region", default: "us-east-1" + t.string "elasticsearch_aws_access_key" + t.string "elasticsearch_aws_secret_access_key" + t.integer "geo_status_timeout", default: 10 + t.boolean "elasticsearch_experimental_indexer" + t.boolean "check_namespace_plan", default: false, null: false + t.integer "mirror_max_delay", default: 300, null: false + t.integer "mirror_max_capacity", default: 100, null: false + t.integer "mirror_capacity_threshold", default: 50, null: false + t.boolean "slack_app_enabled", default: false + t.string "slack_app_id" + t.string "slack_app_secret" + t.string "slack_app_verification_token" + t.boolean "allow_group_owners_to_manage_ldap", default: true, null: false + t.string "email_additional_text" + t.integer "file_template_project_id" + t.boolean "pseudonymizer_enabled", default: false, null: false + t.boolean "snowplow_enabled", default: false, null: false + t.string "snowplow_collector_uri" + t.string "snowplow_site_id" + t.string "snowplow_cookie_domain" + t.integer "custom_project_templates_group_id" + t.boolean "elasticsearch_limit_indexing", default: false, null: false + t.string "geo_node_allowed_ips", default: "0.0.0.0/0, ::/0" + t.string "required_instance_ci_template" t.boolean "lock_memberships_to_ldap", default: false, null: false t.boolean "time_tracking_limit_to_hours", default: false, null: false + t.boolean "grafana_enabled", default: false, null: false t.string "grafana_url", default: "/-/grafana", null: false t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id", using: :btree t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id", using: :btree @@ -407,10 +407,10 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.integer "project_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.string "name", default: "Development", null: false - t.integer "milestone_id" t.integer "group_id" + t.integer "milestone_id" t.integer "weight" + t.string "name", default: "Development", null: false t.index ["group_id"], name: "index_boards_on_group_id", using: :btree t.index ["milestone_id"], name: "index_boards_on_milestone_id", using: :btree t.index ["project_id"], name: "index_boards_on_project_id", using: :btree @@ -611,7 +611,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.index ["pipeline_id"], name: "index_ci_pipeline_chat_data_on_pipeline_id", unique: true, using: :btree end - create_table "ci_pipeline_schedule_variables", force: :cascade do |t| + create_table "ci_pipeline_schedule_variables", id: :serial, force: :cascade do |t| t.string "key", null: false t.text "value" t.text "encrypted_value" @@ -1865,7 +1865,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.index ["user_id"], name: "index_members_on_user_id", using: :btree end - create_table "merge_request_assignees", force: :cascade do |t| + create_table "merge_request_assignees", id: :serial, force: :cascade do |t| t.integer "user_id", null: false t.integer "merge_request_id", null: false t.index ["merge_request_id", "user_id"], name: "index_merge_request_assignees_on_merge_request_id_and_user_id", unique: true, using: :btree @@ -2101,8 +2101,8 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.integer "cached_markdown_version" t.string "runners_token" t.string "runners_token_encrypted" - t.integer "project_creation_level" t.boolean "auto_devops_enabled" + t.integer "project_creation_level" t.datetime_with_timezone "last_ci_minutes_notification_at" t.integer "custom_project_templates_group_id" t.integer "file_template_project_id" @@ -2534,7 +2534,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.index ["project_id"], name: "index_project_import_data_on_project_id", using: :btree end - create_table "project_incident_management_settings", primary_key: "project_id", id: :serial, force: :cascade do |t| + create_table "project_incident_management_settings", primary_key: "project_id", id: :integer, default: nil, force: :cascade do |t| t.boolean "create_issue", default: true, null: false t.boolean "send_email", default: false, null: false t.text "issue_template_key" @@ -2885,6 +2885,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree + t.index ["path"], name: "index_redirect_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"} t.index ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree end @@ -3304,6 +3305,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.string "timezone" t.boolean "time_display_relative" t.boolean "time_format_in_24h" + t.string "timezone_name" t.integer "epic_notes_filter", limit: 2, default: 0, null: false t.string "epics_sort" t.integer "roadmap_epics_state" diff --git a/lib/gitlab/cycle_analytics/base_data_extraction.rb b/lib/gitlab/cycle_analytics/base_data_extraction.rb new file mode 100644 index 00000000000..deadf47f146 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_data_extraction.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module BaseDataExtraction + private + + def projects + group ? extract_projects(options) : [project] + end + + def group + @group ||= options.fetch(:group, nil) + end + + def project + @project ||= options.fetch(:project, nil) + end + + def extract_projects(options) + projects = Project.inside_path(group.full_path) + projects = projects.where(id: options[:projects]) if options[:projects] + projects + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 96aa799e864..0f5186e06e7 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -4,6 +4,7 @@ module Gitlab module CycleAnalytics class BaseEventFetcher include BaseQuery + include BaseDataExtraction attr_reader :projections, :query, :stage, :order, :options @@ -73,18 +74,6 @@ module Gitlab def serialization_context {} end - - def projects - group ? Project.inside_path(group.full_path) : [project] - end - - def group - @group ||= options.fetch(:group, nil) - end - - def project - @project ||= options.fetch(:project, nil) - end end end end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 678a891e941..e641bed704b 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -4,6 +4,7 @@ module Gitlab module CycleAnalytics class BaseStage include BaseQuery + include BaseDataExtraction attr_reader :options @@ -77,18 +78,6 @@ module Gitlab def event_options options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) end - - def projects - group ? Project.inside_path(group.full_path) : [project] - end - - def group - @group ||= options.fetch(:group, nil) - end - - def project - @project ||= options.fetch(:project, nil) - end end end end diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb index 7b5c74e1a1b..f867d511f65 100644 --- a/lib/gitlab/cycle_analytics/group_stage_summary.rb +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -3,15 +3,16 @@ module Gitlab module CycleAnalytics class GroupStageSummary - def initialize(group, from:, current_user:) + def initialize(group, from:, current_user:, options:) @group = group @from = from @current_user = current_user + @options = options end def data - [serialize(Summary::Group::Issue.new(group: @group, from: @from, current_user: @current_user)), - serialize(Summary::Group::Deploy.new(group: @group, from: @from))] + [serialize(Summary::Group::Issue.new(group: @group, from: @from, current_user: @current_user, options: @options)), + serialize(Summary::Group::Deploy.new(group: @group, from: @from, options: @options))] end private diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb index 7f18b61d309..d147879ace4 100644 --- a/lib/gitlab/cycle_analytics/summary/group/base.rb +++ b/lib/gitlab/cycle_analytics/summary/group/base.rb @@ -5,9 +5,10 @@ module Gitlab module Summary module Group class Base - def initialize(group:, from:) + def initialize(group:, from:, options:) @group = group @from = from + @options = options end def title diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb index d8fcd8f2ce4..ec0b23e770d 100644 --- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -10,15 +10,19 @@ module Gitlab end def value - @value ||= Deployment.joins(:project) - .where(projects: { id: projects }) - .where("deployments.created_at > ?", @from) - .success - .count + @value ||= find_deployments end private + def find_deployments + deployments = Deployment.joins(:project) + .where(projects: { id: projects }) + .where("deployments.created_at > ?", @from) + deployments = deployments.where(projects: { id: @options[:projects] }) if @options[:projects] + deployments.success.count + end + def projects Project.inside_path(@group.full_path).ids end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb index 70073e6d843..5df508efa78 100644 --- a/lib/gitlab/cycle_analytics/summary/group/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/group/issue.rb @@ -5,10 +5,11 @@ module Gitlab module Summary module Group class Issue < Group::Base - def initialize(group:, from:, current_user:) + def initialize(group:, from:, current_user:, options:) @group = group @from = from @current_user = current_user + @options = options end def title @@ -16,7 +17,15 @@ module Gitlab end def value - @value ||= IssuesFinder.new(@current_user, group_id: @group.id, include_subgroups: true, created_after: @from).execute.count + @value ||= find_issues + end + + private + + def find_issues + issues = IssuesFinder.new(@current_user, group_id: @group.id, include_subgroups: true, created_after: @from).execute + issues = issues.where(projects: { id: @options[:projects] }) if @options[:projects] + issues.count end end end diff --git a/spec/lib/gitlab/cycle_analytics/group_stage_summary_spec.rb b/spec/lib/gitlab/cycle_analytics/group_stage_summary_spec.rb index eea4f33ccb8..d4dd52ec092 100644 --- a/spec/lib/gitlab/cycle_analytics/group_stage_summary_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/group_stage_summary_spec.rb @@ -8,7 +8,7 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do let(:from) { 1.day.ago } let(:user) { create(:user, :admin) } - subject { described_class.new(group, from: Time.now, current_user: user).data } + subject { described_class.new(group, from: Time.now, current_user: user, options: {}).data } describe "#new_issues" do context 'with from date' do @@ -32,6 +32,18 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do expect(subject.first[:value]).to eq(3) end end + + context 'with projects specified in options' do + before do + Timecop.freeze(5.days.from_now) { create(:issue, project: create(:project, namespace: group)) } + end + + subject { described_class.new(group, from: Time.now, current_user: user, options: { projects: [project.id, project_2.id] }).data } + + it 'finds issues from those projects' do + expect(subject.first[:value]).to eq(2) + end + end end context 'with other projects' do @@ -71,6 +83,20 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do expect(subject.second[:value]).to eq(3) end end + + context 'with projects specified in options' do + before do + Timecop.freeze(5.days.from_now) do + create(:deployment, :success, project: create(:project, :repository, namespace: group, name: 'not_applicable')) + end + end + + subject { described_class.new(group, from: Time.now, current_user: user, options: { projects: [project.id, project_2.id] }).data } + + it 'shows deploys from those projects' do + expect(subject.second[:value]).to eq(2) + end + end end context 'with other projects' do @@ -84,5 +110,6 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do expect(subject.second[:value]).to eq(0) end end + end end diff --git a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb index ffd0b84cb57..1052ee69830 100644 --- a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb @@ -71,6 +71,29 @@ describe Gitlab::CycleAnalytics::IssueStage do end end + context 'when only part of projects is chosen' do + let(:stage) { described_class.new(options: { from: 2.days.ago, current_user: user, group: group, projects: [project_2.id] }) } + + describe '#group_median' do + around do |example| + Timecop.freeze { example.run } + end + + it 'counts median from issues with metrics' do + expect(stage.group_median).to eq(ISSUES_MEDIAN) + end + end + + describe '#events' do + it 'exposes merge requests that close issues' do + result = stage.events + + expect(result.count).to eq(1) + expect(result.map { |event| event[:title] }).to contain_exactly(issue_2_1.title) + end + end + end + context 'when subgroup is given' do let(:subgroup) { create(:group, parent: group) } let(:project_4) { create(:project, group: subgroup) } -- cgit v1.2.1 From f70d931b88829585da7915b11bd4775297b6dac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C5=82gorzata=20Ksionek?= Date: Mon, 15 Jul 2019 16:28:52 +0200 Subject: Fix changelog merge request number --- .../unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelogs/unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml b/changelogs/unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml index a90e73a1410..5e138e1059c 100644 --- a/changelogs/unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml +++ b/changelogs/unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml @@ -1,5 +1,5 @@ --- title: Adjust group level analytics to accept multiple ids -merge_request: 30468 +merge_request: 30744 author: type: added -- cgit v1.2.1 From 5ce4236b66465c4edd8edb87e8f9661fdb4ca8c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C5=82gorzata=20Ksionek?= Date: Thu, 18 Jul 2019 16:07:36 +0200 Subject: Add code review remarks Add cr remarks Improve specs according to the review Fix schema Add cr remarks Fix naming Add cr remarks --- app/models/cycle_analytics/group_level.rb | 5 +- db/schema.rb | 106 ++++++++++----------- lib/gitlab/cycle_analytics/base_data_extraction.rb | 27 ------ lib/gitlab/cycle_analytics/base_event_fetcher.rb | 2 +- lib/gitlab/cycle_analytics/base_stage.rb | 2 +- .../cycle_analytics/group_projects_provider.rb | 27 ++++++ lib/gitlab/cycle_analytics/group_stage_summary.rb | 12 ++- lib/gitlab/cycle_analytics/summary/group/base.rb | 2 + lib/gitlab/cycle_analytics/summary/group/deploy.rb | 11 +-- lib/gitlab/cycle_analytics/summary/group/issue.rb | 6 +- .../cycle_analytics/group_stage_summary_spec.rb | 7 +- .../lib/gitlab/cycle_analytics/issue_stage_spec.rb | 8 +- 12 files changed, 106 insertions(+), 109 deletions(-) delete mode 100644 lib/gitlab/cycle_analytics/base_data_extraction.rb create mode 100644 lib/gitlab/cycle_analytics/group_projects_provider.rb diff --git a/app/models/cycle_analytics/group_level.rb b/app/models/cycle_analytics/group_level.rb index b351ad82f95..a41e1375484 100644 --- a/app/models/cycle_analytics/group_level.rb +++ b/app/models/cycle_analytics/group_level.rb @@ -11,10 +11,7 @@ module CycleAnalytics end def summary - @summary ||= ::Gitlab::CycleAnalytics::GroupStageSummary.new(group, - from: options[:from], - current_user: options[:current_user], - options: options).data + @summary ||= ::Gitlab::CycleAnalytics::GroupStageSummary.new(group, options: options).data end def permissions(*) diff --git a/db/schema.rb b/db/schema.rb index a001b9ba6ac..79cd1a3a797 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -63,6 +63,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.datetime "updated_at" t.string "home_page_url" t.integer "default_branch_protection", default: 2 + t.text "help_text" t.text "restricted_visibility_levels" t.boolean "version_check_enabled", default: true t.integer "max_attachment_size", default: 10, null: false @@ -104,6 +105,8 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.integer "container_registry_token_expire_delay", default: 5 t.text "after_sign_up_text" t.boolean "user_default_external", default: false, null: false + t.boolean "elasticsearch_indexing", default: false, null: false + t.boolean "elasticsearch_search", default: false, null: false t.string "repository_storages", default: "default" t.string "enabled_git_access_protocol" t.boolean "domain_blacklist_enabled", default: false @@ -125,18 +128,37 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.boolean "html_emails_enabled", default: true t.string "plantuml_url" t.boolean "plantuml_enabled" + t.integer "shared_runners_minutes", default: 0, null: false + t.bigint "repository_size_limit", default: 0 t.integer "terminal_max_session_time", default: 0, null: false t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" t.boolean "unique_ips_limit_enabled", default: false, null: false t.string "default_artifacts_expire_in", default: "0", null: false + t.string "elasticsearch_url", default: "http://localhost:9200" + t.boolean "elasticsearch_aws", default: false, null: false + t.string "elasticsearch_aws_region", default: "us-east-1" + t.string "elasticsearch_aws_access_key" + t.string "elasticsearch_aws_secret_access_key" + t.integer "geo_status_timeout", default: 10 t.string "uuid" t.decimal "polling_interval_multiplier", default: "1.0", null: false + t.boolean "elasticsearch_experimental_indexer" t.integer "cached_markdown_version" + t.boolean "check_namespace_plan", default: false, null: false + t.integer "mirror_max_delay", default: 300, null: false + t.integer "mirror_max_capacity", default: 100, null: false + t.integer "mirror_capacity_threshold", default: 50, null: false t.boolean "prometheus_metrics_enabled", default: true, null: false + t.boolean "authorized_keys_enabled", default: true, null: false t.boolean "help_page_hide_commercial_content", default: false t.string "help_page_support_url" + t.boolean "slack_app_enabled", default: false + t.string "slack_app_id" + t.string "slack_app_secret" + t.string "slack_app_verification_token" t.integer "performance_bar_allowed_group_id" + t.boolean "allow_group_owners_to_manage_ldap", default: true, null: false t.boolean "hashed_storage_enabled", default: true, null: false t.boolean "project_export_enabled", default: true, null: false t.boolean "auto_devops_enabled", default: true, null: false @@ -149,22 +171,38 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.boolean "throttle_authenticated_web_enabled", default: false, null: false t.integer "throttle_authenticated_web_requests_per_period", default: 7200, null: false t.integer "throttle_authenticated_web_period_in_seconds", default: 3600, null: false - t.boolean "password_authentication_enabled_for_web" - t.boolean "password_authentication_enabled_for_git", default: true, null: false t.integer "gitaly_timeout_default", default: 55, null: false t.integer "gitaly_timeout_medium", default: 30, null: false t.integer "gitaly_timeout_fast", default: 10, null: false - t.boolean "authorized_keys_enabled", default: true, null: false + t.boolean "mirror_available", default: true, null: false + t.boolean "password_authentication_enabled_for_web" + t.boolean "password_authentication_enabled_for_git", default: true, null: false t.string "auto_devops_domain" + t.boolean "external_authorization_service_enabled", default: false, null: false + t.string "external_authorization_service_url" + t.string "external_authorization_service_default_label" t.boolean "pages_domain_verification_enabled", default: true, null: false t.string "user_default_internal_regex" t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false + t.float "external_authorization_service_timeout", default: 0.5 + t.text "external_auth_client_cert" + t.text "encrypted_external_auth_client_key" + t.string "encrypted_external_auth_client_key_iv" + t.string "encrypted_external_auth_client_key_pass" + t.string "encrypted_external_auth_client_key_pass_iv" + t.string "email_additional_text" t.boolean "enforce_terms", default: false - t.boolean "mirror_available", default: true, null: false + t.integer "file_template_project_id" + t.boolean "pseudonymizer_enabled", default: false, null: false t.boolean "hide_third_party_offers", default: false, null: false + t.boolean "snowplow_enabled", default: false, null: false + t.string "snowplow_collector_uri" + t.string "snowplow_site_id" + t.string "snowplow_cookie_domain" t.boolean "instance_statistics_visibility_private", default: false, null: false t.boolean "web_ide_clientside_preview_enabled", default: false, null: false t.boolean "user_show_add_ssh_key_message", default: true, null: false + t.integer "custom_project_templates_group_id" t.integer "usage_stats_set_by_user_id" t.integer "receive_max_input_size" t.integer "diff_max_patch_bytes", default: 102400, null: false @@ -174,59 +212,21 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.string "runners_registration_token_encrypted" t.integer "local_markdown_version", default: 0, null: false t.integer "first_day_of_week", default: 0, null: false + t.boolean "elasticsearch_limit_indexing", default: false, null: false t.integer "default_project_creation", default: 2, null: false - t.boolean "external_authorization_service_enabled", default: false, null: false - t.string "external_authorization_service_url" - t.string "external_authorization_service_default_label" - t.float "external_authorization_service_timeout", default: 0.5 - t.text "external_auth_client_cert" - t.text "encrypted_external_auth_client_key" - t.string "encrypted_external_auth_client_key_iv" - t.string "encrypted_external_auth_client_key_pass" - t.string "encrypted_external_auth_client_key_pass_iv" t.string "lets_encrypt_notification_email" t.boolean "lets_encrypt_terms_of_service_accepted", default: false, null: false + t.string "geo_node_allowed_ips", default: "0.0.0.0/0, ::/0" t.integer "elasticsearch_shards", default: 5, null: false t.integer "elasticsearch_replicas", default: 1, null: false t.text "encrypted_lets_encrypt_private_key" t.text "encrypted_lets_encrypt_private_key_iv" + t.string "required_instance_ci_template" t.boolean "dns_rebinding_protection_enabled", default: true, null: false t.boolean "default_project_deletion_protection", default: false, null: false - t.text "help_text" - t.boolean "elasticsearch_indexing", default: false, null: false - t.boolean "elasticsearch_search", default: false, null: false - t.integer "shared_runners_minutes", default: 0, null: false - t.bigint "repository_size_limit", default: 0 - t.string "elasticsearch_url", default: "http://localhost:9200" - t.boolean "elasticsearch_aws", default: false, null: false - t.string "elasticsearch_aws_region", default: "us-east-1" - t.string "elasticsearch_aws_access_key" - t.string "elasticsearch_aws_secret_access_key" - t.integer "geo_status_timeout", default: 10 - t.boolean "elasticsearch_experimental_indexer" - t.boolean "check_namespace_plan", default: false, null: false - t.integer "mirror_max_delay", default: 300, null: false - t.integer "mirror_max_capacity", default: 100, null: false - t.integer "mirror_capacity_threshold", default: 50, null: false - t.boolean "slack_app_enabled", default: false - t.string "slack_app_id" - t.string "slack_app_secret" - t.string "slack_app_verification_token" - t.boolean "allow_group_owners_to_manage_ldap", default: true, null: false - t.string "email_additional_text" - t.integer "file_template_project_id" - t.boolean "pseudonymizer_enabled", default: false, null: false - t.boolean "snowplow_enabled", default: false, null: false - t.string "snowplow_collector_uri" - t.string "snowplow_site_id" - t.string "snowplow_cookie_domain" - t.integer "custom_project_templates_group_id" - t.boolean "elasticsearch_limit_indexing", default: false, null: false - t.string "geo_node_allowed_ips", default: "0.0.0.0/0, ::/0" - t.string "required_instance_ci_template" + t.boolean "grafana_enabled", default: false, null: false t.boolean "lock_memberships_to_ldap", default: false, null: false t.boolean "time_tracking_limit_to_hours", default: false, null: false - t.boolean "grafana_enabled", default: false, null: false t.string "grafana_url", default: "/-/grafana", null: false t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id", using: :btree t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id", using: :btree @@ -407,10 +407,10 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.integer "project_id" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.integer "group_id" + t.string "name", default: "Development", null: false t.integer "milestone_id" + t.integer "group_id" t.integer "weight" - t.string "name", default: "Development", null: false t.index ["group_id"], name: "index_boards_on_group_id", using: :btree t.index ["milestone_id"], name: "index_boards_on_milestone_id", using: :btree t.index ["project_id"], name: "index_boards_on_project_id", using: :btree @@ -611,7 +611,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.index ["pipeline_id"], name: "index_ci_pipeline_chat_data_on_pipeline_id", unique: true, using: :btree end - create_table "ci_pipeline_schedule_variables", id: :serial, force: :cascade do |t| + create_table "ci_pipeline_schedule_variables", force: :cascade do |t| t.string "key", null: false t.text "value" t.text "encrypted_value" @@ -1865,7 +1865,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.index ["user_id"], name: "index_members_on_user_id", using: :btree end - create_table "merge_request_assignees", id: :serial, force: :cascade do |t| + create_table "merge_request_assignees", force: :cascade do |t| t.integer "user_id", null: false t.integer "merge_request_id", null: false t.index ["merge_request_id", "user_id"], name: "index_merge_request_assignees_on_merge_request_id_and_user_id", unique: true, using: :btree @@ -2101,8 +2101,8 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.integer "cached_markdown_version" t.string "runners_token" t.string "runners_token_encrypted" - t.boolean "auto_devops_enabled" t.integer "project_creation_level" + t.boolean "auto_devops_enabled" t.datetime_with_timezone "last_ci_minutes_notification_at" t.integer "custom_project_templates_group_id" t.integer "file_template_project_id" @@ -2534,7 +2534,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.index ["project_id"], name: "index_project_import_data_on_project_id", using: :btree end - create_table "project_incident_management_settings", primary_key: "project_id", id: :integer, default: nil, force: :cascade do |t| + create_table "project_incident_management_settings", primary_key: "project_id", id: :serial, force: :cascade do |t| t.boolean "create_issue", default: true, null: false t.boolean "send_email", default: false, null: false t.text "issue_template_key" @@ -2885,7 +2885,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree - t.index ["path"], name: "index_redirect_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"} t.index ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree end @@ -3305,7 +3304,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do t.string "timezone" t.boolean "time_display_relative" t.boolean "time_format_in_24h" - t.string "timezone_name" t.integer "epic_notes_filter", limit: 2, default: 0, null: false t.string "epics_sort" t.integer "roadmap_epics_state" diff --git a/lib/gitlab/cycle_analytics/base_data_extraction.rb b/lib/gitlab/cycle_analytics/base_data_extraction.rb deleted file mode 100644 index deadf47f146..00000000000 --- a/lib/gitlab/cycle_analytics/base_data_extraction.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module BaseDataExtraction - private - - def projects - group ? extract_projects(options) : [project] - end - - def group - @group ||= options.fetch(:group, nil) - end - - def project - @project ||= options.fetch(:project, nil) - end - - def extract_projects(options) - projects = Project.inside_path(group.full_path) - projects = projects.where(id: options[:projects]) if options[:projects] - projects - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 0f5186e06e7..07ae430c45e 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -4,7 +4,7 @@ module Gitlab module CycleAnalytics class BaseEventFetcher include BaseQuery - include BaseDataExtraction + include GroupProjectsProvider attr_reader :projections, :query, :stage, :order, :options diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index e641bed704b..1cd54238bb4 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -4,7 +4,7 @@ module Gitlab module CycleAnalytics class BaseStage include BaseQuery - include BaseDataExtraction + include GroupProjectsProvider attr_reader :options diff --git a/lib/gitlab/cycle_analytics/group_projects_provider.rb b/lib/gitlab/cycle_analytics/group_projects_provider.rb new file mode 100644 index 00000000000..1287a48daaa --- /dev/null +++ b/lib/gitlab/cycle_analytics/group_projects_provider.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module GroupProjectsProvider + def projects + group ? projects_for_group : [project] + end + + def group + @group ||= options.fetch(:group, nil) + end + + def project + @project ||= options.fetch(:project, nil) + end + + private + + def projects_for_group + projects = Project.inside_path(group.full_path) + projects = projects.where(id: options[:projects]) if options[:projects] + projects + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/group_stage_summary.rb b/lib/gitlab/cycle_analytics/group_stage_summary.rb index f867d511f65..a1fc941495d 100644 --- a/lib/gitlab/cycle_analytics/group_stage_summary.rb +++ b/lib/gitlab/cycle_analytics/group_stage_summary.rb @@ -3,16 +3,18 @@ module Gitlab module CycleAnalytics class GroupStageSummary - def initialize(group, from:, current_user:, options:) + attr_reader :group, :from, :current_user, :options + + def initialize(group, options:) @group = group - @from = from - @current_user = current_user + @from = options[:from] + @current_user = options[:current_user] @options = options end def data - [serialize(Summary::Group::Issue.new(group: @group, from: @from, current_user: @current_user, options: @options)), - serialize(Summary::Group::Deploy.new(group: @group, from: @from, options: @options))] + [serialize(Summary::Group::Issue.new(group: group, from: from, current_user: current_user, options: options)), + serialize(Summary::Group::Deploy.new(group: group, from: from, options: options))] end private diff --git a/lib/gitlab/cycle_analytics/summary/group/base.rb b/lib/gitlab/cycle_analytics/summary/group/base.rb index d147879ace4..48d8164bde1 100644 --- a/lib/gitlab/cycle_analytics/summary/group/base.rb +++ b/lib/gitlab/cycle_analytics/summary/group/base.rb @@ -5,6 +5,8 @@ module Gitlab module Summary module Group class Base + attr_reader :group, :from, :options + def initialize(group:, from:, options:) @group = group @from = from diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb index ec0b23e770d..1476a5fceb8 100644 --- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -5,6 +5,8 @@ module Gitlab module Summary module Group class Deploy < Group::Base + include GroupProjectsProvider + def title n_('Deploy', 'Deploys', value) end @@ -17,15 +19,10 @@ module Gitlab def find_deployments deployments = Deployment.joins(:project) - .where(projects: { id: projects }) - .where("deployments.created_at > ?", @from) - deployments = deployments.where(projects: { id: @options[:projects] }) if @options[:projects] + .where(projects: { id: projects.ids }) + .where("deployments.created_at > ?", from) deployments.success.count end - - def projects - Project.inside_path(@group.full_path).ids - end end end end diff --git a/lib/gitlab/cycle_analytics/summary/group/issue.rb b/lib/gitlab/cycle_analytics/summary/group/issue.rb index 5df508efa78..9daae8531d8 100644 --- a/lib/gitlab/cycle_analytics/summary/group/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/group/issue.rb @@ -5,6 +5,8 @@ module Gitlab module Summary module Group class Issue < Group::Base + attr_reader :group, :from, :current_user, :options + def initialize(group:, from:, current_user:, options:) @group = group @from = from @@ -23,8 +25,8 @@ module Gitlab private def find_issues - issues = IssuesFinder.new(@current_user, group_id: @group.id, include_subgroups: true, created_after: @from).execute - issues = issues.where(projects: { id: @options[:projects] }) if @options[:projects] + issues = IssuesFinder.new(current_user, group_id: group.id, include_subgroups: true, created_after: from).execute + issues = issues.where(projects: { id: options[:projects] }) if options[:projects] issues.count end end diff --git a/spec/lib/gitlab/cycle_analytics/group_stage_summary_spec.rb b/spec/lib/gitlab/cycle_analytics/group_stage_summary_spec.rb index d4dd52ec092..d5c2f7cc579 100644 --- a/spec/lib/gitlab/cycle_analytics/group_stage_summary_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/group_stage_summary_spec.rb @@ -8,7 +8,7 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do let(:from) { 1.day.ago } let(:user) { create(:user, :admin) } - subject { described_class.new(group, from: Time.now, current_user: user, options: {}).data } + subject { described_class.new(group, options: { from: Time.now, current_user: user }).data } describe "#new_issues" do context 'with from date' do @@ -38,7 +38,7 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do Timecop.freeze(5.days.from_now) { create(:issue, project: create(:project, namespace: group)) } end - subject { described_class.new(group, from: Time.now, current_user: user, options: { projects: [project.id, project_2.id] }).data } + subject { described_class.new(group, options: { from: Time.now, current_user: user, projects: [project.id, project_2.id] }).data } it 'finds issues from those projects' do expect(subject.first[:value]).to eq(2) @@ -91,7 +91,7 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do end end - subject { described_class.new(group, from: Time.now, current_user: user, options: { projects: [project.id, project_2.id] }).data } + subject { described_class.new(group, options: { from: Time.now, current_user: user, projects: [project.id, project_2.id] }).data } it 'shows deploys from those projects' do expect(subject.second[:value]).to eq(2) @@ -110,6 +110,5 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do expect(subject.second[:value]).to eq(0) end end - end end diff --git a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb index 1052ee69830..dea17e4f3dc 100644 --- a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb @@ -85,11 +85,11 @@ describe Gitlab::CycleAnalytics::IssueStage do end describe '#events' do - it 'exposes merge requests that close issues' do - result = stage.events + subject { stage.events } - expect(result.count).to eq(1) - expect(result.map { |event| event[:title] }).to contain_exactly(issue_2_1.title) + it 'exposes merge requests that close issues' do + expect(subject.count).to eq(1) + expect(subject.map { |event| event[:title] }).to contain_exactly(issue_2_1.title) end end end -- cgit v1.2.1 From 822129b6b71bfff3cf46d59e31349e8d47a48687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C5=82gorzata=20Ksionek?= Date: Tue, 23 Jul 2019 22:00:50 +0200 Subject: Change sql query --- lib/gitlab/cycle_analytics/summary/group/deploy.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/gitlab/cycle_analytics/summary/group/deploy.rb b/lib/gitlab/cycle_analytics/summary/group/deploy.rb index 1476a5fceb8..78d677cf558 100644 --- a/lib/gitlab/cycle_analytics/summary/group/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/group/deploy.rb @@ -18,9 +18,9 @@ module Gitlab private def find_deployments - deployments = Deployment.joins(:project) - .where(projects: { id: projects.ids }) - .where("deployments.created_at > ?", from) + deployments = Deployment.joins(:project).merge(Project.inside_path(group.full_path)) + deployments = deployments.where(projects: { id: options[:projects] }) if options[:projects] + deployments = deployments.where("deployments.created_at > ?", from) deployments.success.count end end -- cgit v1.2.1