diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-22 21:09:50 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-22 21:09:50 +0000 |
| commit | f05ceb978a12b289fe4e3575420b8c9316041e3a (patch) | |
| tree | 3c1e7c3cf846e1af0ff0edb5867134ca477b3b58 | |
| parent | 3ce55b46dfae23d14818ed2630891be8aa3e83e0 (diff) | |
| download | gitlab-ce-f05ceb978a12b289fe4e3575420b8c9316041e3a.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
83 files changed, 944 insertions, 1904 deletions
diff --git a/app/assets/javascripts/notes.js b/app/assets/javascripts/notes.js index f4982507adb..ef7819c170a 100644 --- a/app/assets/javascripts/notes.js +++ b/app/assets/javascripts/notes.js @@ -1336,11 +1336,12 @@ export default class Notes { toggleCommitList(e) { const $element = $(e.currentTarget); const $closestSystemCommitList = $element.siblings('.system-note-commit-list'); + const $svgChevronUpElement = $element.find('svg.js-chevron-up'); + const $svgChevronDownElement = $element.find('svg.js-chevron-down'); + + $svgChevronUpElement.toggleClass('gl-display-none'); + $svgChevronDownElement.toggleClass('gl-display-none'); - $element - .find('.fa') - .toggleClass('fa-angle-down') - .toggleClass('fa-angle-up'); $closestSystemCommitList.toggleClass('hide-shade'); } diff --git a/app/assets/javascripts/pipelines/components/test_reports/test_suite_table.vue b/app/assets/javascripts/pipelines/components/test_reports/test_suite_table.vue index d57b1466177..6a85476ac47 100644 --- a/app/assets/javascripts/pipelines/components/test_reports/test_suite_table.vue +++ b/app/assets/javascripts/pipelines/components/test_reports/test_suite_table.vue @@ -71,22 +71,14 @@ export default { > <div class="table-section section-20 section-wrap"> <div role="rowheader" class="table-mobile-header">{{ __('Suite') }}</div> - <div - v-gl-tooltip - :title="testCase.classname" - class="table-mobile-content pr-md-1 text-truncate" - > + <div class="table-mobile-content pr-md-1 gl-overflow-wrap-break"> {{ testCase.classname }} </div> </div> <div class="table-section section-20 section-wrap"> <div role="rowheader" class="table-mobile-header">{{ __('Name') }}</div> - <div - v-gl-tooltip - :title="testCase.name" - class="table-mobile-content pr-md-1 text-truncate" - > + <div class="table-mobile-content pr-md-1 gl-overflow-wrap-break"> {{ testCase.name }} </div> </div> diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index ff6d9350a5c..fc53b5fafaf 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -6,7 +6,8 @@ class SearchController < ApplicationController include RendersCommits SCOPE_PRELOAD_METHOD = { - projects: :with_web_entity_associations + projects: :with_web_entity_associations, + issues: :with_web_entity_associations }.freeze around_action :allow_gitaly_ref_name_caching diff --git a/app/models/audit_event_partitioned.rb b/app/models/audit_event_partitioned.rb new file mode 100644 index 00000000000..672daebd14a --- /dev/null +++ b/app/models/audit_event_partitioned.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# This model is not yet intended to be used. +# It is in a transitioning phase while we are partitioning +# the table on the database-side. +# Please refer to https://gitlab.com/groups/gitlab-org/-/epics/3206 +# for details. +class AuditEventPartitioned < ApplicationRecord + include PartitionedTable + + self.table_name = 'audit_events_part_5fc467ac26' + + partitioned_by :created_at, strategy: :monthly +end diff --git a/app/models/clusters/applications/runner.rb b/app/models/clusters/applications/runner.rb index 3db68ae7ce7..9f2ce09e9c0 100644 --- a/app/models/clusters/applications/runner.rb +++ b/app/models/clusters/applications/runner.rb @@ -3,7 +3,7 @@ module Clusters module Applications class Runner < ApplicationRecord - VERSION = '0.19.0' + VERSION = '0.19.1' self.table_name = 'clusters_applications_runners' diff --git a/app/models/issue.rb b/app/models/issue.rb index 487230b14fa..f374d15a88f 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -87,6 +87,8 @@ class Issue < ApplicationRecord scope :order_created_at_desc, -> { reorder(created_at: :desc) } scope :preload_associated_models, -> { preload(:assignees, :labels, project: :namespace) } + scope :with_web_entity_associations, -> { preload(:author, :project) } + scope :with_api_entity_associations, -> { preload(:timelogs, :assignees, :author, :notes, :labels, project: [:route, { namespace: :route }] ) } scope :with_label_attributes, ->(label_attributes) { joins(:labels).where(labels: label_attributes) } scope :with_alert_management_alerts, -> { joins(:alert_management_alert) } scope :with_prometheus_alert_events, -> { joins(:issues_prometheus_alert_events) } diff --git a/app/views/shared/milestones/_deprecation_message.html.haml b/app/views/shared/milestones/_deprecation_message.html.haml deleted file mode 100644 index 27cd6d75232..00000000000 --- a/app/views/shared/milestones/_deprecation_message.html.haml +++ /dev/null @@ -1,15 +0,0 @@ -.banner-callout.compact.milestone-deprecation-message.js-milestone-deprecation-message.prepend-top-20 - .banner-graphic= image_tag 'illustrations/milestone_removing-page.svg' - .banner-body.gl-ml-3.gl-mr-3 - %h5.banner-title.gl-mt-0= _('This page will be removed in a future release.') - %p.milestone-banner-text= _('Use group milestones to manage issues from multiple projects in the same milestone.') - = button_tag _('Promote these project milestones into a group milestone.'), class: 'btn btn-link js-popover-link text-align-left milestone-banner-link' - .milestone-banner-buttons.prepend-top-20= link_to _('Learn more'), help_page_url('user/project/milestones/index', anchor: 'promoting-project-milestones-to-group-milestones'), class: 'btn btn-default', target: '_blank' - - %template.js-milestone-deprecation-message-template - .milestone-popover-body - %ol.milestone-popover-instructions-list.gl-mb-0 - %li= _('Click any <strong>project name</strong> in the project list below to navigate to the project milestone.').html_safe - %li= _('Click the <strong>Promote</strong> button in the top right corner to promote it to a group milestone.').html_safe - %hr.popover-hr - .milestone-popover-footer= link_to _('Learn more'), help_page_url('user/project/milestones/index', anchor: 'promoting-project-milestones-to-group-milestones'), class: 'btn btn-link prepend-left-0', target: '_blank' diff --git a/app/views/shared/notes/_note.html.haml b/app/views/shared/notes/_note.html.haml index 95450a5df3c..c75176cb623 100644 --- a/app/views/shared/notes/_note.html.haml +++ b/app/views/shared/notes/_note.html.haml @@ -63,7 +63,8 @@ - if note.system .system-note-commit-list-toggler.hide = _("Toggle commit list") - %i.fa.fa-angle-down + = sprite_icon('chevron-down', size: 16, css_class: 'js-chevron-down gl-ml-1 gl-vertical-align-text-bottom') + = sprite_icon('chevron-up', size: 16, css_class: 'js-chevron-up gl-ml-1 gl-vertical-align-text-bottom gl-display-none') - if note.attachment.url .note-attachment - if note.attachment.image? diff --git a/changelogs/unreleased/225893-replace-fa-angle-up-icons-with-gitlab-svg-angle-up-icon-2.yml b/changelogs/unreleased/225893-replace-fa-angle-up-icons-with-gitlab-svg-angle-up-icon-2.yml new file mode 100644 index 00000000000..437ff22f6d7 --- /dev/null +++ b/changelogs/unreleased/225893-replace-fa-angle-up-icons-with-gitlab-svg-angle-up-icon-2.yml @@ -0,0 +1,5 @@ +--- +title: Replace fa-angle-up icons with GitLab SVG +merge_request: 36429 +author: +type: changed diff --git a/changelogs/unreleased/fix-search-issues-n-plus-1.yml b/changelogs/unreleased/fix-search-issues-n-plus-1.yml new file mode 100644 index 00000000000..4091fbe9df1 --- /dev/null +++ b/changelogs/unreleased/fix-search-issues-n-plus-1.yml @@ -0,0 +1,5 @@ +--- +title: Avoid N+1 of issue associations in Search +merge_request: 36941 +author: +type: performance diff --git a/changelogs/unreleased/issue_215364_move_usage_data.yml b/changelogs/unreleased/issue_215364_move_usage_data.yml new file mode 100644 index 00000000000..b005507e1e7 --- /dev/null +++ b/changelogs/unreleased/issue_215364_move_usage_data.yml @@ -0,0 +1,5 @@ +--- +title: Move service desk usage data to core +merge_request: 37080 +author: +type: changed diff --git a/changelogs/unreleased/migrate-all-merge-request-user-mentions.yml b/changelogs/unreleased/migrate-all-merge-request-user-mentions.yml new file mode 100644 index 00000000000..4af0e9d64c9 --- /dev/null +++ b/changelogs/unreleased/migrate-all-merge-request-user-mentions.yml @@ -0,0 +1,5 @@ +--- +title: Store user mentions from merge request title or description in the DB +merge_request: 34378 +author: +type: changed diff --git a/changelogs/unreleased/update-gitlab-runner-helm-chart-to-0-19-1.yml b/changelogs/unreleased/update-gitlab-runner-helm-chart-to-0-19-1.yml new file mode 100644 index 00000000000..b7791705e9f --- /dev/null +++ b/changelogs/unreleased/update-gitlab-runner-helm-chart-to-0-19-1.yml @@ -0,0 +1,5 @@ +--- +title: Update GitLab Runner Helm Chart to 0.19.1 +merge_request: 37583 +author: +type: other diff --git a/config/initializers/database_config.rb b/config/initializers/database_config.rb index ce732677c74..79ca6f9354a 100644 --- a/config/initializers/database_config.rb +++ b/config/initializers/database_config.rb @@ -20,11 +20,44 @@ Gitlab.ee do end end -# When running on multi-threaded runtimes like Puma or Sidekiq, -# set the number of threads per process as the minimum DB connection pool size. -# This is to avoid connectivity issues as was documented here: -# https://github.com/rails/rails/pull/23057 -if Gitlab::Runtime.multi_threaded? +# TODO get rid of feature flag https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/495 +if ENV['DB_USE_NEW_POOL_SIZE_LOGIC'] == '1' + # Because of the way Ruby on Rails manages database connections, it is + # important that we have at least as many connections as we have + # threads. While there is a 'pool' setting in database.yml, it is not + # very practical because you need to maintain it in tandem with the + # number of application threads. Because of this we override the number + # of allowed connections in the database connection pool based on the + # configured number of application threads. + # + # Gitlab::Runtime.max_threads is the number of "user facing" application + # threads the process has been configured with. We also have auxiliary + # threads that use database connections. Because it is not practical to + # keep an accurate count of the number auxiliary threads as the + # application evolves over time, we just add a fixed headroom to the + # number of user-facing threads. It is OK if this number is too large + # because connections are instantiated lazily. + + headroom = (ENV["DB_POOL_HEADROOM"].presence || 10).to_i + calculated_pool_size = Gitlab::Runtime.max_threads + headroom + + db_config = Gitlab::Database.config || + Rails.application.config.database_configuration[Rails.env] + + db_config['pool'] = calculated_pool_size + ActiveRecord::Base.establish_connection(db_config) + + Gitlab.ee do + if Gitlab::Runtime.sidekiq? && Gitlab::Geo.geo_database_configured? + Rails.configuration.geo_database['pool'] = calculated_pool_size + Geo::TrackingBase.establish_connection(Rails.configuration.geo_database) + end + end +elsif Gitlab::Runtime.multi_threaded? + # When running on multi-threaded runtimes like Puma or Sidekiq, + # set the number of threads per process as the minimum DB connection pool size. + # This is to avoid connectivity issues as was documented here: + # https://github.com/rails/rails/pull/23057 max_threads = Gitlab::Runtime.max_threads db_config = Gitlab::Database.config || Rails.application.config.database_configuration[Rails.env] diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb index 6c8a72d9bd5..b3f12c2ceb1 100644 --- a/config/initializers/postgres_partitioning.rb +++ b/config/initializers/postgres_partitioning.rb @@ -3,8 +3,10 @@ # Make sure we have loaded partitioned models here # (even with eager loading disabled). +Gitlab::Database::Partitioning::PartitionCreator.register(AuditEventPartitioned) + begin - Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions + Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP'] rescue ActiveRecord::ActiveRecordError, PG::Error # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end diff --git a/danger/roulette/Dangerfile b/danger/roulette/Dangerfile index 6c192c3a311..bf5c6e4c821 100644 --- a/danger/roulette/Dangerfile +++ b/danger/roulette/Dangerfile @@ -42,23 +42,29 @@ OPTIONAL_REVIEW_TEMPLATE = "%{role} review is optional for %{category}".freeze NOT_AVAILABLE_TEMPLATE = 'No %{role} available'.freeze TIMEZONE_EXPERIMENT = true -def mr_author - roulette.team.find { |person| person.username == gitlab.mr_author } +def note_for_spins_role(spins, role) + spins.each do |spin| + note = note_for_spin_role(spin, role) + + return note if note + end + + NOT_AVAILABLE_TEMPLATE % { role: role } end -def note_for_category_role(spin, role) +def note_for_spin_role(spin, role) if spin.optional_role == role return OPTIONAL_REVIEW_TEMPLATE % { role: role.capitalize, category: helper.label_for_category(spin.category) } end - spin.public_send(role)&.markdown_name(timezone_experiment: TIMEZONE_EXPERIMENT, author: mr_author) || NOT_AVAILABLE_TEMPLATE % { role: role } # rubocop:disable GitlabSecurity/PublicSend + spin.public_send(role)&.markdown_name(timezone_experiment: spin.timezone_experiment, author: roulette.team_mr_author) # rubocop:disable GitlabSecurity/PublicSend end -def markdown_row_for_spin(spin) - reviewer_note = note_for_category_role(spin, :reviewer) - maintainer_note = note_for_category_role(spin, :maintainer) +def markdown_row_for_spins(category, spins_array) + reviewer_note = note_for_spins_role(spins_array, :reviewer) + maintainer_note = note_for_spins_role(spins_array, :maintainer) - "| #{helper.label_for_category(spin.category)} | #{reviewer_note} | #{maintainer_note} |" + "| #{helper.label_for_category(category)} | #{reviewer_note} | #{maintainer_note} |" end changes = helper.changes_by_category @@ -70,26 +76,20 @@ changes.delete(:docs) categories = changes.keys - [:unknown] # Ensure to spin for database reviewer/maintainer when ~database is applied (e.g. to review SQL queries) -categories << :database if gitlab.mr_labels.include?('database') && !categories.include?(:database) +categories << :database if helper.gitlab_helper&.mr_labels&.include?('database') && !categories.include?(:database) if changes.any? project = helper.project_name - branch_name = gitlab.mr_json['source_branch'] - markdown(MESSAGE) + timezone_roulette_spins = roulette.spin(project, categories, timezone_experiment: true) + random_roulette_spins = roulette.spin(project, categories, timezone_experiment: false) - roulette_spins = roulette.spin(project, categories, branch_name, timezone_experiment: TIMEZONE_EXPERIMENT) - rows = roulette_spins.map do |spin| - # MR includes QA changes, but also other changes, and author isn't an SET - if spin.category == :qa && categories.size > 1 && mr_author && !mr_author.reviewer?(project, spin.category, []) - spin.optional_role = :maintainer - end - - spin.optional_role = :maintainer if spin.category == :test - - markdown_row_for_spin(spin) + rows = timezone_roulette_spins.map do |spin| + fallback_spin = random_roulette_spins.find { |random_roulette_spins| random_roulette_spins.category == spin.category } + markdown_row_for_spins(spin.category, [spin, fallback_spin]) end + markdown(MESSAGE) markdown(CATEGORY_TABLE_HEADER + rows.join("\n")) unless rows.empty? unknown = changes.fetch(:unknown, []) diff --git a/db/migrate/20200716120000_partition_audit_events.rb b/db/migrate/20200716120000_partition_audit_events.rb new file mode 100644 index 00000000000..aef090339ea --- /dev/null +++ b/db/migrate/20200716120000_partition_audit_events.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class PartitionAuditEvents < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + include Gitlab::Database::PartitioningMigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + partition_table_by_date :audit_events, :created_at + end + + def down + drop_partitioned_table_for :audit_events + end +end diff --git a/db/post_migrate/20200601120434_migrate_all_merge_request_user_mentions_to_db.rb b/db/post_migrate/20200601120434_migrate_all_merge_request_user_mentions_to_db.rb new file mode 100644 index 00000000000..fc7b9afe5f9 --- /dev/null +++ b/db/post_migrate/20200601120434_migrate_all_merge_request_user_mentions_to_db.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class MigrateAllMergeRequestUserMentionsToDb < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + DELAY = 2.minutes.to_i + BATCH_SIZE = 100_000 + MIGRATION = 'UserMentions::CreateResourceUserMention' + + JOIN = "LEFT JOIN merge_request_user_mentions on merge_requests.id = merge_request_user_mentions.merge_request_id" + QUERY_CONDITIONS = "(description LIKE '%@%' OR title LIKE '%@%') AND merge_request_user_mentions.merge_request_id IS NULL" + + disable_ddl_transaction! + + class MergeRequest < ActiveRecord::Base + include EachBatch + end + + def up + delay = DELAY + + MergeRequest.each_batch(of: BATCH_SIZE) do |batch, _| + range = batch.pluck('MIN(merge_requests.id)', 'MAX(merge_requests.id)').first + records_count = MergeRequest.joins(JOIN).where(QUERY_CONDITIONS).where(id: range.first..range.last).count + + if records_count > 0 + migrate_in(delay, MIGRATION, ['MergeRequest', JOIN, QUERY_CONDITIONS, false, *range]) + delay += [DELAY, (records_count / 500 + 1).minutes.to_i].max + end + end + end + + def down + # no-op + end +end diff --git a/db/structure.sql b/db/structure.sql index e1c40897c4e..2ab46dc6b7c 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12,6 +12,78 @@ CREATE EXTENSION IF NOT EXISTS btree_gist WITH SCHEMA public; CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public; +CREATE FUNCTION public.table_sync_function_2be879775d() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF (TG_OP = 'DELETE') THEN + DELETE FROM audit_events_part_5fc467ac26 where id = OLD.id; +ELSIF (TG_OP = 'UPDATE') THEN + UPDATE audit_events_part_5fc467ac26 + SET author_id = NEW.author_id, + type = NEW.type, + entity_id = NEW.entity_id, + entity_type = NEW.entity_type, + details = NEW.details, + updated_at = NEW.updated_at, + ip_address = NEW.ip_address, + author_name = NEW.author_name, + entity_path = NEW.entity_path, + target_details = NEW.target_details, + created_at = NEW.created_at + WHERE audit_events_part_5fc467ac26.id = NEW.id; +ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO audit_events_part_5fc467ac26 (id, + author_id, + type, + entity_id, + entity_type, + details, + updated_at, + ip_address, + author_name, + entity_path, + target_details, + created_at) + VALUES (NEW.id, + NEW.author_id, + NEW.type, + NEW.entity_id, + NEW.entity_type, + NEW.details, + NEW.updated_at, + NEW.ip_address, + NEW.author_name, + NEW.entity_path, + NEW.target_details, + NEW.created_at); +END IF; +RETURN NULL; + +END +$$; + +COMMENT ON FUNCTION public.table_sync_function_2be879775d() IS 'Partitioning migration: table sync for audit_events table'; + +CREATE TABLE public.audit_events_part_5fc467ac26 ( + id bigint NOT NULL, + author_id integer NOT NULL, + type character varying NOT NULL, + entity_id integer NOT NULL, + entity_type character varying NOT NULL, + details text, + updated_at timestamp without time zone, + ip_address inet, + author_name text, + entity_path text, + target_details text, + created_at timestamp without time zone NOT NULL, + CONSTRAINT check_492aaa021d CHECK ((char_length(entity_path) <= 5500)), + CONSTRAINT check_83ff8406e2 CHECK ((char_length(author_name) <= 255)), + CONSTRAINT check_d493ec90b5 CHECK ((char_length(target_details) <= 5500)) +) +PARTITION BY RANGE (created_at); + CREATE TABLE public.product_analytics_events_experimental ( id bigint NOT NULL, project_id integer NOT NULL, @@ -17443,6 +17515,9 @@ ALTER TABLE ONLY public.approvers ALTER TABLE ONLY public.ar_internal_metadata ADD CONSTRAINT ar_internal_metadata_pkey PRIMARY KEY (key); +ALTER TABLE ONLY public.audit_events_part_5fc467ac26 + ADD CONSTRAINT audit_events_part_5fc467ac26_pkey PRIMARY KEY (id, created_at); + ALTER TABLE ONLY public.audit_events ADD CONSTRAINT audit_events_pkey PRIMARY KEY (id); @@ -21054,6 +21129,8 @@ ALTER INDEX public.product_analytics_events_experimental_pkey ATTACH PARTITION g ALTER INDEX public.product_analytics_events_experimental_pkey ATTACH PARTITION gitlab_partitions_static.product_analytics_events_experimental_63_pkey; +CREATE TRIGGER table_sync_trigger_ee39a25f9d AFTER INSERT OR DELETE OR UPDATE ON public.audit_events FOR EACH ROW EXECUTE PROCEDURE public.table_sync_function_2be879775d(); + ALTER TABLE ONLY public.chat_names ADD CONSTRAINT fk_00797a2bf9 FOREIGN KEY (service_id) REFERENCES public.services(id) ON DELETE CASCADE; @@ -23860,6 +23937,7 @@ COPY "schema_migrations" (version) FROM STDIN; 20200528123703 20200528125905 20200528171933 +20200601120434 20200601210148 20200602013900 20200602013901 @@ -23995,6 +24073,7 @@ COPY "schema_migrations" (version) FROM STDIN; 20200715135130 20200715202659 20200716044023 +20200716120000 20200716120419 20200716145156 20200718040100 diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index c85fb2d2e47..2fdfca00f08 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -1,6 +1,6 @@ --- stage: Manage -group: Analytics +group: Compliance info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- diff --git a/doc/administration/audit_reports.md b/doc/administration/audit_reports.md new file mode 100644 index 00000000000..d5a08b711be --- /dev/null +++ b/doc/administration/audit_reports.md @@ -0,0 +1,31 @@ +--- +stage: Manage +group: Compliance +description: 'Learn how to create evidence artifacts typically requested by a 3rd party auditor.' +--- + +# Audit Reports + +GitLab can help owners and administrators respond to auditors by generating +comprehensive reports. These **Audit Reports** vary in scope, depending on the +need: + +## Use cases + +- Generate a report of audit events to provide to an external auditor requesting proof of certain logging capabilities. +- Provide a report of all users showing their group and project memberships for a quarterly access review so the auditor can verify compliance with an organization's access management policy. + +## APIs + +- `https://docs.gitlab.com/ee/api/audit_events.html` +- `https://docs.gitlab.com/ee/api/graphql/reference/#user` +- `https://docs.gitlab.com/ee/api/graphql/reference/#groupmember` +- `https://docs.gitlab.com/ee/api/graphql/reference/#projectmember` + +## Features + +- `https://docs.gitlab.com/ee/administration/audit_events.html` +- `https://docs.gitlab.com/ee/administration/logs.html` + +We plan on making Audit Events [downloadable as a CSV](https://gitlab.com/gitlab-org/gitlab/-/issues/1449) +in the near future. diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index c0b6dfbe7a6..98f138fcbab 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -5055,6 +5055,11 @@ type Group { id: ID! """ + Status of the temporary storage increase + """ + isTemporaryStorageIncreaseEnabled: Boolean! + + """ Issues of the group """ issues( @@ -8317,6 +8322,11 @@ type Namespace { id: ID! """ + Status of the temporary storage increase + """ + isTemporaryStorageIncreaseEnabled: Boolean! + + """ Indicates if Large File Storage (LFS) is enabled for namespace """ lfsEnabled: Boolean diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index ccf2daeb2ee..930eac84e9c 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -14080,6 +14080,24 @@ "deprecationReason": null }, { + "name": "isTemporaryStorageIncreaseEnabled", + "description": "Status of the temporary storage increase", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "issues", "description": "Issues of the group", "args": [ @@ -24825,6 +24843,24 @@ "deprecationReason": null }, { + "name": "isTemporaryStorageIncreaseEnabled", + "description": "Status of the temporary storage increase", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "lfsEnabled", "description": "Indicates if Large File Storage (LFS) is enabled for namespace", "args": [ diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 8c0eeb605b4..2805c5ce71d 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -821,6 +821,7 @@ Autogenerated return type of EpicTreeReorder | `fullPath` | ID! | Full path of the namespace | | `groupTimelogsEnabled` | Boolean | Indicates if Group timelogs are enabled for namespace | | `id` | ID! | ID of the namespace | +| `isTemporaryStorageIncreaseEnabled` | Boolean! | Status of the temporary storage increase | | `label` | Label | A label available on this group | | `lfsEnabled` | Boolean | Indicates if Large File Storage (LFS) is enabled for namespace | | `mentionsDisabled` | Boolean | Indicates if a group is disabled from getting mentioned | @@ -1278,6 +1279,7 @@ Contains statistics about a milestone | `fullName` | String! | Full name of the namespace | | `fullPath` | ID! | Full path of the namespace | | `id` | ID! | ID of the namespace | +| `isTemporaryStorageIncreaseEnabled` | Boolean! | Status of the temporary storage increase | | `lfsEnabled` | Boolean | Indicates if Large File Storage (LFS) is enabled for namespace | | `name` | String! | Name of the namespace | | `path` | String! | Path of the namespace | diff --git a/doc/articles/index.md b/doc/articles/index.md index 7fc4c18f771..4b965b0256f 100644 --- a/doc/articles/index.md +++ b/doc/articles/index.md @@ -1,18 +1,5 @@ --- -comments: false +redirect_to: '../topics/index.md' --- -# Technical articles list (deprecated) - -Technical articles are -topic-related documentation, written with a user-friendly approach and language, aiming -to provide the community with guidance on specific processes to achieve certain objectives. - -The list of technical articles was [deprecated](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41138) in favor of having them linked from their topic-related documentation: - -- [Git](../topics/git/index.md) -- [GitLab administrator](../administration/index.md) -- [GitLab CI/CD](../ci/README.md) -- [GitLab Pages](../user/project/pages/index.md) -- [GitLab user](../user/index.md) -- [Install GitLab](../install/README.md) +This document was moved to [another location](../topics/index.md) diff --git a/doc/development/documentation/structure.md b/doc/development/documentation/structure.md index 4cfc57aa57b..54118442cd1 100644 --- a/doc/development/documentation/structure.md +++ b/doc/development/documentation/structure.md @@ -27,7 +27,7 @@ be used. There is no need to add a title called "Introduction" or "Overview," be search for these terms. Just put this information after the title. - **Use cases**: describes real use case scenarios for that feature/configuration. - **Requirements**: describes what software, configuration, account, or knowledge is required. -- **Instructions**: One or more sets of detailed instructions to follow. +- **Instructions**: one or more sets of detailed instructions to follow. - **Troubleshooting** guide (recommended but not required). For additional details on each, see the [template for new docs](#template-for-new-docs), @@ -167,3 +167,23 @@ Disqus, therefore, don't add both keys to the same document. The click events in the feedback section are tracked with Google Tag Manager. The conversions can be viewed on Google Analytics by navigating to **Behavior > Events > Top events > docs**. + +## Guidelines for good practices + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36576/) in GitLab 13.2 as GitLab Development documentation. + +'Good practice' examples demonstrate encouraged ways of writing code while comparing with examples of practices to avoid. +These examples are labeled as "Bad" or "Good". +In GitLab development guidelines, when presenting the cases, it is recommended +to follow a **first-bad-then-good** strategy. First demonstrate the "Bad" practice (how things _could_ be done, which is often still working code), +and then how things _should_ be done better, using a "Good" example. This is typically an improved example of the same code. + +Consider the following guidelines when offering examples: + +- First, offer the "Bad" example, then the "Good" one. +- When only one bad case and one good case is given, use the same code block. +- When more than one bad case or one good case is offered, use separated code blocks for each. +With many examples being presented, a clear separation helps the reader to go directly to the good part. +Consider offering an explanation (for example, a comment, a link to a resource, etc.) on why something is bad practice. +- Better and best cases can be considered part of the good case(s) code block. +In the same code block, precede each with comments: `# Better` and `# Best`. diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index 5907a3592c9..e81a01b7631 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -170,22 +170,14 @@ Some more examples can be found in the [Frontend unit tests section](testing_lev Another common gotcha is that the specs end up verifying the mock is working. If you are using mocks, the mock should support the test, but not be the target of the test. -**Bad:** - ```javascript const spy = jest.spyOn(idGenerator, 'create') spy.mockImplementation = () = '1234' +// Bad expect(idGenerator.create()).toBe('1234') -``` - -**Good:** - -```javascript -const spy = jest.spyOn(idGenerator, 'create') -spy.mockImplementation = () = '1234' -// Actually focusing on the logic of your component and just leverage the controllable mocks output +// Good: actually focusing on the logic of your component and just leverage the controllable mocks output expect(wrapper.find('div').html()).toBe('<div id="1234">...</div>') ``` @@ -212,22 +204,22 @@ Preferentially, in component testing with `@vue/test-utils`, you should query fo - A `data-testid` attribute ([recommended by maintainers of `@vue/test-utils`](https://github.com/vuejs/vue-test-utils/issues/1498#issuecomment-610133465)) - a Vue `ref` (if using `@vue/test-utils`) -Examples: - ```javascript +// Bad it('exists', () => { - // Good - wrapper.find(FooComponent); - wrapper.find('input[name=foo]'); - wrapper.find('[data-testid="foo"]'); - wrapper.find({ ref: 'foo'}); - - // Bad wrapper.find('.js-foo'); wrapper.find('.btn-primary'); wrapper.find('.qa-foo-component'); wrapper.find('[data-qa-selector="foo"]'); }); + +// Good +it('exists', () => { + wrapper.find(FooComponent); + wrapper.find('input[name=foo]'); + wrapper.find('[data-testid="foo"]'); + wrapper.find({ ref: 'foo'}); +}); ``` It is not recommended that you add `.js-*` classes just for testing purposes. Only do this if there are no other feasible options available. @@ -239,23 +231,26 @@ Do not use a `.qa-*` class or `data-qa-selector` attribute for any tests other t When writing describe test blocks to test specific functions/methods, please use the method name as the describe block name. +**Bad**: + ```javascript -// Good -describe('methodName', () => { +describe('#methodName', () => { it('passes', () => { expect(true).toEqual(true); }); }); -// Bad -describe('#methodName', () => { +describe('.methodName', () => { it('passes', () => { expect(true).toEqual(true); }); }); +``` -// Bad -describe('.methodName', () => { +**Good**: + +```javascript +describe('methodName', () => { it('passes', () => { expect(true).toEqual(true); }); @@ -286,61 +281,67 @@ it('tests a promise rejection', async () => { You can also work with Promise chains. In this case, you can make use of the `done` callback and `done.fail` in case an error occurred. Following are some examples: +**Bad**: + ```javascript -// Good +// missing done callback +it('tests a promise', () => { + promise.then(data => { + expect(data).toBe(asExpected); + }); +}); + +// missing catch it('tests a promise', done => { promise .then(data => { expect(data).toBe(asExpected); }) - .then(done) - .catch(done.fail); + .then(done); }); -// Good -it('tests a promise rejection', done => { +// use done.fail in asynchronous tests +it('tests a promise', done => { promise - .then(done.fail) - .catch(error => { - expect(error).toBe(expectedError); + .then(data => { + expect(data).toBe(asExpected); }) .then(done) - .catch(done.fail); -}); - -// Bad (missing done callback) -it('tests a promise', () => { - promise.then(data => { - expect(data).toBe(asExpected); - }); + .catch(fail); }); -// Bad (missing catch) -it('tests a promise', done => { +// missing catch +it('tests a promise rejection', done => { promise - .then(data => { - expect(data).toBe(asExpected); + .catch(error => { + expect(error).toBe(expectedError); }) .then(done); }); +``` -// Bad (use done.fail in asynchronous tests) +**Good**: + +```javascript +// handling success it('tests a promise', done => { promise .then(data => { expect(data).toBe(asExpected); }) .then(done) - .catch(fail); + .catch(done.fail); }); -// Bad (missing catch) +// failure case it('tests a promise rejection', done => { promise + .then(done.fail) .catch(error => { expect(error).toBe(expectedError); }) - .then(done); + .then(done) + .catch(done.fail); }); ``` @@ -564,11 +565,11 @@ Examples: ```javascript const foo = 1; -// good -expect(foo).toBe(1); - -// bad +// Bad expect(foo).toEqual(1); + +// Good +expect(foo).toBe(1); ``` #### Prefer more befitting matchers @@ -621,12 +622,11 @@ Jest has the tricky `toBeDefined` matcher that can produce false positive test. the given value for `undefined` only. ```javascript -// good -expect(wrapper.find('foo').exists()).toBe(true); - -// bad -// if finder returns null, the test will pass +// Bad: if finder returns null, the test will pass expect(wrapper.find('foo')).toBeDefined(); + +// Good +expect(wrapper.find('foo').exists()).toBe(true); ``` #### Avoid using `setImmediate` diff --git a/doc/operations/feature_flags.md b/doc/operations/feature_flags.md index 56a25ab9c15..194d511f83b 100644 --- a/doc/operations/feature_flags.md +++ b/doc/operations/feature_flags.md @@ -69,9 +69,11 @@ It can be set to: ## Feature flag strategies > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/35555) in GitLab 13.0. -> - It's deployed behind a feature flag, disabled by default. +> - It was deployed behind a feature flag, disabled by default. +> - It became [enabled by default](https://gitlab.com/gitlab-org/gitlab/-/issues/214684) in GitLab 13.2. +> - It's recommended for production use. > - It's enabled on GitLab.com. -> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#enable-or-disable-feature-flag-strategies). **(CORE ONLY)** +> - For GitLab self-managed instances, a GitLab administrator can choose to [disable it](#enable-or-disable-feature-flag-strategies). **(CORE ONLY)** You can apply a feature flag strategy across multiple environments, without defining the strategy multiple times. @@ -140,21 +142,21 @@ activation strategy. ### Enable or disable feature flag strategies -This feature is under development, but is ready for testing. It's +This feature is under development, but is ready for production use. It's deployed behind a feature flag that is **enabled by default**. [GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) can disable it for your instance. -To enable it: +To disable it: ```ruby -Feature.enable(:feature_flags_new_version) +Feature.disable(:feature_flags_new_version) ``` -To disable it: +To enable it: ```ruby -Feature.disable(:feature_flags_new_version) +Feature.enable(:feature_flags_new_version) ``` ## Disable a feature flag for a specific environment diff --git a/doc/operations/metrics/dashboards/variables.md b/doc/operations/metrics/dashboards/variables.md index 66ea94368f8..8b0d7f37052 100644 --- a/doc/operations/metrics/dashboards/variables.md +++ b/doc/operations/metrics/dashboards/variables.md @@ -4,9 +4,9 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Using Variables **(CORE)** +# Using variables **(CORE)** -## Query Variables +## Query variables Variables can be specified using double curly braces, such as `"{{ci_environment_slug}}"` ([added](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20793) in GitLab 12.7). @@ -41,7 +41,7 @@ For example, if the dashboard time range is set to 8 hours, the value of [Variables can be defined](../../../operations/metrics/dashboards/yaml.md#templating-templating-properties) in a custom dashboard YAML file. -## Query Variables from URL +## Query variables from URL > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/214500) in GitLab 13.0. diff --git a/doc/operations/metrics/index.md b/doc/operations/metrics/index.md index 204bcde177e..c48c61ff68b 100644 --- a/doc/operations/metrics/index.md +++ b/doc/operations/metrics/index.md @@ -4,7 +4,7 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Monitor metrics for your CI/CD environment **(CORE)** +# Metrics dashboard for your CI/CD environment **(CORE)** After [configuring Prometheus for a cluster](../../user/project/integrations/prometheus.md), GitLab attempts to retrieve performance metrics for any [environment](../../ci/environments/index.md) with diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 25744253856..a866221151d 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -390,7 +390,9 @@ Here are the requirements for using Dependency Scanning in an offline environmen - Keep Docker-In-Docker disabled (default). - GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). - Docker Container Registry with locally available copies of Dependency Scanning [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. -- Host an offline Git copy of the [gemnasium-db advisory database](https://gitlab.com/gitlab-org/security-products/gemnasium-db/) +- Host an offline Git copy of the [gemnasium-db advisory database](https://gitlab.com/gitlab-org/security-products/gemnasium-db/). + This is required because in an offline environment, the Gemnasium analyzer can't fetch the latest + advisories from the online repository. - _Only if scanning Ruby projects_: Host an offline Git copy of the [advisory database](https://github.com/rubysec/ruby-advisory-db). - _Only if scanning npm/yarn projects_: Host an offline copy of the [retire.js](https://github.com/RetireJS/retire.js/) [node](https://github.com/RetireJS/retire.js/blob/master/repository/npmrepository.json) and [js](https://github.com/RetireJS/retire.js/blob/master/repository/jsrepository.json) advisory databases. @@ -428,8 +430,10 @@ For details on saving and transporting Docker images as a file, see Docker's doc ### Set Dependency Scanning CI job variables to use local Dependency Scanning analyzers -Add the following configuration to your `.gitlab-ci.yml` file. You must replace -`SECURE_ANALYZERS_PREFIX` to refer to your local Docker container registry: +Add the following configuration to your `.gitlab-ci.yml` file. You must change the value of +`SECURE_ANALYZERS_PREFIX` to refer to your local Docker container registry. You must also change the +value of `GEMNASIUM_DB_REMOTE_URL` to the location of your offline Git copy of the +[gemnasium-db advisory database](https://gitlab.com/gitlab-org/security-products/gemnasium-db/): ```yaml include: diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index ee9d5eb9348..f1efa4d6bb6 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -51,7 +51,7 @@ A pipeline consists of multiple jobs, including SAST and DAST scanning. If any j ## Requirements -To run SAST jobs, by default, you need GitLab Runner with the +To run SAST jobs, by default, you need a GitLab Runner with the [`docker`](https://docs.gitlab.com/runner/executors/docker.html) or [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html) executor. If you're using the shared Runners on GitLab.com, this is enabled by default. @@ -208,7 +208,7 @@ Read more on [how to use private Maven repositories](../index.md#using-private-m If needed, you can enable Docker-in-Docker to restore the SAST behavior that existed prior to GitLab 13.0. Follow these steps to do so: -1. Configure GitLab Runner with Docker-inDocker in [privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). +1. Configure a GitLab Runner with Docker-in-Docker in [privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode). 1. Set the variable `SAST_DISABLE_DIND` set to `false`: ```yaml @@ -503,8 +503,8 @@ run successfully. For more information, see [Offline environments](../offline_de To use SAST in an offline environment, you need: - To keep Docker-In-Docker disabled (default). -- GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). -- Docker Container Registry with locally available copies of SAST [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. +- A GitLab Runner with the [`docker` or `kubernetes` executor](#requirements). +- A Docker Container Registry with locally available copies of SAST [analyzer](https://gitlab.com/gitlab-org/security-products/analyzers) images. NOTE: **Note:** GitLab Runner has a [default `pull policy` of `always`](https://docs.gitlab.com/runner/executors/docker.html#using-the-always-pull-policy), diff --git a/doc/user/incident_management/index.md b/doc/user/incident_management/index.md index 6d72dbaa90f..1e8b7e12c32 100644 --- a/doc/user/incident_management/index.md +++ b/doc/user/incident_management/index.md @@ -4,7 +4,7 @@ group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Incident Management +# Incident management GitLab offers solutions for handling incidents in your applications and services, such as setting up Prometheus alerts, displaying metrics, and sending notifications. diff --git a/doc/user/project/integrations/prometheus_library/cloudwatch.md b/doc/user/project/integrations/prometheus_library/cloudwatch.md index 911493cdae9..e278c7eb664 100644 --- a/doc/user/project/integrations/prometheus_library/cloudwatch.md +++ b/doc/user/project/integrations/prometheus_library/cloudwatch.md @@ -4,7 +4,7 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Monitoring AWS Resources +# Monitoring AWS resources > [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/12621) in GitLab 9.4 diff --git a/doc/user/project/operations/alert_management.md b/doc/user/project/operations/alert_management.md index 1a96519edd6..40736537d70 100644 --- a/doc/user/project/operations/alert_management.md +++ b/doc/user/project/operations/alert_management.md @@ -4,7 +4,7 @@ group: Health info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Alert Management +# Alert management > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2877) in GitLab 13.0. diff --git a/doc/user/project/operations/dashboard_settings.md b/doc/user/project/operations/dashboard_settings.md index f30ce5024d6..ba5d410231a 100644 --- a/doc/user/project/operations/dashboard_settings.md +++ b/doc/user/project/operations/dashboard_settings.md @@ -4,7 +4,7 @@ group: APM info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers --- -# Metrics dashboard settings +# Dashboard settings You can configure your [Monitoring dashboard](../integrations/prometheus.md) to display the time zone of your choice, and the links of your choice. diff --git a/doc/user/project/operations/error_tracking.md b/doc/user/project/operations/error_tracking.md index a7a16de90e0..3050b26a6bf 100644 --- a/doc/user/project/operations/error_tracking.md +++ b/doc/user/project/operations/error_tracking.md @@ -8,7 +8,7 @@ info: To determine the technical writer assigned to the Stage/Group associated w > [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/169) in GitLab 11.8. -Error tracking allows developers to easily discover and view the errors that their application may be generating. By surfacing error information where the code is being developed, efficiency and awareness can be increased. +Error Tracking allows developers to easily discover and view the errors that their application may be generating. By surfacing error information where the code is being developed, efficiency and awareness can be increased. ## Sentry error tracking diff --git a/lib/gitlab/background_migration/archive_legacy_traces.rb b/lib/gitlab/background_migration/archive_legacy_traces.rb deleted file mode 100644 index 79f38aed9f1..00000000000 --- a/lib/gitlab/background_migration/archive_legacy_traces.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class ArchiveLegacyTraces - def perform(start_id, stop_id) - # This background migration directly refers to ::Ci::Build model which is defined in application code. - # In general, migration code should be isolated as much as possible in order to be idempotent. - # However, `archive!` method is too complicated to be replicated by coping its subsequent code. - # So we chose a way to use ::Ci::Build directly and we don't change the `archive!` method until 11.1 - ::Ci::Build.finished.without_archived_trace - .where(id: start_id..stop_id).find_each do |build| - build.trace.archive! - rescue => e - Rails.logger.error "Failed to archive live trace. id: #{build.id} message: #{e.message}" # rubocop:disable Gitlab/RailsLogger - end - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb b/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb deleted file mode 100644 index a6194616663..00000000000 --- a/lib/gitlab/background_migration/backfill_hashed_project_repositories.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class that will fill the project_repositories table for projects that - # are on hashed storage and an entry is is missing in this table. - class BackfillHashedProjectRepositories < BackfillProjectRepositories - private - - def projects - Project.on_hashed_storage - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb b/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb deleted file mode 100644 index 2a079060380..00000000000 --- a/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config.rb +++ /dev/null @@ -1,213 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This module is used to write the full path of all projects to - # the git repository config file. - # Storing the full project path in the git config allows admins to - # easily identify a project when it is using hashed storage. - module BackfillProjectFullpathInRepoConfig - OrphanedNamespaceError = Class.new(StandardError) - - module Storage - # Class that returns the disk path for a project using hashed storage - class Hashed - attr_accessor :project - - ROOT_PATH_PREFIX = '@hashed' - - def initialize(project) - @project = project - end - - def disk_path - "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}" - end - - def disk_hash - @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) if project.id - end - end - - # Class that returns the disk path for a project using legacy storage - class LegacyProject - attr_accessor :project - - def initialize(project) - @project = project - end - - def disk_path - project.full_path - end - end - end - - # Concern used by Project and Namespace to determine the full - # route to the project - module Routable - extend ActiveSupport::Concern - - def full_path - @full_path ||= build_full_path - end - - def build_full_path - return path unless has_parent? - - raise OrphanedNamespaceError if parent.nil? - - parent.full_path + '/' + path - end - - def has_parent? - read_attribute(association(:parent).reflection.foreign_key) - end - end - - # Class used to interact with repository using Gitaly - class Repository - attr_reader :storage - - def initialize(storage, relative_path) - @storage = storage - @relative_path = relative_path - end - - def gitaly_repository - Gitaly::Repository.new(storage_name: @storage, relative_path: @relative_path) - end - end - - # Namespace can be a user or group. It can be the root or a - # child of another namespace. - class Namespace < ActiveRecord::Base - self.table_name = 'namespaces' - self.inheritance_column = nil - - include Routable - - belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces' - has_many :projects, inverse_of: :parent - has_many :namespaces, inverse_of: :parent - end - - # Project is where the repository (etc.) is stored - class Project < ActiveRecord::Base - self.table_name = 'projects' - - include Routable - include EachBatch - - FULLPATH_CONFIG_KEY = 'gitlab.fullpath' - - belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects' - delegate :disk_path, to: :storage - - def add_fullpath_config - entries = { FULLPATH_CONFIG_KEY => full_path } - - repository_service.set_config(entries) - end - - def remove_fullpath_config - repository_service.delete_config([FULLPATH_CONFIG_KEY]) - end - - def cleanup_repository - repository_service.cleanup - end - - def storage - @storage ||= - if hashed_storage? - Storage::Hashed.new(self) - else - Storage::LegacyProject.new(self) - end - end - - def hashed_storage? - self.storage_version && self.storage_version >= 1 - end - - def repository - @repository ||= Repository.new(repository_storage, disk_path + '.git') - end - - def repository_service - @repository_service ||= Gitlab::GitalyClient::RepositoryService.new(repository) - end - end - - # Base class for Up and Down migration classes - class BackfillFullpathMigration - RETRY_DELAY = 15.minutes - MAX_RETRIES = 2 - - # Base class for retrying one project - class BaseRetryOne - def perform(project_id, retry_count) - project = Project.find(project_id) - - return unless project - - migration_class.new.safe_perform_one(project, retry_count) - end - end - - def perform(start_id, end_id) - Project.includes(:parent).where(id: start_id..end_id).each do |project| - safe_perform_one(project) - end - end - - def safe_perform_one(project, retry_count = 0) - perform_one(project) - rescue GRPC::NotFound, GRPC::InvalidArgument, OrphanedNamespaceError - nil - rescue GRPC::BadStatus - schedule_retry(project, retry_count + 1) if retry_count < MAX_RETRIES - end - - def schedule_retry(project, retry_count) - # Constants provided to BackgroundMigrationWorker must be within the - # scope of Gitlab::BackgroundMigration - retry_class_name = self.class::RetryOne.name.sub('Gitlab::BackgroundMigration::', '') - - BackgroundMigrationWorker.perform_in(RETRY_DELAY, retry_class_name, [project.id, retry_count]) - end - end - - # Class to add the fullpath to the git repo config - class Up < BackfillFullpathMigration - # Class used to retry - class RetryOne < BaseRetryOne - def migration_class - Up - end - end - - def perform_one(project) - project.cleanup_repository - project.add_fullpath_config - end - end - - # Class to rollback adding the fullpath to the git repo config - class Down < BackfillFullpathMigration - # Class used to retry - class RetryOne < BaseRetryOne - def migration_class - Down - end - end - - def perform_one(project) - project.cleanup_repository - project.remove_fullpath_config - end - end - end - end -end diff --git a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb b/lib/gitlab/background_migration/fill_file_store_job_artifact.rb deleted file mode 100644 index 103bd98af14..00000000000 --- a/lib/gitlab/background_migration/fill_file_store_job_artifact.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FillFileStoreJobArtifact - class JobArtifact < ActiveRecord::Base - self.table_name = 'ci_job_artifacts' - end - - def perform(start_id, stop_id) - FillFileStoreJobArtifact::JobArtifact - .where(file_store: nil) - .where(id: (start_id..stop_id)) - .update_all(file_store: 1) - end - end - end -end diff --git a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb b/lib/gitlab/background_migration/fill_file_store_lfs_object.rb deleted file mode 100644 index 77c1f1ffaf0..00000000000 --- a/lib/gitlab/background_migration/fill_file_store_lfs_object.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FillFileStoreLfsObject - class LfsObject < ActiveRecord::Base - self.table_name = 'lfs_objects' - end - - def perform(start_id, stop_id) - FillFileStoreLfsObject::LfsObject - .where(file_store: nil) - .where(id: (start_id..stop_id)) - .update_all(file_store: 1) - end - end - end -end diff --git a/lib/gitlab/background_migration/fill_store_upload.rb b/lib/gitlab/background_migration/fill_store_upload.rb deleted file mode 100644 index cba3e21cea6..00000000000 --- a/lib/gitlab/background_migration/fill_store_upload.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FillStoreUpload - class Upload < ActiveRecord::Base - self.table_name = 'uploads' - self.inheritance_column = :_type_disabled - end - - def perform(start_id, stop_id) - FillStoreUpload::Upload - .where(store: nil) - .where(id: (start_id..stop_id)) - .update_all(store: 1) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_cross_project_label_links.rb b/lib/gitlab/background_migration/fix_cross_project_label_links.rb deleted file mode 100644 index 20a98c8e141..00000000000 --- a/lib/gitlab/background_migration/fix_cross_project_label_links.rb +++ /dev/null @@ -1,140 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class FixCrossProjectLabelLinks - GROUP_NESTED_LEVEL = 10.freeze - - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - class Label < ActiveRecord::Base - self.inheritance_column = :_type_disabled - self.table_name = 'labels' - end - - class LabelLink < ActiveRecord::Base - self.table_name = 'label_links' - end - - class Issue < ActiveRecord::Base - self.table_name = 'issues' - end - - class MergeRequest < ActiveRecord::Base - self.table_name = 'merge_requests' - end - - class Namespace < ActiveRecord::Base - self.inheritance_column = :_type_disabled - self.table_name = 'namespaces' - - def self.groups_with_descendants_ids(start_id, stop_id) - # To isolate migration code, we avoid usage of - # Gitlab::GroupHierarchy#base_and_descendants which already - # does this job better - ids = Namespace.where(type: 'Group', id: Label.where(type: 'GroupLabel').select('distinct group_id')).where(id: start_id..stop_id).pluck(:id) - group_ids = ids - - GROUP_NESTED_LEVEL.times do - ids = Namespace.where(type: 'Group', parent_id: ids).pluck(:id) - break if ids.empty? - - group_ids += ids - end - - group_ids.uniq - end - end - - def perform(start_id, stop_id) - group_ids = Namespace.groups_with_descendants_ids(start_id, stop_id) - project_ids = Project.where(namespace_id: group_ids).select(:id) - - fix_issues(project_ids) - fix_merge_requests(project_ids) - end - - private - - # select IDs of issues which reference a label which is: - # a) a project label of a different project, or - # b) a group label of a different group than issue's project group - def fix_issues(project_ids) - issue_ids = Label - .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'Issue\' - INNER JOIN issues ON issues.id = label_links.target_id - INNER JOIN projects ON projects.id = issues.project_id') - .where('issues.project_id in (?)', project_ids) - .where('(labels.project_id is not null and labels.project_id != issues.project_id) '\ - 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)') - .select('distinct issues.id') - - Issue.where(id: issue_ids).find_each { |issue| check_resource_labels(issue, issue.project_id) } - end - - # select IDs of MRs which reference a label which is: - # a) a project label of a different project, or - # b) a group label of a different group than MR's project group - def fix_merge_requests(project_ids) - mr_ids = Label - .joins('INNER JOIN label_links ON label_links.label_id = labels.id AND label_links.target_type = \'MergeRequest\' - INNER JOIN merge_requests ON merge_requests.id = label_links.target_id - INNER JOIN projects ON projects.id = merge_requests.target_project_id') - .where('merge_requests.target_project_id in (?)', project_ids) - .where('(labels.project_id is not null and labels.project_id != merge_requests.target_project_id) '\ - 'or (labels.group_id is not null and labels.group_id != projects.namespace_id)') - .select('distinct merge_requests.id') - - MergeRequest.where(id: mr_ids).find_each { |merge_request| check_resource_labels(merge_request, merge_request.target_project_id) } - end - - def check_resource_labels(resource, project_id) - local_labels = available_labels(project_id) - - # get all label links for the given resource (issue/MR) - # which reference a label not included in available_labels - # (other than its project labels and labels of ancestor groups) - cross_labels = LabelLink - .select('label_id, labels.title as title, labels.color as color, label_links.id as label_link_id') - .joins('INNER JOIN labels ON labels.id = label_links.label_id') - .where(target_type: resource.class.name.demodulize, target_id: resource.id) - .where('labels.id not in (?)', local_labels.select(:id)) - - cross_labels.each do |label| - matching_label = local_labels.find {|l| l.title == label.title && l.color == label.color} - - next unless matching_label - - Rails.logger.info "#{resource.class.name.demodulize} #{resource.id}: replacing #{label.label_id} with #{matching_label.id}" # rubocop:disable Gitlab/RailsLogger - LabelLink.update(label.label_link_id, label_id: matching_label.id) - end - end - - # get all labels available for the project (including - # group labels of ancestor groups) - def available_labels(project_id) - @labels ||= {} - @labels[project_id] ||= Label - .where("(type = 'GroupLabel' and group_id in (?)) or (type = 'ProjectLabel' and id = ?)", - project_group_ids(project_id), - project_id) - end - - def project_group_ids(project_id) - ids = [Project.find(project_id).namespace_id] - - GROUP_NESTED_LEVEL.times do - group = Namespace.find(ids.last) - break unless group.parent_id - - ids << group.parent_id - end - - ids - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_build_stage.rb b/lib/gitlab/background_migration/migrate_build_stage.rb deleted file mode 100644 index 268c6083d3c..00000000000 --- a/lib/gitlab/background_migration/migrate_build_stage.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateBuildStage - module Migratable - class Stage < ActiveRecord::Base - self.table_name = 'ci_stages' - end - - class Build < ActiveRecord::Base - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled - - def ensure_stage!(attempts: 2) - find_stage || create_stage! - rescue ActiveRecord::RecordNotUnique - retry if (attempts -= 1) > 0 - raise - end - - def find_stage - Stage.find_by(name: self.stage || 'test', - pipeline_id: self.commit_id, - project_id: self.project_id) - end - - def create_stage! - Stage.create!(name: self.stage || 'test', - pipeline_id: self.commit_id, - project_id: self.project_id) - end - end - end - - def perform(start_id, stop_id) - stages = Migratable::Build.where('stage_id IS NULL') - .where('id BETWEEN ? AND ?', start_id, stop_id) - .map { |build| build.ensure_stage! } - .compact.map(&:id) - - MigrateBuildStageIdReference.new.perform(start_id, stop_id) - MigrateStageStatus.new.perform(stages.min, stages.max) - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb b/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb deleted file mode 100644 index 0a8a4313cd5..00000000000 --- a/lib/gitlab/background_migration/migrate_build_stage_id_reference.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateBuildStageIdReference - def perform(start_id, stop_id) - sql = <<-SQL.strip_heredoc - UPDATE ci_builds - SET stage_id = - (SELECT id FROM ci_stages - WHERE ci_stages.pipeline_id = ci_builds.commit_id - AND ci_stages.name = ci_builds.stage) - WHERE ci_builds.id BETWEEN #{start_id.to_i} AND #{stop_id.to_i} - AND ci_builds.stage_id IS NULL - SQL - - ActiveRecord::Base.connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_stage_index.rb b/lib/gitlab/background_migration/migrate_stage_index.rb deleted file mode 100644 index 55608529cee..00000000000 --- a/lib/gitlab/background_migration/migrate_stage_index.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateStageIndex - def perform(start_id, stop_id) - migrate_stage_index_sql(start_id.to_i, stop_id.to_i).tap do |sql| - ActiveRecord::Base.connection.execute(sql) - end - end - - private - - def migrate_stage_index_sql(start_id, stop_id) - <<~SQL - WITH freqs AS ( - SELECT stage_id, stage_idx, COUNT(*) AS freq FROM ci_builds - WHERE stage_id BETWEEN #{start_id} AND #{stop_id} - AND stage_idx IS NOT NULL - GROUP BY stage_id, stage_idx - ), indexes AS ( - SELECT DISTINCT stage_id, first_value(stage_idx) - OVER (PARTITION BY stage_id ORDER BY freq DESC) AS index - FROM freqs - ) - - UPDATE ci_stages SET position = indexes.index - FROM indexes WHERE indexes.stage_id = ci_stages.id - AND ci_stages.position IS NULL; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb deleted file mode 100644 index fcbcaacb2d6..00000000000 --- a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true -# -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateClusterKubernetesNamespaceTable - include Gitlab::Database::MigrationHelpers - - BATCH_SIZE = 1_000 - - module Migratable - class KubernetesNamespace < ActiveRecord::Base - self.table_name = 'clusters_kubernetes_namespaces' - end - - class ClusterProject < ActiveRecord::Base - include EachBatch - - self.table_name = 'cluster_projects' - - belongs_to :project - - def self.with_no_kubernetes_namespace - where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id)) - end - - def namespace - slug = "#{project.path}-#{project.id}".downcase - slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') - end - - def service_account - "#{namespace}-service-account" - end - end - - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - end - - def perform - cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index| - sql_values = sql_values_for(cluster_projects_batch) - - insert_into_cluster_kubernetes_namespace(sql_values) - end - end - - private - - def cluster_projects_with_no_kubernetes_namespace - Migratable::ClusterProject.with_no_kubernetes_namespace - end - - def sql_values_for(cluster_projects) - cluster_projects.map do |cluster_project| - values_for_cluster_project(cluster_project) - end - end - - def values_for_cluster_project(cluster_project) - { - cluster_project_id: cluster_project.id, - cluster_id: cluster_project.cluster_id, - project_id: cluster_project.project_id, - namespace: cluster_project.namespace, - service_account_name: cluster_project.service_account, - created_at: 'NOW()', - updated_at: 'NOW()' - } - end - - def insert_into_cluster_kubernetes_namespace(rows) - Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, # rubocop:disable Gitlab/BulkInsert - rows, - disable_quote: [:created_at, :updated_at]) - end - end - end -end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb deleted file mode 100644 index 9ef6d8654ae..00000000000 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation -# rubocop:disable Metrics/ClassLength - -module Gitlab - module BackgroundMigration - class RemoveRestrictedTodos - PRIVATE_FEATURE = 10 - PRIVATE_PROJECT = 0 - - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - class ProjectAuthorization < ActiveRecord::Base - self.table_name = 'project_authorizations' - end - - class ProjectFeature < ActiveRecord::Base - self.table_name = 'project_features' - end - - class Todo < ActiveRecord::Base - include EachBatch - - self.table_name = 'todos' - end - - class Issue < ActiveRecord::Base - include EachBatch - - self.table_name = 'issues' - end - - def perform(start_id, stop_id) - projects = Project.where('EXISTS (SELECT 1 FROM todos WHERE todos.project_id = projects.id)') - .where(id: start_id..stop_id) - - projects.each do |project| - remove_confidential_issue_todos(project.id) - - if project.visibility_level == PRIVATE_PROJECT - remove_non_members_todos(project.id) - else - remove_restricted_features_todos(project.id) - end - end - end - - private - - def remove_non_members_todos(project_id) - batch_remove_todos_cte(project_id) - end - - def remove_confidential_issue_todos(project_id) - # min access level to access a confidential issue is reporter - min_reporters = authorized_users(project_id) - .select(:user_id) - .where('access_level >= ?', 20) - - confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id) - confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch| - batch.each do |issue| - assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id) - - todos = Todo.where(target_type: 'Issue', target_id: issue.id) - .where('user_id NOT IN (?)', min_reporters) - .where('user_id NOT IN (?)', assigned_users) - todos = todos.where('user_id != ?', issue.author_id) if issue.author_id - - todos.delete_all - end - end - end - - def remove_restricted_features_todos(project_id) - ProjectFeature.where(project_id: project_id).each do |project_features| - target_types = [] - target_types << 'Issue' if private?(project_features.issues_access_level) - target_types << 'MergeRequest' if private?(project_features.merge_requests_access_level) - target_types << 'Commit' if private?(project_features.repository_access_level) - - next if target_types.empty? - - batch_remove_todos_cte(project_id, target_types) - end - end - - def private?(feature_level) - feature_level == PRIVATE_FEATURE - end - - def authorized_users(project_id) - ProjectAuthorization.select(:user_id).where(project_id: project_id) - end - - def unauthorized_project_todos(project_id) - Todo.where(project_id: project_id) - .where('user_id NOT IN (?)', authorized_users(project_id)) - end - - def batch_remove_todos_cte(project_id, target_types = nil) - loop do - count = remove_todos_cte(project_id, target_types) - - break if count == 0 - end - end - - def remove_todos_cte(project_id, target_types = nil) - sql = [] - sql << with_all_todos_sql(project_id, target_types) - sql << as_deleted_sql - sql << "SELECT count(*) FROM deleted" - - result = Todo.connection.exec_query(sql.join(' ')) - result.rows[0][0].to_i - end - - def with_all_todos_sql(project_id, target_types = nil) - if target_types - table = Arel::Table.new(:todos) - in_target = table[:target_type].in(target_types) - target_types_sql = " AND #{in_target.to_sql}" - end - - <<-SQL - WITH all_todos AS ( - SELECT id - FROM "todos" - WHERE "todos"."project_id" = #{project_id} - AND (user_id NOT IN ( - SELECT "project_authorizations"."user_id" - FROM "project_authorizations" - WHERE "project_authorizations"."project_id" = #{project_id}) - #{target_types_sql} - ) - ), - SQL - end - - def as_deleted_sql - <<-SQL - deleted AS ( - DELETE FROM todos - WHERE id IN ( - SELECT id - FROM all_todos - LIMIT 5000 - ) - RETURNING id - ) - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb deleted file mode 100644 index bc434b0cb64..00000000000 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_services.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - # Ensures services which previously received all notes events continue - # to receive confidential ones. - class SetConfidentialNoteEventsOnServices - class Service < ActiveRecord::Base - self.table_name = 'services' - - include ::EachBatch - - def self.services_to_update - where(confidential_note_events: nil, note_events: true) - end - end - - def perform(start_id, stop_id) - Service.services_to_update - .where(id: start_id..stop_id) - .update_all(confidential_note_events: true) - end - end - end -end diff --git a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb b/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb deleted file mode 100644 index 28d8d2c640b..00000000000 --- a/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - # Ensures hooks which previously received all notes events continue - # to receive confidential ones. - class SetConfidentialNoteEventsOnWebhooks - class WebHook < ActiveRecord::Base - self.table_name = 'web_hooks' - - include ::EachBatch - - def self.hooks_to_update - where(confidential_note_events: nil, note_events: true) - end - end - - def perform(start_id, stop_id) - WebHook.hooks_to_update - .where(id: start_id..stop_id) - .update_all(confidential_note_events: true) - end - end - end -end diff --git a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb index d71a50a0af6..b3876018553 100644 --- a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb @@ -12,26 +12,22 @@ module Gitlab ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models' def perform(resource_model, join, conditions, with_notes, start_id, end_id) + return unless Feature.enabled?(:migrate_user_mentions, default_enabled: true) + resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String) model = with_notes ? Gitlab::BackgroundMigration::UserMentions::Models::Note : resource_model resource_user_mention_model = resource_model.user_mention_model records = model.joins(join).where(conditions).where(id: start_id..end_id) - records.in_groups_of(BULK_INSERT_SIZE, false).each do |records| + records.each_batch(of: BULK_INSERT_SIZE) do |records| mentions = [] records.each do |record| mention_record = record.build_mention_values(resource_user_mention_model.resource_foreign_key) mentions << mention_record unless mention_record.blank? end - Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert - resource_user_mention_model.table_name, - mentions, - return_ids: true, - disable_quote: resource_model.no_quote_columns, - on_conflict: :do_nothing - ) + resource_user_mention_model.insert_all(mentions) unless mentions.empty? end end end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit.rb b/lib/gitlab/background_migration/user_mentions/models/commit.rb index 279e93dbf0d..65f4a7a25b6 100644 --- a/lib/gitlab/background_migration/user_mentions/models/commit.rb +++ b/lib/gitlab/background_migration/user_mentions/models/commit.rb @@ -6,6 +6,7 @@ module Gitlab module UserMentions module Models class Commit + include EachBatch include Concerns::IsolatedMentionable include Concerns::MentionableMigrationMethods diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb index 0cdfc6447c7..bdb90b5d2b9 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb @@ -7,6 +7,7 @@ module Gitlab module Models module DesignManagement class Design < ActiveRecord::Base + include EachBatch include Concerns::MentionableMigrationMethods def self.user_mention_model diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb index dc2b7819800..61d9244a4c9 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb @@ -6,6 +6,7 @@ module Gitlab module UserMentions module Models class Epic < ActiveRecord::Base + include EachBatch include Concerns::IsolatedMentionable include Concerns::MentionableMigrationMethods include CacheMarkdownField diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb index 655c1db71ae..6b52afea17c 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb @@ -6,6 +6,7 @@ module Gitlab module UserMentions module Models class MergeRequest < ActiveRecord::Base + include EachBatch include Concerns::IsolatedMentionable include CacheMarkdownField include Concerns::MentionableMigrationMethods diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb index c32292ad704..a3224c8c456 100644 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ b/lib/gitlab/background_migration/user_mentions/models/note.rb @@ -6,6 +6,7 @@ module Gitlab module UserMentions module Models class Note < ActiveRecord::Base + include EachBatch include Concerns::IsolatedMentionable include CacheMarkdownField diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index b8a27e5448a..0410fe8657d 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -53,7 +53,7 @@ module Gitlab def ee? # Support former project name for `dev` and support local Danger run - %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?('../../ee') + %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../../ee', __dir__)) end def gitlab_helper diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index ed4af3f4a43..2e6181d1cab 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative 'teammate' +require_relative 'request_helper' module Gitlab module Danger @@ -12,45 +13,49 @@ module Gitlab database: false }.freeze - Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role) + Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role, :timezone_experiment) + + def team_mr_author + team.find { |person| person.username == mr_author_username } + end # Assigns GitLab team members to be reviewer and maintainer # for each change category that a Merge Request contains. # # @return [Array<Spin>] - def spin(project, categories, branch_name, timezone_experiment: false) - team = - begin - project_team(project) - rescue => err - warn("Reviewer roulette failed to load team data: #{err.message}") - [] - end - - canonical_branch_name = canonical_branch_name(branch_name) - - spin_per_category = categories.each_with_object({}) do |category, memo| + def spin(project, categories, timezone_experiment: false) + spins = categories.map do |category| including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(category, timezone_experiment) - memo[category] = spin_for_category(team, project, category, canonical_branch_name, timezone_experiment: including_timezone) + spin_for_category(project, category, timezone_experiment: including_timezone) end - spin_per_category.map do |category, spin| - case category + backend_spin = spins.find { |spin| spin.category == :backend } + + spins.each do |spin| + including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(spin.category, timezone_experiment) + case spin.category + when :qa + # MR includes QA changes, but also other changes, and author isn't an SET + if categories.size > 1 && !team_mr_author&.reviewer?(project, spin.category, []) + spin.optional_role = :maintainer + end when :test + spin.optional_role = :maintainer + if spin.reviewer.nil? # Fetch an already picked backend reviewer, or pick one otherwise - spin.reviewer = spin_per_category[:backend]&.reviewer || spin_for_category(team, project, :backend, canonical_branch_name).reviewer + spin.reviewer = backend_spin&.reviewer || spin_for_category(project, :backend, timezone_experiment: including_timezone).reviewer end when :engineering_productivity if spin.maintainer.nil? # Fetch an already picked backend maintainer, or pick one otherwise - spin.maintainer = spin_per_category[:backend]&.maintainer || spin_for_category(team, project, :backend, canonical_branch_name).maintainer + spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer end end - - spin end + + spins end # Looks up the current list of GitLab team members and parses it into a @@ -73,14 +78,9 @@ module Gitlab # @return [Array<Teammate>] def project_team(project_name) team.select { |member| member.in_project?(project_name) } - end - - def canonical_branch_name(branch_name) - branch_name.gsub(/^[ce]e-|-[ce]e$/, '') - end - - def new_random(seed) - Random.new(Digest::MD5.hexdigest(seed).to_i(16)) + rescue => err + warn("Reviewer roulette failed to load team data: #{err.message}") + [] end # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the @@ -113,16 +113,35 @@ module Gitlab # @param [Teammate] person # @return [Boolean] def mr_author?(person) - person.username == gitlab.mr_author + person.username == mr_author_username + end + + def mr_author_username + helper.gitlab_helper&.mr_author || `whoami` + end + + def mr_source_branch + return `git rev-parse --abbrev-ref HEAD` unless helper.gitlab_helper&.mr_json + + helper.gitlab_helper.mr_json['source_branch'] + end + + def mr_labels + helper.gitlab_helper&.mr_labels || [] + end + + def new_random(seed) + Random.new(Digest::MD5.hexdigest(seed).to_i(16)) end def spin_role_for_category(team, role, project, category) team.select do |member| - member.public_send("#{role}?", project, category, gitlab.mr_labels) # rubocop:disable GitlabSecurity/PublicSend + member.public_send("#{role}?", project, category, mr_labels) # rubocop:disable GitlabSecurity/PublicSend end end - def spin_for_category(team, project, category, branch_name, timezone_experiment: false) + def spin_for_category(project, category, timezone_experiment: false) + team = project_team(project) reviewers, traintainers, maintainers = %i[reviewer traintainer maintainer].map do |role| spin_role_for_category(team, role, project, category) @@ -132,11 +151,11 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab/issues/26723 # Make traintainers have triple the chance to be picked as a reviewer - random = new_random(branch_name) + random = new_random(mr_source_branch) reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random, timezone_experiment: timezone_experiment) maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment) - Spin.new(category, reviewer, maintainer) + Spin.new(category, reviewer, maintainer, false, timezone_experiment) end end end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index f7da66e77cd..a06c3554d3d 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -3,10 +3,11 @@ module Gitlab module Danger class Teammate - attr_reader :username, :name, :role, :projects, :available, :tz_offset_hours + attr_reader :options, :username, :name, :role, :projects, :available, :tz_offset_hours # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb def initialize(options = {}) + @options = options @username = options['username'] @name = options['name'] @markdown_name = options['markdown_name'] @@ -16,6 +17,16 @@ module Gitlab @tz_offset_hours = options['tz_offset_hours'] end + def to_h + options + end + + def ==(other) + return false unless other.respond_to?(:username) + + other.username == username + end + def in_project?(name) projects&.has_key?(name) end diff --git a/lib/gitlab/i18n/html_todo.yml b/lib/gitlab/i18n/html_todo.yml index 1eef37c3992..51c46cfc6fb 100644 --- a/lib/gitlab/i18n/html_todo.yml +++ b/lib/gitlab/i18n/html_todo.yml @@ -323,20 +323,6 @@ - "Choisissez entre <code>clone</code> ou <code>fetch</code> pour obtenir les dernières modifications du code de l’application" - "Elija entre <code>clone</code> o <code>fetch</code> para obtener el código de la aplicación más reciente" - "Wybierz pomiędzy <code>klonem</code> lub <code>pobierz</code> aby uzyskać najnowszy kod aplikacji" -"Click any <strong>project name</strong> in the project list below to navigate to the project milestone.": - plural_id: - translations: - - "在專案列表點擊任何<strong>專案名稱</strong>，將轉跳到專案的里程碑。" - - "Clique em qualquer <strong>nome de projeto</strong> na lista a seguir para navegar para o marco do projeto." - - "プロジェクトリストで<strong>プロジェクト名</strong>をクリックすると、プロジェクトのマイルストーンに移動します。" - - "Clica em qualquer <strong>nome de projeto</strong> na lista a seguir para navegar para o objetivo do projeto." - - "Выберите из списка любой <strong>проект</strong>, чтобы перейти к этапу проекта." - - "单击下面项目列表中的任何 <strong>项目名称</strong> 跳转到项目里程碑。" - - "Клікніть по будь-якому <strong>імені проекту</strong> зі списку нижче для того, щоб перейти до етапу проекту." - - "Klicke auf einen beliebigen <strong>-Projektnamen</strong> in der folgenden Projektliste, um zum Projektmeilenstein zu navigieren." - - "아래 프로젝트 목록에서 <strong>프로젝트 이름</strong>을 눌러 프로젝트 마일스톤을 봅니다." - - "Cliquez sur n’importe quel <strong>nom de projet</strong> dans la liste des projets ci‐dessous pour naviguer jusqu’au jalon du projet." - - "Haga clic en cualquier <strong>nombre de proyecto</strong> en la lista de proyectos que se muestra a continuación para navegar hasta el hito de proyecto correspondiente." "Click the <strong>Download</strong> button and wait for downloading to complete.": plural_id: translations: @@ -350,19 +336,6 @@ - "Cliquez sur le bouton <strong>Télécharger</strong> et attendez que le téléchargement soit terminé." - "Haga click en el botón <strong>Descargar</strong> y espere a que se complete la descarga." - "<strong>İndirme</strong> düğmesini tıklayın ve indirme işleminin tamamlanmasını bekleyin." -"Click the <strong>Promote</strong> button in the top right corner to promote it to a group milestone.": - plural_id: - translations: - - "點擊左上角的<strong>提升</strong>按鈕，將提升至群組里程碑。" - - "Clique no botão <strong>Promover</strong> no canto superior direito para promover a um marco de grupo." - - "右上の<strong>昇格</strong>ボタンをクリックしてグループマイルストーンへ昇格" - - "Clica no botão <strong>Promover</strong> no canto superior direito para promovê-lo para um objetivo de grupo." - - "点击右上角的 <strong>升级</strong> 按钮以升级到到群组里程碑。" - - "Натисніть кнопку <strong>Перенести</strong> у правому верхному куті щоб перенести етап на рівень групи." - - "Klicke auf die Schaltfläche <strong>Hochstufen</strong> in der oberen rechten Ecke, um einen ihn zu einem Gruppenmeilenstein hochzustufen." - - "오른쪽 상단 모서리의 <strong>승격</strong> 단추를 눌러 그룹 마일스톤으로 보낼 수 있습니다." - - "Cliquez sur le bouton <strong>Promouvoir</strong> en haut à droite pour le promouvoir en tant que jalon de groupe." - - "Haga clic sobre el botón <strong>Promocionar</strong> , situado en la esquina superior derecha para promocionarlo a un hito de grupo." "Click the <strong>Select none</strong> button on the right, since we only need \\\"Google Code Project Hosting\\\".": plural_id: translations: @@ -1046,3 +1019,37 @@ "DeployFreeze|Specify times when deployments are not allowed for an environment. The <code>gitlab-ci.yml</code> file must be updated to make deployment jobs aware of the %{freeze_period_link_start}freeze period%{freeze_period_link_end}.": plural_id: translations: + +# +# Strings below are fixed in the source code but the translations are still present in CrowdIn so the +# locale files will fail the linter. They can be deleted after next CrowdIn sync, likely in: +# https://gitlab.com/gitlab-org/gitlab/-/issues/226008 +# + +"Click the <strong>Promote</strong> button in the top right corner to promote it to a group milestone.": + plural_id: + translations: + - "點擊左上角的<strong>提升</strong>按鈕，將提升至群組里程碑。" + - "Clique no botão <strong>Promover</strong> no canto superior direito para promover a um marco de grupo." + - "右上の<strong>昇格</strong>ボタンをクリックしてグループマイルストーンへ昇格" + - "Clica no botão <strong>Promover</strong> no canto superior direito para promovê-lo para um objetivo de grupo." + - "点击右上角的 <strong>升级</strong> 按钮以升级到到群组里程碑。" + - "Натисніть кнопку <strong>Перенести</strong> у правому верхному куті щоб перенести етап на рівень групи." + - "Klicke auf die Schaltfläche <strong>Hochstufen</strong> in der oberen rechten Ecke, um einen ihn zu einem Gruppenmeilenstein hochzustufen." + - "오른쪽 상단 모서리의 <strong>승격</strong> 단추를 눌러 그룹 마일스톤으로 보낼 수 있습니다." + - "Cliquez sur le bouton <strong>Promouvoir</strong> en haut à droite pour le promouvoir en tant que jalon de groupe." + - "Haga clic sobre el botón <strong>Promocionar</strong> , situado en la esquina superior derecha para promocionarlo a un hito de grupo." +"Click any <strong>project name</strong> in the project list below to navigate to the project milestone.": + plural_id: + translations: + - "在專案列表點擊任何<strong>專案名稱</strong>，將轉跳到專案的里程碑。" + - "Clique em qualquer <strong>nome de projeto</strong> na lista a seguir para navegar para o marco do projeto." + - "プロジェクトリストで<strong>プロジェクト名</strong>をクリックすると、プロジェクトのマイルストーンに移動します。" + - "Clica em qualquer <strong>nome de projeto</strong> na lista a seguir para navegar para o objetivo do projeto." + - "Выберите из списка любой <strong>проект</strong>, чтобы перейти к этапу проекта." + - "单击下面项目列表中的任何 <strong>项目名称</strong> 跳转到项目里程碑。" + - "Клікніть по будь-якому <strong>імені проекту</strong> зі списку нижче для того, щоб перейти до етапу проекту." + - "Klicke auf einen beliebigen <strong>-Projektnamen</strong> in der folgenden Projektliste, um zum Projektmeilenstein zu navigieren." + - "아래 프로젝트 목록에서 <strong>프로젝트 이름</strong>을 눌러 프로젝트 마일스톤을 봅니다." + - "Cliquez sur n’importe quel <strong>nom de projet</strong> dans la liste des projets ci‐dessous pour naviguer jusqu’au jalon du projet." + - "Haga clic en cualquier <strong>nombre de proyecto</strong> en la lista de proyectos que se muestra a continuación para navegar hasta el hito de proyecto correspondiente." diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index e3055ebc7bc..67df1068aa5 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -159,7 +159,8 @@ module Gitlab usage_counters, user_preferences_usage, ingress_modsecurity_usage, - container_expiration_policies_usage + container_expiration_policies_usage, + service_desk_counts ).tap do |data| data[:snippets] = data[:personal_snippets] + data[:project_snippets] end @@ -527,7 +528,9 @@ module Gitlab issues: distinct_count(::Issue.where(time_period), :author_id), notes: distinct_count(::Note.where(time_period), :author_id), projects: distinct_count(::Project.where(time_period), :creator_id), - todos: distinct_count(::Todo.where(time_period), :author_id) + todos: distinct_count(::Todo.where(time_period), :author_id), + service_desk_enabled_projects: distinct_count_service_desk_enabled_projects(time_period), + service_desk_issues: count(::Issue.service_desk.where(time_period)) } end # rubocop: enable CodeReuse/ActiveRecord @@ -615,6 +618,30 @@ module Gitlab private + def distinct_count_service_desk_enabled_projects(time_period) + project_creator_id_start = user_minimum_id + project_creator_id_finish = user_maximum_id + + distinct_count(::Project.service_desk_enabled.where(time_period), :creator_id, start: project_creator_id_start, finish: project_creator_id_finish) # rubocop: disable CodeReuse/ActiveRecord + end + + # rubocop: disable CodeReuse/ActiveRecord + def service_desk_counts + projects_with_service_desk = ::Project.where(service_desk_enabled: true) + + { + service_desk_enabled_projects: count(projects_with_service_desk), + service_desk_issues: count( + ::Issue.where( + project: projects_with_service_desk, + author: ::User.support_bot, + confidential: true + ) + ) + } + end + # rubocop: enable CodeReuse/ActiveRecord + def unique_visit_service strong_memoize(:unique_visit_service) do ::Gitlab::Analytics::UniqueVisits.new diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 461079c2054..7d79e514ff8 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4875,15 +4875,9 @@ msgstr "" msgid "Clears weight." msgstr "" -msgid "Click any <strong>project name</strong> in the project list below to navigate to the project milestone." -msgstr "" - msgid "Click the <strong>Download</strong> button and wait for downloading to complete." msgstr "" -msgid "Click the <strong>Promote</strong> button in the top right corner to promote it to a group milestone." -msgstr "" - msgid "Click the <strong>Select none</strong> button on the right, since we only need \"Google Code Project Hosting\"." msgstr "" @@ -10702,9 +10696,6 @@ msgstr "" msgid "Generate new token" msgstr "" -msgid "GenericReports|Report" -msgstr "" - msgid "Geo" msgstr "" @@ -18982,9 +18973,6 @@ msgstr "" msgid "Promote issue to an epic" msgstr "" -msgid "Promote these project milestones into a group milestone." -msgstr "" - msgid "Promote to group label" msgstr "" @@ -19901,9 +19889,6 @@ msgstr "" msgid "Reporting" msgstr "" -msgid "Reports" -msgstr "" - msgid "Reports|%{combinedString} and %{resolvedString}" msgstr "" @@ -24029,9 +24014,6 @@ msgstr "" msgid "There was an error when unsubscribing from this label." msgstr "" -msgid "There was an error while fetching configuration data." -msgstr "" - msgid "There was an error while fetching value stream analytics data." msgstr "" @@ -24404,9 +24386,6 @@ msgstr "" msgid "This page is unavailable because you are not allowed to read information across multiple projects." msgstr "" -msgid "This page will be removed in a future release." -msgstr "" - msgid "This pipeline does not use the %{codeStart}needs%{codeEnd} keyword and can't be represented as a directed acyclic graph." msgstr "" @@ -25824,9 +25803,6 @@ msgstr "" msgid "Use custom color #FF0000" msgstr "" -msgid "Use group milestones to manage issues from multiple projects in the same milestone." -msgstr "" - msgid "Use hashed storage" msgstr "" diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index f31da943957..a2a95124b31 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -19,6 +19,7 @@ RSpec.describe 'Database schema' do approver_groups: %w[target_id], approvers: %w[target_id user_id], audit_events: %w[author_id entity_id], + audit_events_part_5fc467ac26: %w[author_id entity_id], award_emoji: %w[awardable_id user_id], aws_roles: %w[role_external_id], boards: %w[milestone_id], diff --git a/spec/initializers/database_config_spec.rb b/spec/initializers/database_config_spec.rb index 29d499efcd3..bdae3e27a48 100644 --- a/spec/initializers/database_config_spec.rb +++ b/spec/initializers/database_config_spec.rb @@ -71,6 +71,60 @@ RSpec.describe 'Database config initializer' do end end + context "with new pool size logic" do + let(:max_threads) { 8 } + + before do + stub_env('DB_USE_NEW_POOL_SIZE_LOGIC', '1') + allow(Gitlab::Runtime).to receive(:max_threads).and_return(max_threads) + end + + context "and no existing pool size is set" do + before do + stub_database_config(pool_size: nil) + end + + it "sets it based on the max number of worker threads" do + expect { subject }.to change { Gitlab::Database.config['pool'] }.from(nil).to(18) + end + end + + context "and the existing pool size is smaller than the max number of worker threads" do + before do + stub_database_config(pool_size: 1) + end + + it "sets it based on the max number of worker threads" do + expect { subject }.to change { Gitlab::Database.config['pool'] }.from(1).to(18) + end + end + + context "and the existing pool size is larger than the max number of worker threads" do + before do + stub_database_config(pool_size: 100) + end + + it "sets it based on the max number of worker threads" do + expect { subject }.to change { Gitlab::Database.config['pool'] }.from(100).to(18) + end + end + + context "when specifying headroom through an ENV variable" do + let(:headroom) { 15 } + + before do + stub_database_config(pool_size: 1) + stub_env("DB_POOL_HEADROOM", headroom) + end + + it "adds headroom on top of the calculated size" do + expect { subject }.to change { Gitlab::Database.config['pool'] } + .from(1) + .to(max_threads + headroom) + end + end + end + def stub_database_config(pool_size:) config = { 'adapter' => 'postgresql', diff --git a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb b/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb deleted file mode 100644 index 7991ad69007..00000000000 --- a/spec/lib/gitlab/background_migration/archive_legacy_traces_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::ArchiveLegacyTraces do - include TraceHelpers - - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:builds) { table(:ci_builds) } - let(:job_artifacts) { table(:ci_job_artifacts) } - - before do - namespaces.create!(id: 123, name: 'gitlab1', path: 'gitlab1') - projects.create!(id: 123, name: 'gitlab1', path: 'gitlab1', namespace_id: 123) - @build = builds.create!(id: 1, project_id: 123, status: 'success', type: 'Ci::Build') - end - - context 'when trace file exsits at the right place' do - before do - create_legacy_trace(@build, 'trace in file') - end - - it 'correctly archive legacy traces' do - expect(job_artifacts.count).to eq(0) - expect(File.exist?(legacy_trace_path(@build))).to be_truthy - - described_class.new.perform(1, 1) - - expect(job_artifacts.count).to eq(1) - expect(File.exist?(legacy_trace_path(@build))).to be_falsy - expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in file') - end - end - - context 'when trace file does not exsits at the right place' do - it 'does not raise errors nor create job artifact' do - expect { described_class.new.perform(1, 1) }.not_to raise_error - - expect(job_artifacts.count).to eq(0) - end - end - - context 'when trace data exsits in database' do - before do - create_legacy_trace_in_db(@build, 'trace in db') - end - - it 'correctly archive legacy traces' do - expect(job_artifacts.count).to eq(0) - expect(@build.read_attribute(:trace)).not_to be_empty - - described_class.new.perform(1, 1) - - @build.reload - expect(job_artifacts.count).to eq(1) - expect(@build.read_attribute(:trace)).to be_nil - expect(File.read(archived_trace_path(job_artifacts.first))).to eq('trace in db') - end - end -end diff --git a/spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb deleted file mode 100644 index 79b344ea6fa..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_hashed_project_repositories_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillHashedProjectRepositories do - it_behaves_like 'backfill migration for project repositories', :hashed -end diff --git a/spec/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config_spec.rb deleted file mode 100644 index 1b2e1ed0c1a..00000000000 --- a/spec/lib/gitlab/background_migration/backfill_project_fullpath_in_repo_config_spec.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:group) { namespaces.create!(name: 'foo', path: 'foo') } - let(:subgroup) { namespaces.create!(name: 'bar', path: 'bar', parent_id: group.id) } - - describe described_class::Storage::Hashed do - let(:project) { double(id: 555) } - - subject(:project_storage) { described_class.new(project) } - - it 'has the correct disk_path' do - expect(project_storage.disk_path).to eq('@hashed/91/a7/91a73fd806ab2c005c13b4dc19130a884e909dea3f72d46e30266fe1a1f588d8') - end - end - - describe described_class::Storage::LegacyProject do - let(:project) { double(full_path: 'this/is/the/full/path') } - - subject(:project_storage) { described_class.new(project) } - - it 'has the correct disk_path' do - expect(project_storage.disk_path).to eq('this/is/the/full/path') - end - end - - describe described_class::Project do - let(:project_record) { projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz') } - - subject(:project) { described_class.find(project_record.id) } - - describe '#full_path' do - it 'returns path containing all parent namespaces' do - expect(project.full_path).to eq('foo/bar/baz') - end - - it 'raises OrphanedNamespaceError when any parent namespace does not exist' do - subgroup.update_attribute(:parent_id, non_existing_record_id) - - expect { project.full_path }.to raise_error(Gitlab::BackgroundMigration::BackfillProjectFullpathInRepoConfig::OrphanedNamespaceError) - end - end - end - - describe described_class::Up do - describe '#perform' do - subject(:migrate) { described_class.new.perform(projects.minimum(:id), projects.maximum(:id)) } - - it 'asks the gitaly client to set config' do - projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz') - projects.create!(namespace_id: subgroup.id, name: 'buzz', path: 'buzz', storage_version: 1) - - expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service| - allow(repository_service).to receive(:cleanup) - expect(repository_service).to receive(:set_config).with('gitlab.fullpath' => 'foo/bar/baz') - end - - expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service| - allow(repository_service).to receive(:cleanup) - expect(repository_service).to receive(:set_config).with('gitlab.fullpath' => 'foo/bar/buzz') - end - - migrate - end - end - end - - describe described_class::Down do - describe '#perform' do - subject(:migrate) { described_class.new.perform(projects.minimum(:id), projects.maximum(:id)) } - - it 'asks the gitaly client to set config' do - projects.create!(namespace_id: subgroup.id, name: 'baz', path: 'baz') - - expect_next_instance_of(Gitlab::GitalyClient::RepositoryService) do |repository_service| - allow(repository_service).to receive(:cleanup) - expect(repository_service).to receive(:delete_config).with(['gitlab.fullpath']) - end - - migrate - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/fix_cross_project_label_links_spec.rb b/spec/lib/gitlab/background_migration/fix_cross_project_label_links_spec.rb deleted file mode 100644 index 8e3ace083fc..00000000000 --- a/spec/lib/gitlab/background_migration/fix_cross_project_label_links_spec.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::FixCrossProjectLabelLinks do - let(:namespaces_table) { table(:namespaces) } - let(:projects_table) { table(:projects) } - let(:issues_table) { table(:issues) } - let(:merge_requests_table) { table(:merge_requests) } - let(:labels_table) { table(:labels) } - let(:label_links_table) { table(:label_links) } - - let!(:group1) { namespaces_table.create(id: 10, type: 'Group', name: 'group1', path: 'group1') } - let!(:group2) { namespaces_table.create(id: 20, type: 'Group', name: 'group2', path: 'group2') } - - let!(:project1) { projects_table.create(id: 1, name: 'project1', path: 'group1/project1', namespace_id: 10) } - let!(:project2) { projects_table.create(id: 3, name: 'project2', path: 'group1/project2', namespace_id: 20) } - - let!(:label1) { labels_table.create(id: 1, title: 'bug', color: 'red', group_id: 10, type: 'GroupLabel') } - let!(:label2) { labels_table.create(id: 2, title: 'bug', color: 'red', group_id: 20, type: 'GroupLabel') } - - def create_merge_request(id, project_id) - merge_requests_table.create(id: id, - target_project_id: project_id, - target_branch: 'master', - source_project_id: project_id, - source_branch: 'mr name', - title: "mr name#{id}") - end - - def create_issue(id, project_id) - issues_table.create(id: id, title: "issue#{id}", project_id: project_id) - end - - def create_resource(target_type, id, project_id) - target_type == 'Issue' ? create_issue(id, project_id) : create_merge_request(id, project_id) - end - - shared_examples_for 'resource with cross-project labels' do - it 'updates only cross-project label links which exist in the local project or group' do - create_resource(target_type, 1, 1) - create_resource(target_type, 2, 3) - labels_table.create(id: 3, title: 'bug', color: 'red', project_id: 3, type: 'ProjectLabel') - link = label_links_table.create(label_id: 2, target_type: target_type, target_id: 1) - link2 = label_links_table.create(label_id: 3, target_type: target_type, target_id: 2) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(1) - expect(link2.reload.label_id).to eq(3) - end - - it 'ignores cross-project label links if label color is different' do - labels_table.create(id: 3, title: 'bug', color: 'green', group_id: 20, type: 'GroupLabel') - create_resource(target_type, 1, 1) - link = label_links_table.create(label_id: 3, target_type: target_type, target_id: 1) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(3) - end - - it 'ignores cross-project label links if label name is different' do - labels_table.create(id: 3, title: 'bug1', color: 'red', group_id: 20, type: 'GroupLabel') - create_resource(target_type, 1, 1) - link = label_links_table.create(label_id: 3, target_type: target_type, target_id: 1) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(3) - end - - context 'with nested group' do - before do - namespaces_table.create(id: 11, type: 'Group', name: 'subgroup1', path: 'group1/subgroup1', parent_id: 10) - projects_table.create(id: 2, name: 'subproject1', path: 'group1/subgroup1/subproject1', namespace_id: 11) - create_resource(target_type, 1, 2) - end - - it 'ignores label links referencing ancestor group labels' do - labels_table.create(id: 4, title: 'bug', color: 'red', project_id: 2, type: 'ProjectLabel') - label_links_table.create(label_id: 4, target_type: target_type, target_id: 1) - link = label_links_table.create(label_id: 1, target_type: target_type, target_id: 1) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(1) - end - - it 'checks also issues and MRs in subgroups' do - link = label_links_table.create(label_id: 2, target_type: target_type, target_id: 1) - - subject.perform(1, 100) - - expect(link.reload.label_id).to eq(1) - end - end - end - - context 'resource is Issue' do - it_behaves_like 'resource with cross-project labels' do - let(:target_type) { 'Issue' } - end - end - - context 'resource is Merge Request' do - it_behaves_like 'resource with cross-project labels' do - let(:target_type) { 'MergeRequest' } - end - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb b/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb deleted file mode 100644 index 65d45ec694f..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_build_stage_spec.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateBuildStage do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:pipelines) { table(:ci_pipelines) } - let(:stages) { table(:ci_stages) } - let(:jobs) { table(:ci_builds) } - - let(:statuses) do - { - created: 0, - pending: 1, - running: 2, - success: 3, - failed: 4, - canceled: 5, - skipped: 6, - manual: 7 - } - end - - before do - namespace = namespaces.create!(name: 'gitlab-org', path: 'gitlab-org') - projects.create!(id: 123, name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id) - pipelines.create!(id: 1, project_id: 123, ref: 'master', sha: 'adf43c3a') - - jobs.create!(id: 1, commit_id: 1, project_id: 123, - stage_idx: 2, stage: 'build', status: :success) - jobs.create!(id: 2, commit_id: 1, project_id: 123, - stage_idx: 2, stage: 'build', status: :success) - jobs.create!(id: 3, commit_id: 1, project_id: 123, - stage_idx: 1, stage: 'test', status: :failed) - jobs.create!(id: 4, commit_id: 1, project_id: 123, - stage_idx: 1, stage: 'test', status: :success) - jobs.create!(id: 5, commit_id: 1, project_id: 123, - stage_idx: 3, stage: 'deploy', status: :pending) - jobs.create!(id: 6, commit_id: 1, project_id: 123, - stage_idx: 3, stage: nil, status: :pending) - end - - it 'correctly migrates builds stages' do - expect(stages.count).to be_zero - - described_class.new.perform(1, 6) - - expect(stages.count).to eq 3 - expect(stages.all.pluck(:name)).to match_array %w[test build deploy] - expect(jobs.where(stage_id: nil)).to be_one - expect(jobs.find_by(stage_id: nil).id).to eq 6 - expect(stages.all.pluck(:status)).to match_array [statuses[:success], - statuses[:failed], - statuses[:pending]] - end - - it 'recovers from unique constraint violation only twice', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/28128' do - allow(described_class::Migratable::Stage) - .to receive(:find_by).and_return(nil) - - expect(described_class::Migratable::Stage) - .to receive(:find_by).exactly(3).times - - expect { described_class.new.perform(1, 6) } - .to raise_error ActiveRecord::RecordNotUnique - end - - context 'when invalid class can be loaded due to single table inheritance' do - let(:commit_status) do - jobs.create!(id: 7, commit_id: 1, project_id: 123, stage_idx: 4, - stage: 'post-deploy', status: :failed) - end - - before do - commit_status.update_column(:type, 'SomeClass') - end - - it 'does ignore single table inheritance type' do - expect { described_class.new.perform(1, 7) }.not_to raise_error - expect(jobs.find(7)).to have_attributes(stage_id: (a_value > 0)) - end - end -end diff --git a/spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb b/spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb deleted file mode 100644 index 81874ff7982..00000000000 --- a/spec/lib/gitlab/background_migration/migrate_stage_index_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::MigrateStageIndex do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:pipelines) { table(:ci_pipelines) } - let(:stages) { table(:ci_stages) } - let(:jobs) { table(:ci_builds) } - let(:namespace) { namespaces.create(name: 'gitlab-org', path: 'gitlab-org') } - let(:project) { projects.create!(namespace_id: namespace.id, name: 'gitlab', path: 'gitlab') } - let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a') } - let(:stage1) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'build') } - let(:stage2) { stages.create(project_id: project.id, pipeline_id: pipeline.id, name: 'test') } - - before do - jobs.create!(commit_id: pipeline.id, project_id: project.id, - stage_idx: 2, stage_id: stage1.id) - jobs.create!(commit_id: pipeline.id, project_id: project.id, - stage_idx: 2, stage_id: stage1.id) - jobs.create!(commit_id: pipeline.id, project_id: project.id, - stage_idx: 10, stage_id: stage1.id) - jobs.create!(commit_id: pipeline.id, project_id: project.id, - stage_idx: 3, stage_id: stage2.id) - end - - it 'correctly migrates stages indices' do - expect(stages.all.pluck(:position)).to all(be_nil) - - described_class.new.perform(stage1.id, stage2.id) - - expect(stages.all.order(:id).pluck(:position)).to eq [2, 3] - end -end diff --git a/spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb b/spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb deleted file mode 100644 index 73faca54b52..00000000000 --- a/spec/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table_spec.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::PopulateClusterKubernetesNamespaceTable do - include MigrationHelpers::ClusterHelpers - - let(:migration) { described_class.new } - let(:clusters_table) { table(:clusters) } - let(:cluster_projects_table) { table(:cluster_projects) } - let(:cluster_kubernetes_namespaces_table) { table(:clusters_kubernetes_namespaces) } - let(:projects_table) { table(:projects) } - let(:namespaces_table) { table(:namespaces) } - let(:provider_gcp_table) { table(:cluster_providers_gcp) } - let(:platform_kubernetes_table) { table(:cluster_platforms_kubernetes) } - - before do - create_cluster_project_list(10) - end - - shared_examples 'consistent kubernetes namespace attributes' do - it 'populates namespace and service account information' do - migration.perform - - clusters_with_namespace.each do |cluster| - cluster_project = cluster_projects_table.find_by(cluster_id: cluster.id) - project = projects_table.find(cluster_project.project_id) - kubernetes_namespace = cluster_kubernetes_namespaces_table.find_by(cluster_id: cluster.id) - namespace = "#{project.path}-#{project.id}" - - expect(kubernetes_namespace).to be_present - expect(kubernetes_namespace.cluster_project_id).to eq(cluster_project.id) - expect(kubernetes_namespace.project_id).to eq(cluster_project.project_id) - expect(kubernetes_namespace.cluster_id).to eq(cluster_project.cluster_id) - expect(kubernetes_namespace.namespace).to eq(namespace) - expect(kubernetes_namespace.service_account_name).to eq("#{namespace}-service-account") - end - end - end - - context 'when no Clusters::Project has a Clusters::KubernetesNamespace' do - let(:cluster_projects) { cluster_projects_table.all } - - it 'creates a Clusters::KubernetesNamespace per Clusters::Project' do - expect do - migration.perform - end.to change(Clusters::KubernetesNamespace, :count).by(cluster_projects_table.count) - end - - it_behaves_like 'consistent kubernetes namespace attributes' do - let(:clusters_with_namespace) { clusters_table.all } - end - end - - context 'when every Clusters::Project has Clusters::KubernetesNamespace' do - before do - create_kubernetes_namespace(clusters_table.all) - end - - it 'does not create any Clusters::KubernetesNamespace' do - expect do - migration.perform - end.not_to change(Clusters::KubernetesNamespace, :count) - end - end - - context 'when only some Clusters::Project have Clusters::KubernetesNamespace related' do - let(:with_kubernetes_namespace) { clusters_table.first(6) } - let(:with_no_kubernetes_namespace) { clusters_table.last(4) } - - before do - create_kubernetes_namespace(with_kubernetes_namespace) - end - - it 'creates limited number of Clusters::KubernetesNamespace' do - expect do - migration.perform - end.to change(Clusters::KubernetesNamespace, :count).by(with_no_kubernetes_namespace.count) - end - - it 'does not modify clusters with Clusters::KubernetesNamespace' do - migration.perform - - with_kubernetes_namespace.each do |cluster| - kubernetes_namespace = cluster_kubernetes_namespaces_table.where(cluster_id: cluster.id) - expect(kubernetes_namespace.count).to eq(1) - end - end - - it_behaves_like 'consistent kubernetes namespace attributes' do - let(:clusters_with_namespace) { with_no_kubernetes_namespace } - end - end -end diff --git a/spec/lib/gitlab/background_migration/remove_restricted_todos_spec.rb b/spec/lib/gitlab/background_migration/remove_restricted_todos_spec.rb deleted file mode 100644 index 7019d5d4212..00000000000 --- a/spec/lib/gitlab/background_migration/remove_restricted_todos_spec.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::RemoveRestrictedTodos do - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:users) { table(:users) } - let(:todos) { table(:todos) } - let(:issues) { table(:issues) } - let(:assignees) { table(:issue_assignees) } - let(:project_authorizations) { table(:project_authorizations) } - let(:project_features) { table(:project_features) } - - let(:todo_params) { { author_id: 1, target_type: 'Issue', action: 1, state: :pending } } - - before do - users.create(id: 1, email: 'user@example.com', projects_limit: 10) - users.create(id: 2, email: 'reporter@example.com', projects_limit: 10) - users.create(id: 3, email: 'guest@example.com', projects_limit: 10) - - namespace = namespaces.create(name: 'gitlab-org', path: 'gitlab-org') - projects.create!(id: 1, name: 'project-1', path: 'project-1', visibility_level: 0, namespace_id: namespace.id) - projects.create!(id: 2, name: 'project-2', path: 'project-2', visibility_level: 0, namespace_id: namespace.id) - - issues.create(id: 1, project_id: 1) - issues.create(id: 2, project_id: 2) - - project_authorizations.create(user_id: 2, project_id: 2, access_level: 20) # reporter - project_authorizations.create(user_id: 3, project_id: 2, access_level: 10) # guest - - todos.create(todo_params.merge(user_id: 1, project_id: 1, target_id: 1)) # out of project ids range - todos.create(todo_params.merge(user_id: 1, project_id: 2, target_id: 2)) # non member - todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 2)) # reporter - todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 2)) # guest - end - - subject { described_class.new.perform(2, 5) } - - context 'when a project is private' do - it 'removes todos of users without project access' do - expect { subject }.to change { Todo.count }.from(4).to(3) - end - - context 'with a confidential issue' do - it 'removes todos of users without project access and guests for confidential issues' do - issues.create(id: 3, project_id: 2, confidential: true) - issues.create(id: 4, project_id: 1, confidential: true) # not in the batch - todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 3)) - todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 3)) - todos.create(todo_params.merge(user_id: 1, project_id: 1, target_id: 4)) - - expect { subject }.to change { Todo.count }.from(7).to(5) - end - end - end - - context 'when a project is public' do - before do - projects.find(2).update_attribute(:visibility_level, 20) - end - - context 'when all features have the same visibility as the project, no confidential issues' do - it 'does not remove any todos' do - expect { subject }.not_to change { Todo.count } - end - end - - context 'with confidential issues' do - before do - users.create(id: 4, email: 'author@example.com', projects_limit: 10) - users.create(id: 5, email: 'assignee@example.com', projects_limit: 10) - issues.create(id: 3, project_id: 2, confidential: true, author_id: 4) - assignees.create(user_id: 5, issue_id: 3) - - todos.create(todo_params.merge(user_id: 1, project_id: 2, target_id: 3)) # to be deleted - todos.create(todo_params.merge(user_id: 2, project_id: 2, target_id: 3)) # authorized user - todos.create(todo_params.merge(user_id: 3, project_id: 2, target_id: 3)) # to be deleted guest - todos.create(todo_params.merge(user_id: 4, project_id: 2, target_id: 3)) # conf issue author - todos.create(todo_params.merge(user_id: 5, project_id: 2, target_id: 3)) # conf issue assignee - end - - it 'removes confidential issue todos for non authorized users' do - expect { subject }.to change { Todo.count }.from(9).to(7) - end - end - - context 'features visibility restrictions' do - before do - todo_params.merge!(project_id: 2, user_id: 1, target_id: 3) - todos.create(todo_params.merge(user_id: 1, target_id: 3, target_type: 'MergeRequest')) - todos.create(todo_params.merge(user_id: 1, target_id: 3, target_type: 'Commit')) - end - - context 'when issues are restricted to project members' do - before do - project_features.create(issues_access_level: 10, pages_access_level: 10, project_id: 2) - end - - it 'removes non members issue todos' do - expect { subject }.to change { Todo.count }.from(6).to(5) - end - end - - context 'when merge requests are restricted to project members' do - before do - project_features.create(merge_requests_access_level: 10, pages_access_level: 10, project_id: 2) - end - - it 'removes non members issue todos' do - expect { subject }.to change { Todo.count }.from(6).to(5) - end - end - - context 'when repository and merge requests are restricted to project members' do - before do - project_features.create(repository_access_level: 10, merge_requests_access_level: 10, pages_access_level: 10, project_id: 2) - end - - it 'removes non members commit and merge requests todos' do - expect { subject }.to change { Todo.count }.from(6).to(4) - end - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/set_confidential_note_events_on_services_spec.rb b/spec/lib/gitlab/background_migration/set_confidential_note_events_on_services_spec.rb deleted file mode 100644 index 364edf3ed2a..00000000000 --- a/spec/lib/gitlab/background_migration/set_confidential_note_events_on_services_spec.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnServices do - let(:services) { table(:services) } - - describe '#perform' do - it 'migrates services where note_events is true' do - service = services.create(confidential_note_events: nil, note_events: true) - - subject.perform(service.id, service.id) - - expect(service.reload.confidential_note_events).to eq(true) - end - - it 'ignores services where note_events is false' do - service = services.create(confidential_note_events: nil, note_events: false) - - subject.perform(service.id, service.id) - - expect(service.reload.confidential_note_events).to eq(nil) - end - - it 'ignores services where confidential_note_events has already been set' do - service = services.create(confidential_note_events: false, note_events: true) - - subject.perform(service.id, service.id) - - expect(service.reload.confidential_note_events).to eq(false) - end - end -end diff --git a/spec/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks_spec.rb b/spec/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks_spec.rb deleted file mode 100644 index 28b06ac3ba3..00000000000 --- a/spec/lib/gitlab/background_migration/set_confidential_note_events_on_webhooks_spec.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::SetConfidentialNoteEventsOnWebhooks do - let(:web_hooks) { table(:web_hooks) } - - describe '#perform' do - it 'migrates hooks where note_events is true' do - hook = web_hooks.create(confidential_note_events: nil, note_events: true) - - subject.perform(hook.id, hook.id) - - expect(hook.reload.confidential_note_events).to eq(true) - end - - it 'ignores hooks where note_events is false' do - hook = web_hooks.create(confidential_note_events: nil, note_events: false) - - subject.perform(hook.id, hook.id) - - expect(hook.reload.confidential_note_events).to eq(nil) - end - - it 'ignores hooks where confidential_note_events has already been set' do - hook = web_hooks.create(confidential_note_events: false, note_events: true) - - subject.perform(hook.id, hook.id) - - expect(hook.reload.confidential_note_events).to eq(false) - end - end -end diff --git a/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb b/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb index 08a4bbe38ac..392b44d1a1f 100644 --- a/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb +++ b/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb @@ -75,6 +75,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent let(:resource) { merge_request } it_behaves_like 'resource mentions migration', MigrateMergeRequestMentionsToDb, MergeRequest + + context 'when FF disabled' do + before do + stub_feature_flags(migrate_user_mentions: false) + end + + it_behaves_like 'resource migration not run', MigrateMergeRequestMentionsToDb, MergeRequest + end end context 'migrate commit mentions' do @@ -96,6 +104,14 @@ RSpec.describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMent let(:resource) { commit } it_behaves_like 'resource notes mentions migration', MigrateCommitNotesMentionsToDb, Commit + + context 'when FF disabled' do + before do + stub_feature_flags(migrate_user_mentions: false) + end + + it_behaves_like 'resource notes migration not run', MigrateCommitNotesMentionsToDb, Commit + end end end diff --git a/spec/lib/gitlab/danger/helper_spec.rb b/spec/lib/gitlab/danger/helper_spec.rb index e19ec5c51b3..d608708b351 100644 --- a/spec/lib/gitlab/danger/helper_spec.rb +++ b/spec/lib/gitlab/danger/helper_spec.rb @@ -98,21 +98,21 @@ RSpec.describe Gitlab::Danger::Helper do it 'delegates to CHANGELOG-EE.md existence if CI_PROJECT_NAME is set to something else' do stub_env('CI_PROJECT_NAME', 'something else') - expect(Dir).to receive(:exist?).with('../../ee') { true } + expect(Dir).to receive(:exist?).with(File.expand_path('../../../../ee', __dir__)) { true } is_expected.to be_truthy end it 'returns true if ee exists' do stub_env('CI_PROJECT_NAME', nil) - expect(Dir).to receive(:exist?).with('../../ee') { true } + expect(Dir).to receive(:exist?).with(File.expand_path('../../../../ee', __dir__)) { true } is_expected.to be_truthy end it "returns false if ee doesn't exist" do stub_env('CI_PROJECT_NAME', nil) - expect(Dir).to receive(:exist?).with('../../ee') { false } + expect(Dir).to receive(:exist?).with(File.expand_path('../../../../ee', __dir__)) { false } is_expected.to be_falsy end diff --git a/spec/lib/gitlab/danger/roulette_spec.rb b/spec/lib/gitlab/danger/roulette_spec.rb index 676edca2459..9517b0a8f63 100644 --- a/spec/lib/gitlab/danger/roulette_spec.rb +++ b/spec/lib/gitlab/danger/roulette_spec.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require 'fast_spec_helper' require 'webmock/rspec' require 'timecop' @@ -11,102 +10,95 @@ RSpec.describe Gitlab::Danger::Roulette do Timecop.freeze(Time.utc(2020, 06, 22, 10)) { example.run } end + let(:backend_available) { true } + let(:backend_tz_offset_hours) { 2.0 } let(:backend_maintainer) do - { - username: 'backend-maintainer', - name: 'Backend maintainer', - role: 'Backend engineer', - projects: { 'gitlab' => 'maintainer backend' }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'backend-maintainer', + 'name' => 'Backend maintainer', + 'role' => 'Backend engineer', + 'projects' => { 'gitlab' => 'maintainer backend' }, + 'available' => backend_available, + 'tz_offset_hours' => backend_tz_offset_hours + ) end let(:frontend_reviewer) do - { - username: 'frontend-reviewer', - name: 'Frontend reviewer', - role: 'Frontend engineer', - projects: { 'gitlab' => 'reviewer frontend' }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'frontend-reviewer', + 'name' => 'Frontend reviewer', + 'role' => 'Frontend engineer', + 'projects' => { 'gitlab' => 'reviewer frontend' }, + 'available' => true, + 'tz_offset_hours' => 2.0 + ) end let(:frontend_maintainer) do - { - username: 'frontend-maintainer', - name: 'Frontend maintainer', - role: 'Frontend engineer', - projects: { 'gitlab' => "maintainer frontend" }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'frontend-maintainer', + 'name' => 'Frontend maintainer', + 'role' => 'Frontend engineer', + 'projects' => { 'gitlab' => "maintainer frontend" }, + 'available' => true, + 'tz_offset_hours' => 2.0 + ) end let(:software_engineer_in_test) do - { - username: 'software-engineer-in-test', - name: 'Software Engineer in Test', - role: 'Software Engineer in Test, Create:Source Code', - projects: { - 'gitlab' => 'reviewer qa', - 'gitlab-qa' => 'maintainer' - }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'software-engineer-in-test', + 'name' => 'Software Engineer in Test', + 'role' => 'Software Engineer in Test, Create:Source Code', + 'projects' => { 'gitlab' => 'reviewer qa', 'gitlab-qa' => 'maintainer' }, + 'available' => true, + 'tz_offset_hours' => 2.0 + ) end let(:engineering_productivity_reviewer) do - { - username: 'eng-prod-reviewer', - name: 'EP engineer', - role: 'Engineering Productivity', - projects: { 'gitlab' => 'reviewer backend' }, - available: true, - tz_offset_hours: 2.0 - } + Gitlab::Danger::Teammate.new( + 'username' => 'eng-prod-reviewer', + 'name' => 'EP engineer', + 'role' => 'Engineering Productivity', + 'projects' => { 'gitlab' => 'reviewer backend' }, + 'available' => true, + 'tz_offset_hours' => 2.0 + ) end let(:teammate_json) do [ - backend_maintainer, - frontend_maintainer, - frontend_reviewer, - software_engineer_in_test, - engineering_productivity_reviewer + backend_maintainer.to_h, + frontend_maintainer.to_h, + frontend_reviewer.to_h, + software_engineer_in_test.to_h, + engineering_productivity_reviewer.to_h ].to_json end subject(:roulette) { Object.new.extend(described_class) } - def matching_teammate(person) - satisfy do |teammate| - teammate.username == person[:username] && - teammate.name == person[:name] && - teammate.role == person[:role] && - teammate.projects == person[:projects] - end - end - - def matching_spin(category, reviewer: { username: nil }, maintainer: { username: nil }, optional: nil) - satisfy do |spin| - bool = spin.category == category - bool &&= spin.reviewer&.username == reviewer[:username] - - bool &&= - if maintainer - spin.maintainer&.username == maintainer[:username] - else - spin.maintainer.nil? - end - - bool && spin.optional_role == optional + describe 'Spin#==' do + it 'compares Spin attributes' do + spin1 = described_class::Spin.new(:backend, frontend_reviewer, frontend_maintainer, false, false) + spin2 = described_class::Spin.new(:backend, frontend_reviewer, frontend_maintainer, false, false) + spin3 = described_class::Spin.new(:backend, frontend_reviewer, frontend_maintainer, false, true) + spin4 = described_class::Spin.new(:backend, frontend_reviewer, frontend_maintainer, true, false) + spin5 = described_class::Spin.new(:backend, frontend_reviewer, backend_maintainer, false, false) + spin6 = described_class::Spin.new(:backend, backend_maintainer, frontend_maintainer, false, false) + spin7 = described_class::Spin.new(:frontend, frontend_reviewer, frontend_maintainer, false, false) + + expect(spin1).to eq(spin2) + expect(spin1).not_to eq(spin3) + expect(spin1).not_to eq(spin4) + expect(spin1).not_to eq(spin5) + expect(spin1).not_to eq(spin6) + expect(spin1).not_to eq(spin7) end end describe '#spin' do let!(:project) { 'gitlab' } - let!(:branch_name) { 'a-branch' } + let!(:mr_source_branch) { 'a-branch' } let!(:mr_labels) { ['backend', 'devops::create'] } - let!(:author) { Gitlab::Danger::Teammate.new('username' => 'filipa') } + let!(:author) { Gitlab::Danger::Teammate.new('username' => 'johndoe') } let(:timezone_experiment) { false } let(:spins) do # Stub the request at the latest time so that we can modify the raw data, e.g. available fields. @@ -114,12 +106,13 @@ RSpec.describe Gitlab::Danger::Roulette do .stub_request(:get, described_class::ROULETTE_DATA_URL) .to_return(body: teammate_json) - subject.spin(project, categories, branch_name, timezone_experiment: timezone_experiment) + subject.spin(project, categories, timezone_experiment: timezone_experiment) end before do - allow(subject).to receive_message_chain(:gitlab, :mr_author).and_return(author.username) - allow(subject).to receive_message_chain(:gitlab, :mr_labels).and_return(mr_labels) + allow(subject).to receive(:mr_author_username).and_return(author.username) + allow(subject).to receive(:mr_labels).and_return(mr_labels) + allow(subject).to receive(:mr_source_branch).and_return(mr_source_branch) end context 'when timezone_experiment == false' do @@ -127,16 +120,16 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:backend] } it 'assigns backend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins[0].reviewer).to eq(engineering_productivity_reviewer) + expect(spins[0].maintainer).to eq(backend_maintainer) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, backend_maintainer, false, false)]) end context 'when teammate is not available' do - before do - backend_maintainer[:available] = false - end + let(:backend_available) { false } it 'assigns backend reviewer and no maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: nil)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, nil, false, false)]) end end end @@ -145,7 +138,7 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:frontend] } it 'assigns frontend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:frontend, reviewer: frontend_reviewer, maintainer: frontend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:frontend, frontend_reviewer, frontend_maintainer, false, false)]) end end @@ -153,7 +146,7 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:qa] } it 'assigns QA reviewer' do - expect(spins).to contain_exactly(matching_spin(:qa, reviewer: software_engineer_in_test)) + expect(spins).to eq([described_class::Spin.new(:qa, software_engineer_in_test, nil, false, false)]) end end @@ -161,7 +154,7 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:engineering_productivity] } it 'assigns Engineering Productivity reviewer and fallback to backend maintainer' do - expect(spins).to contain_exactly(matching_spin(:engineering_productivity, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:engineering_productivity, engineering_productivity_reviewer, backend_maintainer, false, false)]) end end @@ -169,7 +162,7 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:test] } it 'assigns corresponding SET' do - expect(spins).to contain_exactly(matching_spin(:test, reviewer: software_engineer_in_test)) + expect(spins).to eq([described_class::Spin.new(:test, software_engineer_in_test, nil, :maintainer, false)]) end end end @@ -181,16 +174,14 @@ RSpec.describe Gitlab::Danger::Roulette do let(:categories) { [:backend] } it 'assigns backend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, backend_maintainer, false, true)]) end context 'when teammate is not in a good timezone' do - before do - backend_maintainer[:tz_offset_hours] = 5.0 - end + let(:backend_tz_offset_hours) { 5.0 } it 'assigns backend reviewer and no maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: nil)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, nil, false, true)]) end end end @@ -203,22 +194,33 @@ RSpec.describe Gitlab::Danger::Roulette do end it 'assigns backend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, backend_maintainer, false, false)]) end context 'when teammate is not in a good timezone' do - before do - backend_maintainer[:tz_offset_hours] = 5.0 - end + let(:backend_tz_offset_hours) { 5.0 } it 'assigns backend reviewer and maintainer' do - expect(spins).to contain_exactly(matching_spin(:backend, reviewer: engineering_productivity_reviewer, maintainer: backend_maintainer)) + expect(spins).to eq([described_class::Spin.new(:backend, engineering_productivity_reviewer, backend_maintainer, false, false)]) end end end end end + RSpec::Matchers.define :match_teammates do |expected| + match do |actual| + expected.each do |expected_person| + actual_person_found = actual.find { |actual_person| actual_person.name == expected_person.username } + + actual_person_found && + actual_person_found.name == expected_person.name && + actual_person_found.role == expected_person.role && + actual_person_found.projects == expected_person.projects + end + end + end + describe '#team' do subject(:team) { roulette.team } @@ -254,15 +256,13 @@ RSpec.describe Gitlab::Danger::Roulette do end it 'returns an array of teammates' do - expected_teammates = [ - matching_teammate(backend_maintainer), - matching_teammate(frontend_reviewer), - matching_teammate(frontend_maintainer), - matching_teammate(software_engineer_in_test), - matching_teammate(engineering_productivity_reviewer) - ] - - is_expected.to contain_exactly(*expected_teammates) + is_expected.to match_teammates([ + backend_maintainer, + frontend_reviewer, + frontend_maintainer, + software_engineer_in_test, + engineering_productivity_reviewer + ]) end it 'memoizes the result' do @@ -281,7 +281,9 @@ RSpec.describe Gitlab::Danger::Roulette do end it 'filters team by project_name' do - is_expected.to contain_exactly(matching_teammate(software_engineer_in_test)) + is_expected.to match_teammates([ + software_engineer_in_test + ]) end end @@ -289,32 +291,32 @@ RSpec.describe Gitlab::Danger::Roulette do let(:person_tz_offset_hours) { 0.0 } let(:person1) do Gitlab::Danger::Teammate.new( - 'username' => 'rymai', + 'username' => 'user1', 'available' => true, 'tz_offset_hours' => person_tz_offset_hours ) end let(:person2) do Gitlab::Danger::Teammate.new( - 'username' => 'godfat', + 'username' => 'user2', 'available' => true, 'tz_offset_hours' => person_tz_offset_hours) end let(:author) do Gitlab::Danger::Teammate.new( - 'username' => 'filipa', + 'username' => 'johndoe', 'available' => true, 'tz_offset_hours' => 0.0) end let(:unavailable) do Gitlab::Danger::Teammate.new( - 'username' => 'jacopo-beschi', + 'username' => 'janedoe', 'available' => false, 'tz_offset_hours' => 0.0) end before do - allow(subject).to receive_message_chain(:gitlab, :mr_author).and_return(author.username) + allow(subject).to receive(:mr_author_username).and_return(author.username) end (-4..4).each do |utc_offset| @@ -328,7 +330,7 @@ RSpec.describe Gitlab::Danger::Roulette do selected = subject.spin_for_person(persons, random: Random.new, timezone_experiment: timezone_experiment) - expect(selected.username).to be_in(persons.map(&:username)) + expect(persons.map(&:username)).to include(selected.username) end end end @@ -349,7 +351,7 @@ RSpec.describe Gitlab::Danger::Roulette do if timezone_experiment expect(selected).to be_nil else - expect(selected.username).to be_in(persons.map(&:username)) + expect(persons.map(&:username)).to include(selected.username) end end end diff --git a/spec/lib/gitlab/danger/teammate_spec.rb b/spec/lib/gitlab/danger/teammate_spec.rb index a0540a9fbf5..385eb4e9b69 100644 --- a/spec/lib/gitlab/danger/teammate_spec.rb +++ b/spec/lib/gitlab/danger/teammate_spec.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'fast_spec_helper' - require 'timecop' require 'rspec-parameterized' @@ -10,16 +8,16 @@ require 'gitlab/danger/teammate' RSpec.describe Gitlab::Danger::Teammate do using RSpec::Parameterized::TableSyntax - subject { described_class.new(options.stringify_keys) } + subject { described_class.new(options) } let(:tz_offset_hours) { 2.0 } let(:options) do { - username: 'luigi', - projects: projects, - role: role, - markdown_name: '[Luigi](https://gitlab.com/luigi) (`@luigi`)', - tz_offset_hours: tz_offset_hours + 'username' => 'luigi', + 'projects' => projects, + 'role' => role, + 'markdown_name' => '[Luigi](https://gitlab.com/luigi) (`@luigi`)', + 'tz_offset_hours' => tz_offset_hours } end let(:capabilities) { ['reviewer backend'] } @@ -28,6 +26,26 @@ RSpec.describe Gitlab::Danger::Teammate do let(:labels) { [] } let(:project) { double } + describe '#==' do + it 'compares Teammate username' do + joe1 = described_class.new('username' => 'joe', 'projects' => projects) + joe2 = described_class.new('username' => 'joe', 'projects' => []) + jane1 = described_class.new('username' => 'jane', 'projects' => projects) + jane2 = described_class.new('username' => 'jane', 'projects' => []) + + expect(joe1).to eq(joe2) + expect(jane1).to eq(jane2) + expect(jane1).not_to eq(nil) + expect(described_class.new('username' => nil)).not_to eq(nil) + end + end + + describe '#to_h' do + it 'returns the given options' do + expect(subject.to_h).to eq(options) + end + end + context 'when having multiple capabilities' do let(:capabilities) { ['reviewer backend', 'maintainer frontend', 'trainee_maintainer qa'] } @@ -153,21 +171,21 @@ RSpec.describe Gitlab::Danger::Teammate do describe '#markdown_name' do context 'when timezone_experiment == false' do it 'returns markdown name as-is' do - expect(subject.markdown_name).to eq(options[:markdown_name]) - expect(subject.markdown_name(timezone_experiment: false)).to eq(options[:markdown_name]) + expect(subject.markdown_name).to eq(options['markdown_name']) + expect(subject.markdown_name(timezone_experiment: false)).to eq(options['markdown_name']) end end context 'when timezone_experiment == true' do it 'returns markdown name with timezone info' do - expect(subject.markdown_name(timezone_experiment: true)).to eq("#{options[:markdown_name]} (UTC+2)") + expect(subject.markdown_name(timezone_experiment: true)).to eq("#{options['markdown_name']} (UTC+2)") end context 'when offset is 1.5' do let(:tz_offset_hours) { 1.5 } it 'returns markdown name with timezone info, not truncated' do - expect(subject.markdown_name(timezone_experiment: true)).to eq("#{options[:markdown_name]} (UTC+1.5)") + expect(subject.markdown_name(timezone_experiment: true)).to eq("#{options['markdown_name']} (UTC+1.5)") end end @@ -185,12 +203,12 @@ RSpec.describe Gitlab::Danger::Teammate do with_them do it 'returns markdown name with timezone info' do - author = described_class.new(options.merge(username: 'mario', tz_offset_hours: author_offset).stringify_keys) + author = described_class.new(options.merge('username' => 'mario', 'tz_offset_hours' => author_offset)) floored_offset_hours = subject.__send__(:floored_offset_hours) utc_offset = floored_offset_hours >= 0 ? "+#{floored_offset_hours}" : floored_offset_hours - expect(subject.markdown_name(timezone_experiment: true, author: author)).to eq("#{options[:markdown_name]} (UTC#{utc_offset}, #{diff_text})") + expect(subject.markdown_name(timezone_experiment: true, author: author)).to eq("#{options['markdown_name']} (UTC#{utc_offset}, #{diff_text})") end end end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index a340e69df65..3bc8151b7db 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -155,13 +155,13 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do expect(described_class.uncached_data[:usage_activity_by_stage][:manage]).to include( events: 2, groups: 2, - users_created: Gitlab.ee? ? 6 : 5, + users_created: 6, omniauth_providers: ['google_oauth2'] ) expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:manage]).to include( events: 1, groups: 1, - users_created: Gitlab.ee? ? 4 : 3, + users_created: 4, omniauth_providers: ['google_oauth2'] ) end @@ -203,21 +203,26 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do user = create(:user) project = create(:project, creator: user) issue = create(:issue, project: project, author: user) + create(:issue, project: project, author: User.support_bot) create(:note, project: project, noteable: issue, author: user) create(:todo, project: project, target: issue, author: user) end expect(described_class.uncached_data[:usage_activity_by_stage][:plan]).to include( - issues: 2, + issues: 3, notes: 2, projects: 2, - todos: 2 + todos: 2, + service_desk_enabled_projects: 2, + service_desk_issues: 2 ) expect(described_class.uncached_data[:usage_activity_by_stage_monthly][:plan]).to include( - issues: 1, + issues: 2, notes: 1, projects: 1, - todos: 1 + todos: 1, + service_desk_enabled_projects: 1, + service_desk_issues: 1 ) end end @@ -974,4 +979,17 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do }) end end + + describe '.service_desk_counts' do + subject { described_class.send(:service_desk_counts) } + + let(:project) { create(:project, :service_desk_enabled) } + + it 'gathers Service Desk data' do + create_list(:issue, 2, :confidential, author: User.support_bot, project: project) + + expect(subject).to eq(service_desk_enabled_projects: 1, + service_desk_issues: 2) + end + end end diff --git a/spec/migrations/migrate_all_merge_request_user_mentions_to_db_spec.rb b/spec/migrations/migrate_all_merge_request_user_mentions_to_db_spec.rb new file mode 100644 index 00000000000..f7789469efb --- /dev/null +++ b/spec/migrations/migrate_all_merge_request_user_mentions_to_db_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20200601120434_migrate_all_merge_request_user_mentions_to_db') + +RSpec.describe MigrateAllMergeRequestUserMentionsToDb, :migration do + let(:users) { table(:users) } + let(:projects) { table(:projects) } + let(:namespaces) { table(:namespaces) } + let(:merge_requests) { table(:merge_requests) } + let(:merge_request_user_mentions) { table(:merge_request_user_mentions) } + + let(:user) { users.create!(name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0) } + let(:group) { namespaces.create!(name: 'group1', path: 'group1', owner_id: user.id, type: 'Group') } + let(:project) { projects.create!(name: 'gitlab1', path: 'gitlab1', namespace_id: group.id, visibility_level: 0) } + + let(:opened_state) { 1 } + let(:closed_state) { 2 } + let(:merged_state) { 3 } + + # migrateable resources + let(:common_args) { { source_branch: 'master', source_project_id: project.id, target_project_id: project.id, author_id: user.id, description: 'mr description with @root mention' } } + let!(:resource1) { merge_requests.create!(common_args.merge(title: "title 1", state_id: opened_state, target_branch: 'feature1')) } + let!(:resource2) { merge_requests.create!(common_args.merge(title: "title 2", state_id: closed_state, target_branch: 'feature2')) } + let!(:resource3) { merge_requests.create!(common_args.merge(title: "title 3", state_id: merged_state, target_branch: 'feature3')) } + + # non-migrateable resources + # this merge request is already migrated, as it has a record in the merge_request_user_mentions table + let!(:resource4) { merge_requests.create!(common_args.merge(title: "title 3", state_id: opened_state, target_branch: 'feature4')) } + let!(:user_mention) { merge_request_user_mentions.create!(merge_request_id: resource4.id, mentioned_users_ids: [1]) } + + let!(:resource5) { merge_requests.create!(common_args.merge(title: "title 3", description: 'description with no mention', state_id: opened_state, target_branch: 'feature5')) } + + it_behaves_like 'schedules resource mentions migration', MergeRequest, false +end diff --git a/spec/requests/search_controller_spec.rb b/spec/requests/search_controller_spec.rb new file mode 100644 index 00000000000..52bfc480313 --- /dev/null +++ b/spec/requests/search_controller_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe SearchController, type: :request do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, :public, :repository, :wiki_repo, name: 'awesome project', group: group) } + + before_all do + login_as(user) + end + + def send_search_request(params) + get search_path, params: params + end + + shared_examples 'an efficient database result' do + it 'avoids N+1 database queries' do + create(object, *creation_traits, creation_args) + + control = ActiveRecord::QueryRecorder.new(skip_cached: false) { send_search_request(params) } + create_list(object, 3, *creation_traits, creation_args) + + expect { send_search_request(params) }.not_to exceed_all_query_limit(control).with_threshold(threshold) + end + end + + describe 'GET /search' do + let(:creation_traits) { [] } + + context 'for issues scope' do + let(:object) { :issue } + let(:creation_args) { { project: project } } + let(:params) { { search: '*', scope: 'issues' } } + let(:threshold) { 0 } + + it_behaves_like 'an efficient database result' + end + + context 'for merge_request scope' do + let(:creation_traits) { [:unique_branches] } + let(:object) { :merge_request } + let(:creation_args) { { source_project: project } } + let(:params) { { search: '*', scope: 'merge_requests' } } + let(:threshold) { 0 } + + it_behaves_like 'an efficient database result' + end + + context 'for project scope' do + let(:creation_traits) { [:public] } + let(:object) { :project } + let(:creation_args) { {} } + let(:params) { { search: '*', scope: 'projects' } } + # some N+1 queries still exist + # each project requires 3 extra queries + # - one count for forks + # - one count for open MRs + # - one count for open Issues + let(:threshold) { 9 } + + it_behaves_like 'an efficient database result' + end + end +end diff --git a/spec/support/shared_examples/lib/gitlab/background_migration/mentions_migration_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/background_migration/mentions_migration_shared_examples.rb index af65b61021c..8cf6babe146 100644 --- a/spec/support/shared_examples/lib/gitlab/background_migration/mentions_migration_shared_examples.rb +++ b/spec/support/shared_examples/lib/gitlab/background_migration/mentions_migration_shared_examples.rb @@ -82,3 +82,25 @@ RSpec.shared_examples 'schedules resource mentions migration' do |resource_class end end end + +RSpec.shared_examples 'resource migration not run' do |migration_class, resource_class| + it 'does not migrate mentions' do + join = migration_class::JOIN + conditions = migration_class::QUERY_CONDITIONS + + expect do + subject.perform(resource_class.name, join, conditions, false, resource_class.minimum(:id), resource_class.maximum(:id)) + end.to change { user_mentions.count }.by(0) + end +end + +RSpec.shared_examples 'resource notes migration not run' do |migration_class, resource_class| + it 'does not migrate mentions' do + join = migration_class::JOIN + conditions = migration_class::QUERY_CONDITIONS + + expect do + subject.perform(resource_class.name, join, conditions, true, Note.minimum(:id), Note.maximum(:id)) + end.to change { user_mentions.count }.by(0) + end +end |