diff options
144 files changed, 2131 insertions, 555 deletions
diff --git a/.eslintignore b/.eslintignore index 73e11dfd974..7ca59654678 100644 --- a/.eslintignore +++ b/.eslintignore @@ -7,3 +7,4 @@ /public/ /tmp/ /vendor/ +/sitespeed-result/ diff --git a/.prettierignore b/.prettierignore index ea7e6268b93..2c5104fb1b8 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,6 +4,7 @@ /public/ /vendor/ /tmp/ +/sitespeed-result/ # ignore stylesheets for now as this clashes with our linter *.css diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index cc51fd54f51..08bbe86fd1b 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -d4ea957f6131538cd78e490a585ea3a455251064 +40511f7a14ded77c826809d054d740a66e1c106f diff --git a/app/assets/javascripts/boards/components/board_column.vue b/app/assets/javascripts/boards/components/board_column.vue index cc7262f3a39..69abf886ad7 100644 --- a/app/assets/javascripts/boards/components/board_column.vue +++ b/app/assets/javascripts/boards/components/board_column.vue @@ -41,7 +41,7 @@ export default { watch: { filterParams: { handler() { - if (this.list.id) { + if (this.list.id && !this.list.collapsed) { this.fetchItemsForList({ listId: this.list.id }); } }, diff --git a/app/assets/javascripts/boards/stores/actions.js b/app/assets/javascripts/boards/stores/actions.js index 5c06f284bb7..b52a9a71ae8 100644 --- a/app/assets/javascripts/boards/stores/actions.js +++ b/app/assets/javascripts/boards/stores/actions.js @@ -240,7 +240,7 @@ export default { }, updateList: ( - { commit, state: { issuableType } }, + { commit, state: { issuableType, boardItemsByListId = {} }, dispatch }, { listId, position, collapsed, backupList }, ) => { gqlClient @@ -255,6 +255,12 @@ export default { .then(({ data }) => { if (data?.updateBoardList?.errors.length) { commit(types.UPDATE_LIST_FAILURE, backupList); + return; + } + + // Only fetch when board items havent been fetched on a collapsed list + if (!boardItemsByListId[listId]) { + dispatch('fetchItemsForList', { listId }); } }) .catch(() => { diff --git a/app/assets/javascripts/content_editor/extensions/image.js b/app/assets/javascripts/content_editor/extensions/image.js index 8bcfb490749..4dd8a1376ad 100644 --- a/app/assets/javascripts/content_editor/extensions/image.js +++ b/app/assets/javascripts/content_editor/extensions/image.js @@ -1,10 +1,65 @@ import { Image } from '@tiptap/extension-image'; -import { defaultMarkdownSerializer } from 'prosemirror-markdown/src/to_markdown'; +import { VueNodeViewRenderer } from '@tiptap/vue-2'; +import { Plugin, PluginKey } from 'prosemirror-state'; +import { __ } from '~/locale'; +import ImageWrapper from '../components/wrappers/image.vue'; +import { uploadFile } from '../services/upload_file'; +import { getImageAlt, readFileAsDataURL } from '../services/utils'; + +export const acceptedMimes = ['image/jpeg', 'image/png', 'image/gif', 'image/jpg']; + +const resolveImageEl = (element) => + element.nodeName === 'IMG' ? element : element.querySelector('img'); + +const startFileUpload = async ({ editor, file, uploadsPath, renderMarkdown }) => { + const encodedSrc = await readFileAsDataURL(file); + const { view } = editor; + + editor.commands.setImage({ uploading: true, src: encodedSrc }); + + const { state } = view; + const position = state.selection.from - 1; + const { tr } = state; + + try { + const { src, canonicalSrc } = await uploadFile({ file, uploadsPath, renderMarkdown }); + + view.dispatch( + tr.setNodeMarkup(position, undefined, { + uploading: false, + src: encodedSrc, + alt: getImageAlt(src), + canonicalSrc, + }), + ); + } catch (e) { + editor.commands.deleteRange({ from: position, to: position + 1 }); + editor.emit('error', __('An error occurred while uploading the image. Please try again.')); + } +}; + +const handleFileEvent = ({ editor, file, uploadsPath, renderMarkdown }) => { + if (acceptedMimes.includes(file?.type)) { + startFileUpload({ editor, file, uploadsPath, renderMarkdown }); + + return true; + } + + return false; +}; const ExtendedImage = Image.extend({ + defaultOptions: { + ...Image.options, + uploadsPath: null, + renderMarkdown: null, + }, addAttributes() { return { ...this.parent?.(), + uploading: { + default: false, + }, src: { default: null, /* @@ -14,17 +69,25 @@ const ExtendedImage = Image.extend({ * attribute. */ parseHTML: (element) => { - const img = element.querySelector('img'); + const img = resolveImageEl(element); return { src: img.dataset.src || img.getAttribute('src'), }; }, }, + canonicalSrc: { + default: null, + parseHTML: (element) => { + return { + canonicalSrc: element.dataset.canonicalSrc, + }; + }, + }, alt: { default: null, parseHTML: (element) => { - const img = element.querySelector('img'); + const img = resolveImageEl(element); return { alt: img.getAttribute('alt'), @@ -44,9 +107,58 @@ const ExtendedImage = Image.extend({ }, ]; }, + addCommands() { + return { + ...this.parent(), + uploadImage: ({ file }) => () => { + const { uploadsPath, renderMarkdown } = this.options; + + handleFileEvent({ file, uploadsPath, renderMarkdown, editor: this.editor }); + }, + }; + }, + addProseMirrorPlugins() { + const { editor } = this; + + return [ + new Plugin({ + key: new PluginKey('handleDropAndPasteImages'), + props: { + handlePaste: (_, event) => { + const { uploadsPath, renderMarkdown } = this.options; + + return handleFileEvent({ + editor, + file: event.clipboardData.files[0], + uploadsPath, + renderMarkdown, + }); + }, + handleDrop: (_, event) => { + const { uploadsPath, renderMarkdown } = this.options; + + return handleFileEvent({ + editor, + file: event.dataTransfer.files[0], + uploadsPath, + renderMarkdown, + }); + }, + }, + }), + ]; + }, + addNodeView() { + return VueNodeViewRenderer(ImageWrapper); + }, }); -const serializer = defaultMarkdownSerializer.nodes.image; +const serializer = (state, node) => { + const { alt, canonicalSrc, src, title } = node.attrs; + const quotedTitle = title ? ` ${state.quote(title)}` : ''; + + state.write(`}${quotedTitle})`); +}; export const configure = ({ renderMarkdown, uploadsPath }) => { return { diff --git a/app/assets/javascripts/content_editor/services/utils.js b/app/assets/javascripts/content_editor/services/utils.js index cf5234bbff8..ca2f9762ff8 100644 --- a/app/assets/javascripts/content_editor/services/utils.js +++ b/app/assets/javascripts/content_editor/services/utils.js @@ -3,3 +3,15 @@ export const hasSelection = (tiptapEditor) => { return from < to; }; + +export const getImageAlt = (src) => { + return src.replace(/^.*\/|\..*$/g, '').replace(/\W+/g, ' '); +}; + +export const readFileAsDataURL = (file) => { + return new Promise((resolve) => { + const reader = new FileReader(); + reader.addEventListener('load', (e) => resolve(e.target.result), { once: true }); + reader.readAsDataURL(file); + }); +}; diff --git a/app/assets/javascripts/design_management/components/design_notes/design_note.vue b/app/assets/javascripts/design_management/components/design_notes/design_note.vue index 833d7081a2c..1e1f5135290 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_note.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_note.vue @@ -1,6 +1,7 @@ <script> import { GlTooltipDirective, GlIcon, GlLink, GlSafeHtmlDirective } from '@gitlab/ui'; import { ApolloMutation } from 'vue-apollo'; +import { getIdFromGraphQLId } from '~/graphql_shared/utils'; import { __ } from '~/locale'; import TimelineEntryItem from '~/vue_shared/components/notes/timeline_entry_item.vue'; import TimeAgoTooltip from '~/vue_shared/components/time_ago_tooltip.vue'; @@ -48,6 +49,9 @@ export default { author() { return this.note.author; }, + authorId() { + return getIdFromGraphQLId(this.author.id); + }, noteAnchorId() { return findNoteId(this.note.id); }, @@ -94,7 +98,7 @@ export default { v-once :href="author.webUrl" class="js-user-link" - :data-user-id="author.id" + :data-user-id="authorId" :data-username="author.username" > <span class="note-header-author-name gl-font-weight-bold">{{ author.name }}</span> diff --git a/app/assets/javascripts/lib/graphql.js b/app/assets/javascripts/lib/graphql.js index d91fc61ba21..0804213cafa 100644 --- a/app/assets/javascripts/lib/graphql.js +++ b/app/assets/javascripts/lib/graphql.js @@ -97,7 +97,7 @@ export default (resolvers = {}, config = {}) => { */ const fetchIntervention = (url, options) => { - return fetch(stripWhitespaceFromQuery(url, path), options); + return fetch(stripWhitespaceFromQuery(url, uri), options); }; const requestLink = ApolloLink.split( diff --git a/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue b/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue index 63e3f62c0c6..33d86dec767 100644 --- a/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue +++ b/app/assets/javascripts/members/components/filter_sort/members_filtered_search_bar.vue @@ -1,8 +1,7 @@ <script> import { GlFilteredSearchToken } from '@gitlab/ui'; import { mapState } from 'vuex'; -// eslint-disable-next-line import/no-deprecated -import { getParameterByName, setUrlParams, urlParamsToObject } from '~/lib/utils/url_utility'; +import { getParameterByName, setUrlParams, queryToObject } from '~/lib/utils/url_utility'; import { s__ } from '~/locale'; import { SEARCH_TOKEN_TYPE, @@ -68,8 +67,7 @@ export default { }, }, created() { - // eslint-disable-next-line import/no-deprecated - const query = urlParamsToObject(window.location.search); + const query = queryToObject(window.location.search); const tokens = this.tokens .filter((token) => query[token.type]) diff --git a/app/assets/javascripts/notes/components/notes_app.vue b/app/assets/javascripts/notes/components/notes_app.vue index 47d0c26b106..29c60b96d8a 100644 --- a/app/assets/javascripts/notes/components/notes_app.vue +++ b/app/assets/javascripts/notes/components/notes_app.vue @@ -66,6 +66,7 @@ export default { data() { return { currentFilter: null, + renderSkeleton: !this.shouldShow, }; }, computed: { @@ -93,7 +94,7 @@ export default { return this.noteableData.noteableType; }, allDiscussions() { - if (this.isLoading) { + if (this.renderSkeleton || this.isLoading) { const prerenderedNotesCount = parseInt(this.notesData.prerenderedNotesCount, 10) || 0; return new Array(prerenderedNotesCount).fill({ @@ -122,6 +123,10 @@ export default { if (!this.isNotesFetched) { this.fetchNotes(); } + + setTimeout(() => { + this.renderSkeleton = !this.shouldShow; + }); }, discussionTabCounterText(val) { if (this.discussionsCount) { diff --git a/app/assets/javascripts/projects/project_new.js b/app/assets/javascripts/projects/project_new.js index 04ea6f760f6..ee02f446795 100644 --- a/app/assets/javascripts/projects/project_new.js +++ b/app/assets/javascripts/projects/project_new.js @@ -74,6 +74,7 @@ const deriveProjectPathFromUrl = ($projectImportUrl) => { const bindEvents = () => { const $newProjectForm = $('#new_project'); const $projectImportUrl = $('#project_import_url'); + const $projectImportUrlWarning = $('.js-import-url-warning'); const $projectPath = $('.tab-pane.active #project_path'); const $useTemplateBtn = $('.template-button > input'); const $projectFieldsForm = $('.project-fields-form'); @@ -134,7 +135,25 @@ const bindEvents = () => { $projectPath.val($projectPath.val().trim()); }); - $projectImportUrl.keyup(() => deriveProjectPathFromUrl($projectImportUrl)); + function updateUrlPathWarningVisibility() { + const url = $projectImportUrl.val(); + const URL_PATTERN = /(?:git|https?):\/\/.*\/.*\.git$/; + const isUrlValid = URL_PATTERN.test(url); + $projectImportUrlWarning.toggleClass('hide', isUrlValid); + } + + let isProjectImportUrlDirty = false; + $projectImportUrl.on('blur', () => { + isProjectImportUrlDirty = true; + updateUrlPathWarningVisibility(); + }); + $projectImportUrl.on('keyup', () => { + deriveProjectPathFromUrl($projectImportUrl); + // defer error message till first input blur + if (isProjectImportUrlDirty) { + updateUrlPathWarningVisibility(); + } + }); $('.js-import-git-toggle-button').on('click', () => { const $projectMirror = $('#project_mirror'); diff --git a/app/assets/javascripts/repository/components/blob_viewers/download_viewer.vue b/app/assets/javascripts/repository/components/blob_viewers/download_viewer.vue new file mode 100644 index 00000000000..48fa33eb558 --- /dev/null +++ b/app/assets/javascripts/repository/components/blob_viewers/download_viewer.vue @@ -0,0 +1,51 @@ +<script> +import { GlIcon, GlLink } from '@gitlab/ui'; +import { numberToHumanSize } from '~/lib/utils/number_utils'; +import { sprintf, __ } from '~/locale'; + +export default { + components: { + GlIcon, + GlLink, + }, + props: { + fileName: { + type: String, + required: true, + }, + filePath: { + type: String, + required: true, + }, + fileSize: { + type: Number, + required: false, + default: 0, + }, + }, + computed: { + downloadFileSize() { + return numberToHumanSize(this.fileSize); + }, + downloadText() { + if (this.fileSize > 0) { + return sprintf(__('Download (%{fileSizeReadable})'), { + fileSizeReadable: this.downloadFileSize, + }); + } + return __('Download'); + }, + }, +}; +</script> + +<template> + <div class="gl-text-center gl-py-13 gl-bg-gray-50"> + <gl-link :href="filePath" rel="nofollow" :download="fileName" target="_blank"> + <div> + <gl-icon :size="16" name="download" class="gl-text-gray-900" /> + </div> + <h4>{{ downloadText }}</h4> + </gl-link> + </div> +</template> diff --git a/app/assets/javascripts/repository/components/blob_viewers/index.js b/app/assets/javascripts/repository/components/blob_viewers/index.js index abe274d1568..4e16b16041f 100644 --- a/app/assets/javascripts/repository/components/blob_viewers/index.js +++ b/app/assets/javascripts/repository/components/blob_viewers/index.js @@ -5,8 +5,7 @@ export const loadViewer = (type) => { case 'text': return () => import(/* webpackChunkName: 'blob_text_viewer' */ './text_viewer.vue'); case 'download': - // TODO (follow-up): import the download viewer - return null; // () => import(/* webpackChunkName: 'blob_download_viewer' */ './download_viewer.vue'); + return () => import(/* webpackChunkName: 'blob_download_viewer' */ './download_viewer.vue'); default: return null; } @@ -19,5 +18,10 @@ export const viewerProps = (type, blob) => { fileName: blob.name, readOnly: true, }, + download: { + fileName: blob.name, + filePath: blob.rawPath, + fileSize: blob.rawSize, + }, }[type]; }; diff --git a/app/assets/stylesheets/utilities.scss b/app/assets/stylesheets/utilities.scss index 82d768c2351..5e2ec774655 100644 --- a/app/assets/stylesheets/utilities.scss +++ b/app/assets/stylesheets/utilities.scss @@ -87,6 +87,12 @@ padding-bottom: $gl-spacing-scale-8; } +// Will be moved to @gitlab/ui in https://gitlab.com/gitlab-org/gitlab-ui/-/issues/1495 +.gl-py-13 { + padding-top: $gl-spacing-scale-13; + padding-bottom: $gl-spacing-scale-13; +} + .gl-transition-property-stroke-opacity { transition-property: stroke-opacity; } diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 83ede2867cc..4328f3f7a4b 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -325,7 +325,11 @@ module Ci build.run_after_commit do build.run_status_commit_hooks! - BuildFinishedWorker.perform_async(id) + if Feature.enabled?(:ci_build_finished_worker_namespace_changed, build.project, default_enabled: :yaml) + Ci::BuildFinishedWorker.perform_async(id) + else + ::BuildFinishedWorker.perform_async(id) + end end end diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index a1eaae8a21a..0663052f51d 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -11,11 +11,48 @@ module Ci scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) } def self.upsert_from_build!(build) - entry = self.new(build: build, project: build.project, protected: build.protected?) + entry = self.new(args_from_build(build)) entry.validate! self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) end + + def self.args_from_build(build) + args = { + build: build, + project: build.project, + protected: build.protected? + } + + if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml) + args.merge(instance_runners_enabled: shareable?(build)) + else + args + end + end + private_class_method :args_from_build + + def self.shareable?(build) + shared_runner_enabled?(build) && + builds_access_level?(build) && + project_not_removed?(build) + end + private_class_method :shareable? + + def self.shared_runner_enabled?(build) + build.project.shared_runners.exists? + end + private_class_method :shared_runner_enabled? + + def self.project_not_removed?(build) + !build.project.pending_delete? + end + private_class_method :project_not_removed? + + def self.builds_access_level?(build) + build.project.project_feature.builds_access_level.nil? || build.project.project_feature.builds_access_level > 0 + end + private_class_method :builds_access_level? end end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 6b8c5d21345..5d079f57267 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -224,7 +224,7 @@ module Ci end after_transition [:created, :waiting_for_resource, :preparing, :pending, :running] => :success do |pipeline| - # We wait a little bit to ensure that all BuildFinishedWorkers finish first + # We wait a little bit to ensure that all Ci::BuildFinishedWorkers finish first # because this is where some metrics like code coverage is parsed and stored # in CI build records which the daily build metrics worker relies on. pipeline.run_after_commit { Ci::DailyBuildGroupReportResultsWorker.perform_in(10.minutes, pipeline.id) } diff --git a/app/models/concerns/partitioned_table.rb b/app/models/concerns/partitioned_table.rb index 9f1cec5d520..eab5d4c35bb 100644 --- a/app/models/concerns/partitioned_table.rb +++ b/app/models/concerns/partitioned_table.rb @@ -10,12 +10,12 @@ module PartitionedTable monthly: Gitlab::Database::Partitioning::MonthlyStrategy }.freeze - def partitioned_by(partitioning_key, strategy:) + def partitioned_by(partitioning_key, strategy:, **kwargs) strategy_class = PARTITIONING_STRATEGIES[strategy.to_sym] || raise(ArgumentError, "Unknown partitioning strategy: #{strategy}") - @partitioning_strategy = strategy_class.new(self, partitioning_key) + @partitioning_strategy = strategy_class.new(self, partitioning_key, **kwargs) - Gitlab::Database::Partitioning::PartitionCreator.register(self) + Gitlab::Database::Partitioning::PartitionManager.register(self) end end end diff --git a/app/models/hooks/web_hook_log.rb b/app/models/hooks/web_hook_log.rb index 0c96d5d4b6d..8c0565e4a38 100644 --- a/app/models/hooks/web_hook_log.rb +++ b/app/models/hooks/web_hook_log.rb @@ -9,7 +9,7 @@ class WebHookLog < ApplicationRecord self.primary_key = :id - partitioned_by :created_at, strategy: :monthly + partitioned_by :created_at, strategy: :monthly, retain_for: 3.months belongs_to :web_hook diff --git a/app/services/groups/transfer_service.rb b/app/services/groups/transfer_service.rb index 518d061c654..966d04ceb70 100644 --- a/app/services/groups/transfer_service.rb +++ b/app/services/groups/transfer_service.rb @@ -46,6 +46,7 @@ module Groups def ensure_allowed_transfer raise_transfer_error(:group_is_already_root) if group_is_already_root? raise_transfer_error(:same_parent_as_current) if same_parent? + raise_transfer_error(:has_subscription) if has_subscription? raise_transfer_error(:invalid_policies) unless valid_policies? raise_transfer_error(:namespace_with_same_path) if namespace_with_same_path? raise_transfer_error(:group_contains_images) if group_projects_contain_registry_images? @@ -73,6 +74,10 @@ module Groups @new_parent_group && @new_parent_group.id == @group.parent_id end + def has_subscription? + @group.paid? + end + def transfer_to_subgroup? @new_parent_group && \ @group.self_and_descendants.pluck_primary_key.include?(@new_parent_group.id) diff --git a/app/services/service_ping/permit_data_categories_service.rb b/app/services/service_ping/permit_data_categories_service.rb new file mode 100644 index 00000000000..ff48c022b56 --- /dev/null +++ b/app/services/service_ping/permit_data_categories_service.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module ServicePing + class PermitDataCategoriesService + STANDARD_CATEGORY = 'Standard' + SUBSCRIPTION_CATEGORY = 'Subscription' + OPERATIONAL_CATEGORY = 'Operational' + OPTIONAL_CATEGORY = 'Optional' + CATEGORIES = [ + STANDARD_CATEGORY, + SUBSCRIPTION_CATEGORY, + OPERATIONAL_CATEGORY, + OPTIONAL_CATEGORY + ].to_set.freeze + + def execute + return [] unless product_intelligence_enabled? + + CATEGORIES + end + + def product_intelligence_enabled? + pings_enabled? && !User.single_user&.requires_usage_stats_consent? + end + + private + + def pings_enabled? + ::Gitlab::CurrentSettings.usage_ping_enabled? + end + end +end + +ServicePing::PermitDataCategoriesService.prepend_mod_with('ServicePing::PermitDataCategoriesService') diff --git a/app/services/service_ping/submit_service.rb b/app/services/service_ping/submit_service.rb index f12f9c43109..06e4fcbaf32 100644 --- a/app/services/service_ping/submit_service.rb +++ b/app/services/service_ping/submit_service.rb @@ -18,8 +18,7 @@ module ServicePing SubmissionError = Class.new(StandardError) def execute - return unless Gitlab::CurrentSettings.usage_ping_enabled? - return if User.single_user&.requires_usage_stats_consent? + return unless ServicePing::PermitDataCategoriesService.new.product_intelligence_enabled? usage_data = Gitlab::UsageData.data(force_refresh: true) diff --git a/app/views/groups/settings/_advanced.html.haml b/app/views/groups/settings/_advanced.html.haml index d7a145924de..fea0736ffc8 100644 --- a/app/views/groups/settings/_advanced.html.haml +++ b/app/views/groups/settings/_advanced.html.haml @@ -24,21 +24,6 @@ "data-bind-in" => "#{'create_chat_team' if Gitlab.config.mattermost.enabled}" = f.submit s_('GroupSettings|Change group URL'), class: 'btn gl-button btn-warning' -.sub-section - %h4.warning-title= s_('GroupSettings|Transfer group') - = form_for @group, url: transfer_group_path(@group), method: :put, html: { class: 'js-group-transfer-form' } do |f| - .form-group - = dropdown_tag('Select parent group', options: { toggle_class: 'js-groups-dropdown', title: 'Parent Group', filter: true, dropdown_class: 'dropdown-open-top dropdown-group-transfer', placeholder: 'Search groups', data: { data: parent_group_options(@group), qa_selector: 'select_group_dropdown' } }) - = hidden_field_tag 'new_parent_group_id' - - %ul - - side_effects_link_start = '<a href="https://docs.gitlab.com/ee/user/project/index.html#redirects-when-changing-repository-paths" target="_blank">'.html_safe - - warning_text = s_("GroupSettings|Be careful. Changing a group's parent can have unintended %{side_effects_link_start}side effects%{side_effects_link_end}.") % { side_effects_link_start: side_effects_link_start, side_effects_link_end: '</a>'.html_safe } - %li= warning_text.html_safe - %li= s_('GroupSettings|You can only transfer the group to a group you manage.') - %li= s_('GroupSettings|You will need to update your local repositories to point to the new location.') - %li= s_("GroupSettings|If the parent group's visibility is lower than the group current visibility, visibility levels for subgroups and projects will be changed to match the new parent group's visibility.") - = f.submit s_('GroupSettings|Transfer group'), class: 'btn gl-button btn-warning', data: { qa_selector: "transfer_group_button" } - += render 'groups/settings/transfer', group: @group = render 'groups/settings/remove', group: @group = render_if_exists 'groups/settings/restore', group: @group diff --git a/app/views/groups/settings/_transfer.html.haml b/app/views/groups/settings/_transfer.html.haml new file mode 100644 index 00000000000..1472ae42152 --- /dev/null +++ b/app/views/groups/settings/_transfer.html.haml @@ -0,0 +1,22 @@ +.sub-section + %h4.warning-title= s_('GroupSettings|Transfer group') + = form_for group, url: transfer_group_path(group), method: :put, html: { class: 'js-group-transfer-form' } do |f| + .form-group + = dropdown_tag('Select parent group', options: { toggle_class: 'js-groups-dropdown', title: 'Parent Group', filter: true, dropdown_class: 'dropdown-open-top dropdown-group-transfer', placeholder: 'Search groups', disabled: group.paid?, data: { data: parent_group_options(group), qa_selector: 'select_group_dropdown' } }) + = hidden_field_tag 'new_parent_group_id' + + %ul + - side_effects_link_start = '<a href="https://docs.gitlab.com/ee/user/project/index.html#redirects-when-changing-repository-paths" target="_blank">'.html_safe + - warning_text = s_("GroupSettings|Be careful. Changing a group's parent can have unintended %{side_effects_link_start}side effects%{side_effects_link_end}.") % { side_effects_link_start: side_effects_link_start, side_effects_link_end: '</a>'.html_safe } + %li= warning_text.html_safe + %li= s_('GroupSettings|You can only transfer the group to a group you manage.') + %li= s_('GroupSettings|You will need to update your local repositories to point to the new location.') + %li= s_("GroupSettings|If the parent group's visibility is lower than the group current visibility, visibility levels for subgroups and projects will be changed to match the new parent group's visibility.") + + - if group.paid? + .gl-alert.gl-alert-info.gl-mb-5{ data: { testid: 'group-to-transfer-has-linked-subscription-alert' } } + = sprite_icon('information-o', size: 16, css_class: 'gl-icon gl-alert-icon gl-alert-icon-no-title') + .gl-alert-body + = html_escape(_("This group can't be transfered because it is linked to a subscription. To transfer this group, %{linkStart}link the subscription%{linkEnd} with a different group.")) % { linkStart: "<a href=\"#{help_page_path('subscriptions/index', anchor: 'change-the-linked-namespace')}\">".html_safe, linkEnd: '</a>'.html_safe } + + = f.submit s_('GroupSettings|Transfer group'), class: 'btn gl-button btn-warning', data: { qa_selector: "transfer_group_button" } diff --git a/app/views/layouts/devise.html.haml b/app/views/layouts/devise.html.haml index 3acc41f04b7..5c9c6a06ac1 100644 --- a/app/views/layouts/devise.html.haml +++ b/app/views/layouts/devise.html.haml @@ -14,7 +14,7 @@ .row.mt-3 .col-sm-12 %h1.mb-3.font-weight-normal - = current_appearance&.title.presence || "GitLab" + = current_appearance&.title.presence || _('GitLab') .row.mb-3 .col-sm-7.order-12.order-sm-1.brand-holder - unless recently_confirmed_com? diff --git a/app/views/search/results/_blob_data.html.haml b/app/views/search/results/_blob_data.html.haml index 16d640273b0..fb2825ad15e 100644 --- a/app/views/search/results/_blob_data.html.haml +++ b/app/views/search/results/_blob_data.html.haml @@ -5,6 +5,7 @@ = sprite_icon('document') %strong = search_blob_title(project, path) + = copy_file_path_button(path) - if blob.data .file-content.code.term{ data: { qa_selector: 'file_text_content' } } = render 'shared/file_highlight', blob: blob, first_line_number: blob.startline, blob_link: blob_link, highlight_line: blob.highlight_line diff --git a/app/views/shared/_import_form.html.haml b/app/views/shared/_import_form.html.haml index 65e02341936..f03314563cb 100644 --- a/app/views/shared/_import_form.html.haml +++ b/app/views/shared/_import_form.html.haml @@ -8,7 +8,18 @@ = _('Git repository URL') = f.text_field :import_url, value: import_url.sanitized_url, autocomplete: 'off', class: 'form-control gl-form-input', placeholder: 'https://gitlab.company.com/group/project.git', required: true + = render 'shared/global_alert', + variant: :warning, + alert_class: 'gl-mt-3 js-import-url-warning hide', + dismissible: false, + close_button_class: 'js-close-2fa-enabled-success-alert' do + .gl-alert-body + = s_('Import|A repository URL usually ends in a .git suffix, although this is not required. Double check to make sure your repository URL is correct.') + .gl-alert.gl-alert-not-dismissible.gl-alert-warning.gl-mt-3.hide#project_import_url_warning + .gl-alert-container + = sprite_icon('warning-solid', css_class: 'gl-icon s16 gl-alert-icon gl-alert-icon-no-title') + .gl-alert-content{ role: 'alert' } .row .form-group.col-md-6 = f.label :import_url_user, class: 'label-bold' do diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 04c557965b1..de1b17b588f 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -247,6 +247,15 @@ :idempotent: true :tags: - :exclude_from_kubernetes +- :name: cronjob:database_partition_management + :worker_name: Database::PartitionManagementWorker + :feature_category: :database + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:environments_auto_stop_cron :worker_name: Environments::AutoStopCronWorker :feature_category: :continuous_delivery @@ -1365,6 +1374,15 @@ :weight: 1 :idempotent: :tags: [] +- :name: pipeline_background:ci_archive_trace + :worker_name: Ci::ArchiveTraceWorker + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: + :tags: [] - :name: pipeline_background:ci_build_trace_chunk_flush :worker_name: Ci::BuildTraceChunkFlushWorker :feature_category: :continuous_integration @@ -1585,6 +1603,15 @@ :weight: 5 :idempotent: :tags: [] +- :name: pipeline_processing:ci_build_finished + :worker_name: Ci::BuildFinishedWorker + :feature_category: :continuous_integration + :has_external_dependencies: + :urgency: :high + :resource_boundary: :cpu + :weight: 5 + :idempotent: + :tags: [] - :name: pipeline_processing:ci_build_prepare :worker_name: Ci::BuildPrepareWorker :feature_category: :continuous_integration diff --git a/app/workers/archive_trace_worker.rb b/app/workers/archive_trace_worker.rb index 629526ec17c..ecde05f94dc 100644 --- a/app/workers/archive_trace_worker.rb +++ b/app/workers/archive_trace_worker.rb @@ -1,16 +1,5 @@ # frozen_string_literal: true -class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker - - sidekiq_options retry: 3 - include PipelineBackgroundQueue - - # rubocop: disable CodeReuse/ActiveRecord - def perform(job_id) - Ci::Build.without_archived_trace.find_by(id: job_id).try do |job| - Ci::ArchiveTraceService.new.execute(job, worker_name: self.class.name) - end - end - # rubocop: enable CodeReuse/ActiveRecord +class ArchiveTraceWorker < ::Ci::ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker + # DEPRECATED: Not triggered since https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64934/ end diff --git a/app/workers/build_finished_worker.rb b/app/workers/build_finished_worker.rb index a3eaacec8a2..0d41f7b9438 100644 --- a/app/workers/build_finished_worker.rb +++ b/app/workers/build_finished_worker.rb @@ -1,61 +1,9 @@ # frozen_string_literal: true -class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker - include ApplicationWorker +class BuildFinishedWorker < ::Ci::BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker + # DEPRECATED: Not triggered since https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64934/ - sidekiq_options retry: 3 - include PipelineQueue - - queue_namespace :pipeline_processing + # We need to explicitly specify these settings. They aren't inheriting from the parent class. urgency :high worker_resource_boundary :cpu - - ARCHIVE_TRACES_IN = 2.minutes.freeze - - # rubocop: disable CodeReuse/ActiveRecord - def perform(build_id) - Ci::Build.find_by(id: build_id).try do |build| - process_build(build) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - private - - # Processes a single CI build that has finished. - # - # This logic resides in a separate method so that EE can extend it more - # easily. - # - # @param [Ci::Build] build The build to process. - def process_build(build) - # We execute these in sync to reduce IO. - build.parse_trace_sections! - build.update_coverage - Ci::BuildReportResultService.new.execute(build) - - # We execute these async as these are independent operations. - BuildHooksWorker.perform_async(build.id) - ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat? - - if build.failed? - ::Ci::MergeRequests::AddTodoWhenBuildFailsWorker.perform_async(build.id) - end - - ## - # We want to delay sending a build trace to object storage operation to - # validate that this fixes a race condition between this and flushing live - # trace chunks and chunks being removed after consolidation and putting - # them into object storage archive. - # - # TODO This is temporary fix we should improve later, after we validate - # that this is indeed the culprit. - # - # See https://gitlab.com/gitlab-org/gitlab/-/issues/267112 for more - # details. - # - ArchiveTraceWorker.perform_in(ARCHIVE_TRACES_IN, build.id) - end end - -BuildFinishedWorker.prepend_mod_with('BuildFinishedWorker') diff --git a/app/workers/ci/archive_trace_worker.rb b/app/workers/ci/archive_trace_worker.rb new file mode 100644 index 00000000000..16288faf370 --- /dev/null +++ b/app/workers/ci/archive_trace_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Ci + class ArchiveTraceWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + sidekiq_options retry: 3 + include PipelineBackgroundQueue + + # rubocop: disable CodeReuse/ActiveRecord + def perform(job_id) + Ci::Build.without_archived_trace.find_by(id: job_id).try do |job| + Ci::ArchiveTraceService.new.execute(job, worker_name: self.class.name) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/app/workers/ci/archive_traces_cron_worker.rb b/app/workers/ci/archive_traces_cron_worker.rb index c748bc33ada..5fe3adf870f 100644 --- a/app/workers/ci/archive_traces_cron_worker.rb +++ b/app/workers/ci/archive_traces_cron_worker.rb @@ -12,7 +12,7 @@ module Ci # rubocop: disable CodeReuse/ActiveRecord def perform # Archive stale live traces which still resides in redis or database - # This could happen when ArchiveTraceWorker sidekiq jobs were lost by receiving SIGKILL + # This could happen when Ci::ArchiveTraceWorker sidekiq jobs were lost by receiving SIGKILL # More details in https://gitlab.com/gitlab-org/gitlab-foss/issues/36791 Ci::Build.with_stale_live_trace.find_each(batch_size: 100) do |build| Ci::ArchiveTraceService.new.execute(build, worker_name: self.class.name) diff --git a/app/workers/ci/build_finished_worker.rb b/app/workers/ci/build_finished_worker.rb new file mode 100644 index 00000000000..1d6e3b1fa3c --- /dev/null +++ b/app/workers/ci/build_finished_worker.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Ci + class BuildFinishedWorker # rubocop:disable Scalability/IdempotentWorker + include ApplicationWorker + + sidekiq_options retry: 3 + include PipelineQueue + + queue_namespace :pipeline_processing + urgency :high + worker_resource_boundary :cpu + + ARCHIVE_TRACES_IN = 2.minutes.freeze + + # rubocop: disable CodeReuse/ActiveRecord + def perform(build_id) + Ci::Build.find_by(id: build_id).try do |build| + process_build(build) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + private + + # Processes a single CI build that has finished. + # + # This logic resides in a separate method so that EE can extend it more + # easily. + # + # @param [Ci::Build] build The build to process. + def process_build(build) + # We execute these in sync to reduce IO. + build.parse_trace_sections! + build.update_coverage + Ci::BuildReportResultService.new.execute(build) + + # We execute these async as these are independent operations. + BuildHooksWorker.perform_async(build.id) + ChatNotificationWorker.perform_async(build.id) if build.pipeline.chat? + + if build.failed? + ::Ci::MergeRequests::AddTodoWhenBuildFailsWorker.perform_async(build.id) + end + + ## + # We want to delay sending a build trace to object storage operation to + # validate that this fixes a race condition between this and flushing live + # trace chunks and chunks being removed after consolidation and putting + # them into object storage archive. + # + # TODO This is temporary fix we should improve later, after we validate + # that this is indeed the culprit. + # + # See https://gitlab.com/gitlab-org/gitlab/-/issues/267112 for more + # details. + # + archive_trace_worker_class(build).perform_in(ARCHIVE_TRACES_IN, build.id) + end + + def archive_trace_worker_class(build) + if Feature.enabled?(:ci_build_finished_worker_namespace_changed, build.project, default_enabled: :yaml) + Ci::ArchiveTraceWorker + else + ::ArchiveTraceWorker + end + end + end +end + +Ci::BuildFinishedWorker.prepend_mod_with('Ci::BuildFinishedWorker') diff --git a/app/workers/concerns/gitlab/github_import/object_importer.rb b/app/workers/concerns/gitlab/github_import/object_importer.rb index ab333d020fb..1eff53cea01 100644 --- a/app/workers/concerns/gitlab/github_import/object_importer.rb +++ b/app/workers/concerns/gitlab/github_import/object_importer.rb @@ -36,25 +36,13 @@ module Gitlab importer_class.new(object, project, client).execute - increment_counters(project) + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :imported) info(project.id, message: 'importer finished') rescue StandardError => e error(project.id, e, hash) end - # Counters incremented: - # - global (prometheus): for metrics in Grafana - # - project (redis): used in FinishImportWorker to report number of objects imported - def increment_counters(project) - counter.increment - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :imported) - end - - def counter - @counter ||= Gitlab::Metrics.counter(counter_name, counter_description) - end - def object_type raise NotImplementedError end @@ -70,16 +58,6 @@ module Gitlab raise NotImplementedError end - # Returns the name (as a Symbol) of the Prometheus counter. - def counter_name - raise NotImplementedError - end - - # Returns the description (as a String) of the Prometheus counter. - def counter_description - raise NotImplementedError - end - private attr_accessor :github_id diff --git a/app/workers/database/partition_management_worker.rb b/app/workers/database/partition_management_worker.rb new file mode 100644 index 00000000000..c9b1cd6d261 --- /dev/null +++ b/app/workers/database/partition_management_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Database + class PartitionManagementWorker + include ApplicationWorker + + sidekiq_options retry: 3 + include CronjobQueue # rubocop:disable Scalability/CronWorkerContext + + feature_category :database + idempotent! + + def perform + Gitlab::Database::Partitioning::PartitionManager.new.sync_partitions + ensure + Gitlab::Database::Partitioning::PartitionMonitoring.new.report_metrics + end + end +end diff --git a/app/workers/gitlab/github_import/import_diff_note_worker.rb b/app/workers/gitlab/github_import/import_diff_note_worker.rb index 5ee5fcaacd6..85b7d6c76bd 100644 --- a/app/workers/gitlab/github_import/import_diff_note_worker.rb +++ b/app/workers/gitlab/github_import/import_diff_note_worker.rb @@ -16,14 +16,6 @@ module Gitlab def object_type :diff_note end - - def counter_name - :github_importer_imported_diff_notes - end - - def counter_description - 'The number of imported GitHub pull request review comments' - end end end end diff --git a/app/workers/gitlab/github_import/import_issue_worker.rb b/app/workers/gitlab/github_import/import_issue_worker.rb index a3921e86c84..8fdc0219ffd 100644 --- a/app/workers/gitlab/github_import/import_issue_worker.rb +++ b/app/workers/gitlab/github_import/import_issue_worker.rb @@ -16,14 +16,6 @@ module Gitlab def object_type :issue end - - def counter_name - :github_importer_imported_issues - end - - def counter_description - 'The number of imported GitHub issues' - end end end end diff --git a/app/workers/gitlab/github_import/import_lfs_object_worker.rb b/app/workers/gitlab/github_import/import_lfs_object_worker.rb index ea755fc9a37..2a95366bac7 100644 --- a/app/workers/gitlab/github_import/import_lfs_object_worker.rb +++ b/app/workers/gitlab/github_import/import_lfs_object_worker.rb @@ -16,14 +16,6 @@ module Gitlab def object_type :lfs_object end - - def counter_name - :github_importer_imported_lfs_objects - end - - def counter_description - 'The number of imported GitHub Lfs Objects' - end end end end diff --git a/app/workers/gitlab/github_import/import_note_worker.rb b/app/workers/gitlab/github_import/import_note_worker.rb index d612e0b2e5b..2125c953778 100644 --- a/app/workers/gitlab/github_import/import_note_worker.rb +++ b/app/workers/gitlab/github_import/import_note_worker.rb @@ -16,14 +16,6 @@ module Gitlab def object_type :note end - - def counter_name - :github_importer_imported_notes - end - - def counter_description - 'The number of imported GitHub comments' - end end end end diff --git a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb index 2db404cca5d..91dab3470d9 100644 --- a/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_merged_by_worker.rb @@ -18,14 +18,6 @@ module Gitlab def object_type :pull_request_merged_by end - - def counter_name - :github_importer_imported_pull_requests_merged_by - end - - def counter_description - 'The number of imported GitHub pull requests merged by' - end end end end diff --git a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb index 7ea867ddb39..de10fe40589 100644 --- a/app/workers/gitlab/github_import/import_pull_request_review_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_review_worker.rb @@ -18,14 +18,6 @@ module Gitlab def object_type :pull_request_review end - - def counter_name - :github_importer_imported_pull_request_reviews - end - - def counter_description - 'The number of imported GitHub pull request reviews' - end end end end diff --git a/app/workers/gitlab/github_import/import_pull_request_worker.rb b/app/workers/gitlab/github_import/import_pull_request_worker.rb index f1d01adb736..79938a157d7 100644 --- a/app/workers/gitlab/github_import/import_pull_request_worker.rb +++ b/app/workers/gitlab/github_import/import_pull_request_worker.rb @@ -16,14 +16,6 @@ module Gitlab def object_type :pull_request end - - def counter_name - :github_importer_imported_pull_requests - end - - def counter_description - 'The number of imported GitHub pull requests' - end end end end diff --git a/app/workers/partition_creation_worker.rb b/app/workers/partition_creation_worker.rb index 2b21741d6c2..bb4834ab2dd 100644 --- a/app/workers/partition_creation_worker.rb +++ b/app/workers/partition_creation_worker.rb @@ -10,8 +10,7 @@ class PartitionCreationWorker idempotent! def perform - Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions - ensure - Gitlab::Database::Partitioning::PartitionMonitoring.new.report_metrics + # This worker has been removed in favor of Database::PartitionManagementWorker + Database::PartitionManagementWorker.new.perform end end diff --git a/config/feature_flags/development/helm_packages.yml b/config/feature_flags/development/ci_build_finished_worker_namespace_changed.yml index 2e188ecd46f..3b25bc1ed39 100644 --- a/config/feature_flags/development/helm_packages.yml +++ b/config/feature_flags/development/ci_build_finished_worker_namespace_changed.yml @@ -1,8 +1,8 @@ --- -name: helm_packages -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61014 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/331693 -milestone: '14.0' +name: ci_build_finished_worker_namespace_changed +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/64934 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/335499 +milestone: '14.1' type: development -group: group::package +group: group::pipeline execution default_enabled: false diff --git a/config/feature_flags/development/ci_pending_builds_maintain_shared_runners_data.yml b/config/feature_flags/development/ci_pending_builds_maintain_shared_runners_data.yml new file mode 100644 index 00000000000..5a8b89edfad --- /dev/null +++ b/config/feature_flags/development/ci_pending_builds_maintain_shared_runners_data.yml @@ -0,0 +1,8 @@ +--- +name: ci_pending_builds_maintain_shared_runners_data +introduced_by_url: +rollout_issue_url: +milestone: '14.1' +type: development +group: group::pipeline execution +default_enabled: false diff --git a/config/feature_flags/development/partition_pruning_dry_run.yml b/config/feature_flags/development/partition_pruning_dry_run.yml new file mode 100644 index 00000000000..427afa5fc94 --- /dev/null +++ b/config/feature_flags/development/partition_pruning_dry_run.yml @@ -0,0 +1,8 @@ +--- +name: partition_pruning_dry_run +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65093 +rollout_issue_url: +milestone: '14.1' +type: development +group: group::database +default_enabled: false diff --git a/config/feature_flags/development/runner_graphql_query.yml b/config/feature_flags/development/runner_graphql_query.yml index 6d6e7425a51..b7af0a2bb22 100644 --- a/config/feature_flags/development/runner_graphql_query.yml +++ b/config/feature_flags/development/runner_graphql_query.yml @@ -4,5 +4,5 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/59763 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/328700 type: development group: group::runner -default_enabled: false +default_enabled: true milestone: '13.12' diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 5c4088a7f87..42c7063378b 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -540,9 +540,9 @@ Settings.cron_jobs['authorized_project_update_periodic_recalculate_worker']['job Settings.cron_jobs['update_container_registry_info_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['update_container_registry_info_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['update_container_registry_info_worker']['job_class'] = 'UpdateContainerRegistryInfoWorker' -Settings.cron_jobs['postgres_dynamic_partitions_creator'] ||= Settingslogic.new({}) -Settings.cron_jobs['postgres_dynamic_partitions_creator']['cron'] ||= '21 */6 * * *' -Settings.cron_jobs['postgres_dynamic_partitions_creator']['job_class'] ||= 'PartitionCreationWorker' +Settings.cron_jobs['postgres_dynamic_partitions_manager'] ||= Settingslogic.new({}) +Settings.cron_jobs['postgres_dynamic_partitions_manager']['cron'] ||= '21 */6 * * *' +Settings.cron_jobs['postgres_dynamic_partitions_manager']['job_class'] ||= 'Database::PartitionManagementWorker' Settings.cron_jobs['ci_platform_metrics_update_cron_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['ci_platform_metrics_update_cron_worker']['cron'] ||= '47 9 * * *' Settings.cron_jobs['ci_platform_metrics_update_cron_worker']['job_class'] = 'CiPlatformMetricsUpdateCronWorker' diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb index 060e3ce44d5..d4be1e7670d 100644 --- a/config/initializers/postgres_partitioning.rb +++ b/config/initializers/postgres_partitioning.rb @@ -3,15 +3,15 @@ # Make sure we have loaded partitioned models here # (even with eager loading disabled). -Gitlab::Database::Partitioning::PartitionCreator.register(AuditEvent) -Gitlab::Database::Partitioning::PartitionCreator.register(WebHookLog) +Gitlab::Database::Partitioning::PartitionManager.register(AuditEvent) +Gitlab::Database::Partitioning::PartitionManager.register(WebHookLog) if Gitlab.ee? - Gitlab::Database::Partitioning::PartitionCreator.register(IncidentManagement::PendingEscalations::Alert) + Gitlab::Database::Partitioning::PartitionManager.register(IncidentManagement::PendingEscalations::Alert) end begin - Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP'] + Gitlab::Database::Partitioning::PartitionManager.new.sync_partitions unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP'] rescue ActiveRecord::ActiveRecordError, PG::Error # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end diff --git a/config/metrics/counts_all/20210216175520_ci_runners.yml b/config/metrics/counts_all/20210216175520_ci_runners.yml index d0004bf8902..d13a77aecc0 100644 --- a/config/metrics/counts_all/20210216175520_ci_runners.yml +++ b/config/metrics/counts_all/20210216175520_ci_runners.yml @@ -1,7 +1,7 @@ --- data_category: Optional key_path: counts.ci_runners -description: Total configured Runners in project +description: Total configured Runners of all types product_section: ops product_stage: verify product_group: group::pipeline execution diff --git a/config/metrics/counts_all/20210502045402_ci_runners_instance_type_active.yml b/config/metrics/counts_all/20210502045402_ci_runners_instance_type_active.yml index 2435c75973a..e108406ef11 100644 --- a/config/metrics/counts_all/20210502045402_ci_runners_instance_type_active.yml +++ b/config/metrics/counts_all/20210502045402_ci_runners_instance_type_active.yml @@ -2,7 +2,7 @@ data_category: Optional key_path: counts.ci_runners_instance_type_active name: "count_active_instance_ci_runners" -description: Total active group Runners +description: Total active Shared (Instance) Runners product_section: ops product_stage: verify product_group: group::pipeline execution diff --git a/config/metrics/counts_all/20210502050341_ci_runners_group_type_active.yml b/config/metrics/counts_all/20210502050341_ci_runners_group_type_active.yml index afe7a4fe62a..6fe5d74a7e7 100644 --- a/config/metrics/counts_all/20210502050341_ci_runners_group_type_active.yml +++ b/config/metrics/counts_all/20210502050341_ci_runners_group_type_active.yml @@ -2,7 +2,7 @@ data_category: Optional key_path: counts.ci_runners_group_type_active name: "count_active_group_ci_runners" -description: Total active instance Runners +description: Total active Group Runners product_section: ops product_stage: verify product_group: group::pipeline execution diff --git a/config/metrics/counts_all/20210502050834_ci_runners_project_type_active.yml b/config/metrics/counts_all/20210502050834_ci_runners_project_type_active.yml index ce221f1f343..eaf9a3b5dd7 100644 --- a/config/metrics/counts_all/20210502050834_ci_runners_project_type_active.yml +++ b/config/metrics/counts_all/20210502050834_ci_runners_project_type_active.yml @@ -2,7 +2,7 @@ data_category: Optional key_path: counts.ci_runners_project_type_active name: "count_active_project_ci_runners" -description: Total active project Runners +description: Total active Specific (Project) Runners product_section: ops product_stage: verify product_group: group::pipeline execution diff --git a/config/metrics/counts_all/20210502050942_ci_runners_online.yml b/config/metrics/counts_all/20210502050942_ci_runners_online.yml index 4d2dc2d9a75..7e63e095cd4 100644 --- a/config/metrics/counts_all/20210502050942_ci_runners_online.yml +++ b/config/metrics/counts_all/20210502050942_ci_runners_online.yml @@ -2,7 +2,7 @@ data_category: Optional key_path: counts.ci_runners_online name: "counts_online_runners" -description: Total online Runners +description: Total online Runners of all types product_section: ops product_stage: verify product_group: group::pipeline execution diff --git a/config/metrics/counts_all/20210502051651_ci_runners_instance_type_active_online.yml b/config/metrics/counts_all/20210502051651_ci_runners_instance_type_active_online.yml index 5c8a4a11d0b..2e451b5955e 100644 --- a/config/metrics/counts_all/20210502051651_ci_runners_instance_type_active_online.yml +++ b/config/metrics/counts_all/20210502051651_ci_runners_instance_type_active_online.yml @@ -2,7 +2,7 @@ data_category: Optional key_path: counts.ci_runners_instance_type_active_online name: "count_instance_active_online_ci_runners" -description: Total active and online instance Runners +description: Total active and online Shared (Instance) Runners product_section: ops product_stage: verify product_group: group::pipeline execution diff --git a/config/metrics/counts_all/20210502051922_ci_runners_group_type_active_online.yml b/config/metrics/counts_all/20210502051922_ci_runners_group_type_active_online.yml index 48c98352d74..30c6432d611 100644 --- a/config/metrics/counts_all/20210502051922_ci_runners_group_type_active_online.yml +++ b/config/metrics/counts_all/20210502051922_ci_runners_group_type_active_online.yml @@ -2,7 +2,7 @@ data_category: Optional key_path: counts.ci_runners_group_type_active_online name: "count_group_active_online_ci_runners" -description: Total active and online group Runners +description: Total active and online Group Runners product_section: ops product_stage: verify product_group: group::pipeline execution diff --git a/config/metrics/counts_all/20210502052036_ci_runners_project_type_active_online.yml b/config/metrics/counts_all/20210502052036_ci_runners_project_type_active_online.yml index 80d326dc91c..8f47542eac2 100644 --- a/config/metrics/counts_all/20210502052036_ci_runners_project_type_active_online.yml +++ b/config/metrics/counts_all/20210502052036_ci_runners_project_type_active_online.yml @@ -2,7 +2,7 @@ data_category: Optional key_path: counts.ci_runners_project_type_active_online name: "count_project_active_online_ci_runners" -description: Total active and online project Runners +description: Total active and online Specific (Project) Runners product_section: ops product_stage: verify product_group: group::pipeline execution diff --git a/config/metrics/objects_schemas/collected_data_categories_schema.json b/config/metrics/objects_schemas/collected_data_categories_schema.json new file mode 100644 index 00000000000..c1ff96d3953 --- /dev/null +++ b/config/metrics/objects_schemas/collected_data_categories_schema.json @@ -0,0 +1,7 @@ +{ + "type": "array", + "items": { + "type": ["string", "null"], + "enum": ["Standard", "Subscription", "Operational", "Optional"] + } +} diff --git a/config/metrics/settings/20210702140138_collected_data_categories.yml b/config/metrics/settings/20210702140138_collected_data_categories.yml new file mode 100644 index 00000000000..dca7e1737db --- /dev/null +++ b/config/metrics/settings/20210702140138_collected_data_categories.yml @@ -0,0 +1,23 @@ +--- +key_path: settings.collected_data_categories +name: collected_data_categories +description: List of collected data categories corresponding to instance settings +product_section: growth +product_stage: growth +product_group: group::product intelligence +product_category: collection +value_type: object +status: implemented +milestone: "14.1" +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65336 +time_frame: none +data_source: system +data_category: Standard +distribution: +- ce +- ee +tier: +- free +- premium +- ultimate +value_json_schema: 'config/metrics/objects_schemas/collected_data_categories_schema.json' diff --git a/db/migrate/20210705144657_add_instance_runners_enabled_to_ci_pending_build.rb b/db/migrate/20210705144657_add_instance_runners_enabled_to_ci_pending_build.rb new file mode 100644 index 00000000000..b362fd930a3 --- /dev/null +++ b/db/migrate/20210705144657_add_instance_runners_enabled_to_ci_pending_build.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddInstanceRunnersEnabledToCiPendingBuild < ActiveRecord::Migration[6.1] + include Gitlab::Database::MigrationHelpers + + disable_ddl_transaction! + + def up + with_lock_retries do + add_column :ci_pending_builds, :instance_runners_enabled, :boolean, null: false, default: false + end + end + + def down + with_lock_retries do + remove_column :ci_pending_builds, :instance_runners_enabled + end + end +end diff --git a/db/schema_migrations/20210705144657 b/db/schema_migrations/20210705144657 new file mode 100644 index 00000000000..557dbdbd95c --- /dev/null +++ b/db/schema_migrations/20210705144657 @@ -0,0 +1 @@ +9ba27b5e2599262846a06736db72fb0d31dc904e2ef4d167c1ee9530feb6367f
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 6b583c8da46..749aeb60a5b 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10833,7 +10833,8 @@ CREATE TABLE ci_pending_builds ( build_id bigint NOT NULL, project_id bigint NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, - protected boolean DEFAULT false NOT NULL + protected boolean DEFAULT false NOT NULL, + instance_runners_enabled boolean DEFAULT false NOT NULL ); CREATE SEQUENCE ci_pending_builds_id_seq diff --git a/doc/administration/packages/index.md b/doc/administration/packages/index.md index 85c7a96ef55..2c2e3fc0442 100644 --- a/doc/administration/packages/index.md +++ b/doc/administration/packages/index.md @@ -26,6 +26,7 @@ The Package Registry supports the following formats: <tr><td><a href="https://docs.gitlab.com/ee/user/packages/nuget_repository/index.html">NuGet</a></td><td>12.8+</td></tr> <tr><td><a href="https://docs.gitlab.com/ee/user/packages/pypi_repository/index.html">PyPI</a></td><td>12.10+</td></tr> <tr><td><a href="https://docs.gitlab.com/ee/user/packages/generic_packages/index.html">Generic packages</a></td><td>13.5+</td></tr> +<tr><td><a href="https://docs.gitlab.com/ee/user/packages/helm_repository/index.html">Helm Charts</a></td><td>14.1+</td></tr> </table> </div> </div> diff --git a/doc/api/packages.md b/doc/api/packages.md index 863494147d5..73092e68c82 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -26,7 +26,7 @@ GET /projects/:id/packages | `id` | integer/string | yes | ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) | | `order_by`| string | no | The field to use as order. One of `created_at` (default), `name`, `version`, or `type`. | | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | -| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, or `golang`. (_Introduced in GitLab 12.9_) +| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, `helm`, or `golang`. (_Introduced in GitLab 12.9_) | `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_Introduced in GitLab 12.9_) | `include_versionless` | boolean | no | When set to true, versionless packages are included in the response. (_Introduced in GitLab 13.8_) | `status` | string | no | Filter the returned packages by status. One of `default` (default), `hidden`, or `processing`. (_Introduced in GitLab 13.9_) @@ -91,7 +91,7 @@ GET /groups/:id/packages | `exclude_subgroups` | boolean | false | If the parameter is included as true, packages from projects from subgroups are not listed. Default is `false`. | | `order_by`| string | no | The field to use as order. One of `created_at` (default), `name`, `version`, `type`, or `project_path`. | | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | -| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, or `golang`. (_Introduced in GitLab 12.9_) | +| `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, `helm`, or `golang`. (_Introduced in GitLab 12.9_) | | `package_name` | string | no | Filter the project packages with a fuzzy search by name. (_[Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30980) in GitLab 13.0_) | `include_versionless` | boolean | no | When set to true, versionless packages are included in the response. (_Introduced in GitLab 13.8_) | `status` | string | no | Filter the returned packages by status. One of `default` (default), `hidden`, or `processing`. (_Introduced in GitLab 13.9_) diff --git a/doc/api/packages/helm.md b/doc/api/packages/helm.md index 39054652908..a76fa9d3755 100644 --- a/doc/api/packages/helm.md +++ b/doc/api/packages/helm.md @@ -22,23 +22,6 @@ These endpoints do not adhere to the standard API authentication methods. See the [Helm registry documentation](../../user/packages/helm_repository/index.md) for details on which headers and token types are supported. -## Enable the Helm API - -The Helm API for GitLab is behind a feature flag that is disabled by default. GitLab -administrators with access to the GitLab Rails console can enable this API for your instance. - -To enable it: - -```ruby -Feature.enable(:helm_packages) -``` - -To disable it: - -```ruby -Feature.disable(:helm_packages) -``` - ## Download a chart index > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62757) in GitLab 14.1. diff --git a/doc/api/plan_limits.md b/doc/api/plan_limits.md index 14c1c3f6f47..c89c7b46d54 100644 --- a/doc/api/plan_limits.md +++ b/doc/api/plan_limits.md @@ -38,6 +38,7 @@ Example response: { "conan_max_file_size": 3221225472, "generic_packages_max_file_size": 5368709120, + "helm_max_file_size": 5242880, "maven_max_file_size": 3221225472, "npm_max_file_size": 524288000, "nuget_max_file_size": 524288000, @@ -59,6 +60,7 @@ PUT /application/plan_limits | `plan_name` | string | yes | Name of the plan to update. | | `conan_max_file_size` | integer | no | Maximum Conan package file size in bytes. | | `generic_packages_max_file_size` | integer | no | Maximum generic package file size in bytes. | +| `helm_max_file_size` | integer | no | Maximum Helm chart file size in bytes. | | `maven_max_file_size` | integer | no | Maximum Maven package file size in bytes. | | `npm_max_file_size` | integer | no | Maximum NPM package file size in bytes. | | `nuget_max_file_size` | integer | no | Maximum NuGet package file size in bytes. | @@ -75,6 +77,7 @@ Example response: { "conan_max_file_size": 3221225472, "generic_packages_max_file_size": 5368709120, + "helm_max_file_size": 5242880, "maven_max_file_size": 3221225472, "npm_max_file_size": 524288000, "nuget_max_file_size": 524288000, diff --git a/doc/development/database/database_reviewer_guidelines.md b/doc/development/database/database_reviewer_guidelines.md index 16734dada13..7a9c08d9d49 100644 --- a/doc/development/database/database_reviewer_guidelines.md +++ b/doc/development/database/database_reviewer_guidelines.md @@ -62,6 +62,9 @@ The following guides provide a quick introduction and links to follow on more ad - Guide on [understanding EXPLAIN plans](../understanding_explain_plans.md). - [Explaining the unexplainable series in `depesz`](https://www.depesz.com/tag/unexplainable/). +We also have licensed access to The Art of PostgreSQL available, if you are interested in getting access please check out the +[issue (confidential)](https://gitlab.com/gitlab-org/database-team/team-tasks/-/issues/23). + Finally, you can find various guides in the [Database guides](index.md) page that cover more specific topics and use cases. The most frequently required during database reviewing are the following: diff --git a/doc/development/database_review.md b/doc/development/database_review.md index d5dc98f9559..88639758a9d 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -151,7 +151,7 @@ test its execution using `CREATE INDEX CONCURRENTLY` in the `#database-lab` Slac - Provide a public link to the plan from either: - [postgres.ai](https://postgres.ai/): Follow the link in `#database-lab` and generate a shareable, public link by clicking the **Share** button in the upper right corner. - - [explain.depesz.com](https://explain.depesz.com): Paste both the plan and the query used in the form. + - [explain.depesz.com](https://explain.depesz.com) or [explain.dalibo.com](https://explain.dalibo.com): Paste both the plan and the query used in the form. - When providing query plans, make sure it hits enough data: - You can use a GitLab production replica to test your queries on a large scale, through the `#database-lab` Slack channel or through [ChatOps](understanding_explain_plans.md#chatops). diff --git a/doc/development/experiment_guide/gitlab_experiment.md b/doc/development/experiment_guide/gitlab_experiment.md index 3c7d60e1f60..33222b0492c 100644 --- a/doc/development/experiment_guide/gitlab_experiment.md +++ b/doc/development/experiment_guide/gitlab_experiment.md @@ -513,6 +513,11 @@ expect(experiment(:example)).to track(:my_event, value: 1, property: '_property_ experiment(:example, :variant_name, foo: :bar).track(:my_event, value: 1, property: '_property_') ``` +### Recording and assignment tracking + +To test assignment tracking and the `record!` method, you can use or adopt the following +shared example: [tracks assignment and records the subject](https://gitlab.com/gitlab-org/gitlab/blob/master/spec/support/shared_examples/lib/gitlab/experimentation_shared_examples.rb). + ## Experiments in the client layer This is in flux as of GitLab 13.10, and can't be documented just yet. diff --git a/doc/development/understanding_explain_plans.md b/doc/development/understanding_explain_plans.md index f9d1e7e2eee..c3fefd40171 100644 --- a/doc/development/understanding_explain_plans.md +++ b/doc/development/understanding_explain_plans.md @@ -825,3 +825,5 @@ For more information about the available options, run: A more extensive guide on understanding query plans can be found in the [presentation](https://public.dalibo.com/exports/conferences/_archives/_2012/201211_explain/understanding_explain.pdf) from [Dalibo.org](https://www.dalibo.com/en/). + +Depesz's blog also has a good [section](https://www.depesz.com/tag/unexplainable) dedicated to query plans. diff --git a/doc/development/usage_ping/dictionary.md b/doc/development/usage_ping/dictionary.md index 153c385c4bb..246af7634d8 100644 --- a/doc/development/usage_ping/dictionary.md +++ b/doc/development/usage_ping/dictionary.md @@ -666,7 +666,7 @@ Tiers: `free`, `premium`, `ultimate` ### `counts.ci_runners` -Total configured Runners in project +Total configured Runners of all types [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210216175520_ci_runners.yml) @@ -680,7 +680,7 @@ Tiers: `free`, `premium`, `ultimate` ### `counts.ci_runners_group_type_active` -Total active instance Runners +Total active Group Runners [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210502050341_ci_runners_group_type_active.yml) @@ -694,7 +694,7 @@ Tiers: `free`, `premium`, `ultimate` ### `counts.ci_runners_group_type_active_online` -Total active and online group Runners +Total active and online Group Runners [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210502051922_ci_runners_group_type_active_online.yml) @@ -708,7 +708,7 @@ Tiers: `free`, `premium`, `ultimate` ### `counts.ci_runners_instance_type_active` -Total active group Runners +Total active Shared (Instance) Runners [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210502045402_ci_runners_instance_type_active.yml) @@ -722,7 +722,7 @@ Tiers: `free`, `premium`, `ultimate` ### `counts.ci_runners_instance_type_active_online` -Total active and online instance Runners +Total active and online Shared (Instance) Runners [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210502051651_ci_runners_instance_type_active_online.yml) @@ -736,7 +736,7 @@ Tiers: `free`, `premium`, `ultimate` ### `counts.ci_runners_online` -Total online Runners +Total online Runners of all types [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210502050942_ci_runners_online.yml) @@ -750,7 +750,7 @@ Tiers: `free`, `premium`, `ultimate` ### `counts.ci_runners_project_type_active` -Total active project Runners +Total active Specific (Project) Runners [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210502050834_ci_runners_project_type_active.yml) @@ -764,7 +764,7 @@ Tiers: `free`, `premium`, `ultimate` ### `counts.ci_runners_project_type_active_online` -Total active and online project Runners +Total active and online Specific (Project) Runners [YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/counts_all/20210502052036_ci_runners_project_type_active_online.yml) @@ -18316,6 +18316,22 @@ Status: `data_available` Tiers: `free`, `premium`, `ultimate` +### `settings.collected_data_categories` + +List of collected data categories corresponding to instance settings + +[Object JSON schema](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/objects_schemas/collected_data_categories_schema.json) + +[YAML definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/config/metrics/settings/20210702140138_collected_data_categories.yml) + +Group: `group::product intelligence` + +Data Category: `Standard` + +Status: `implemented` + +Tiers: `free`, `premium`, `ultimate` + ### `settings.gitaly_apdex` Gitaly application performance diff --git a/doc/user/application_security/secret_detection/index.md b/doc/user/application_security/secret_detection/index.md index 5f5ad7aa0b6..938bd3b41d5 100644 --- a/doc/user/application_security/secret_detection/index.md +++ b/doc/user/application_security/secret_detection/index.md @@ -275,7 +275,7 @@ Post-processing is currently limited to a project's default branch, see the abov sequenceDiagram autonumber Rails->>+Sidekiq: gl-secret-detection-report.json - Sidekiq-->+Sidekiq: BuildFinishedWorker + Sidekiq-->+Sidekiq: Ci::BuildFinishedWorker Sidekiq-->+RevocationAPI: GET revocable keys types RevocationAPI-->>-Sidekiq: OK Sidekiq->>+RevocationAPI: POST revoke revocable keys diff --git a/doc/user/packages/helm_repository/index.md b/doc/user/packages/helm_repository/index.md index 428c95dac0a..99fde6a97eb 100644 --- a/doc/user/packages/helm_repository/index.md +++ b/doc/user/packages/helm_repository/index.md @@ -18,24 +18,6 @@ packages whenever you need to use them as a dependency. For documentation of the specific API endpoints that Helm package manager clients use, see the [Helm API documentation](../../../api/packages/helm.md). -## Enable the Helm repository feature - -Helm repository support is still a work in progress. It's gated behind a feature flag that's -**disabled by default**. [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) -can opt to enable it. - -To enable it: - -```ruby -Feature.enable(:helm_packages) -``` - -To disable it: - -```ruby -Feature.disable(:helm_packages) -``` - ## Build a Helm package Creating a Helm package is documented [in the Helm documentation](https://helm.sh/docs/intro/using_helm/#creating-your-own-charts). @@ -73,8 +55,16 @@ Once built, a chart can be uploaded to the `stable` channel with `curl` or `helm To install the latest version of a chart, use the following command: ```shell -helm repo add project-1 https://gitlab.example.com/api/v4/projects/1/packages/helm/stable +helm repo add --username <username> --password <personal_access_token> project-1 https://gitlab.example.com/api/v4/projects/1/packages/helm/stable helm install my-release project-1/mychart ``` +If the repo has previously been added, you may need to run: + +```shell +helm repo update +``` + +To update the Helm client with the most currently available charts. + See [Using Helm](https://helm.sh/docs/intro/using_helm/) for more information. diff --git a/doc/user/packages/package_registry/index.md b/doc/user/packages/package_registry/index.md index a7c59cbd4f4..cb4e677687e 100644 --- a/doc/user/packages/package_registry/index.md +++ b/doc/user/packages/package_registry/index.md @@ -35,7 +35,7 @@ For information on how to create and upload a package, view the GitLab documenta ## Use GitLab CI/CD to build packages You can use [GitLab CI/CD](../../../ci/index.md) to build packages. -For Maven, NuGet, npm, Conan, and PyPI packages, and Composer dependencies, you can +For Maven, NuGet, npm, Conan, Helm, and PyPI packages, and Composer dependencies, you can authenticate with GitLab by using the `CI_JOB_TOKEN`. CI/CD templates, which you can use to get started, are in [this repository](https://gitlab.com/gitlab-org/gitlab/-/tree/master/lib/gitlab/ci/templates). diff --git a/lib/api/helm_packages.rb b/lib/api/helm_packages.rb index 12928528255..4280744d8b4 100644 --- a/lib/api/helm_packages.rb +++ b/lib/api/helm_packages.rb @@ -29,10 +29,6 @@ module API require_packages_enabled! end - after_validation do - not_found! unless Feature.enabled?(:helm_packages, authorized_user_project) - end - params do requires :id, type: String, desc: 'The ID or full path of a project' end diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index 82ea1ce26fb..4c68399cb68 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -4,16 +4,17 @@ module Gitlab module Database module Partitioning class MonthlyStrategy - attr_reader :model, :partitioning_key + attr_reader :model, :partitioning_key, :retain_for # We create this many partitions in the future HEADROOM = 6.months delegate :table_name, to: :model - def initialize(model, partitioning_key) + def initialize(model, partitioning_key, retain_for: nil) @model = model @partitioning_key = partitioning_key + @retain_for = retain_for end def current_partitions @@ -27,13 +28,21 @@ module Gitlab desired_partitions - current_partitions end + def extra_partitions + current_partitions - desired_partitions + end + private def desired_partitions [].tap do |parts| min_date, max_date = relevant_range - parts << partition_for(upper_bound: min_date) + if pruning_old_partitions? && min_date <= oldest_active_date + min_date = oldest_active_date.beginning_of_month + else + parts << partition_for(upper_bound: min_date) + end while min_date < max_date next_date = min_date.next_month @@ -52,13 +61,17 @@ module Gitlab # to start from MINVALUE to a specific date `x`. The range returned # does not include the range of the first, half-unbounded partition. def relevant_range - if first_partition = current_partitions.min + if (first_partition = current_partitions.min) # Case 1: First partition starts with MINVALUE, i.e. from is nil -> start with first real partition # Case 2: Rather unexpectedly, first partition does not start with MINVALUE, i.e. from is not nil # In this case, use first partition beginning as a start min_date = first_partition.from || first_partition.to end + if pruning_old_partitions? + min_date ||= oldest_active_date + end + # In case we don't have a partition yet min_date ||= Date.today min_date = min_date.beginning_of_month @@ -72,6 +85,14 @@ module Gitlab TimePartition.new(table_name, lower_bound, upper_bound) end + def pruning_old_partitions? + Feature.enabled?(:partition_pruning_dry_run) && retain_for.present? + end + + def oldest_active_date + (Date.today - retain_for).beginning_of_month + end + def connection ActiveRecord::Base.connection end diff --git a/lib/gitlab/database/partitioning/partition_creator.rb b/lib/gitlab/database/partitioning/partition_manager.rb index d4b2b8d50e2..c2a9422a42a 100644 --- a/lib/gitlab/database/partitioning/partition_creator.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -3,7 +3,7 @@ module Gitlab module Database module Partitioning - class PartitionCreator + class PartitionManager def self.register(model) raise ArgumentError, "Only models with a #partitioning_strategy can be registered." unless model.respond_to?(:partitioning_strategy) @@ -15,7 +15,7 @@ module Gitlab end LEASE_TIMEOUT = 1.minute - LEASE_KEY = 'database_partition_creation_%s' + MANAGEMENT_LEASE_KEY = 'database_partition_management_%s' attr_reader :models @@ -23,23 +23,25 @@ module Gitlab @models = models end - def create_partitions + def sync_partitions Gitlab::AppLogger.info("Checking state of dynamic postgres partitions") models.each do |model| # Double-checking before getting the lease: - # The prevailing situation is no missing partitions - next if missing_partitions(model).empty? + # The prevailing situation is no missing partitions and no extra partitions + next if missing_partitions(model).empty? && extra_partitions(model).empty? - only_with_exclusive_lease(model) do + only_with_exclusive_lease(model, lease_key: MANAGEMENT_LEASE_KEY) do partitions_to_create = missing_partitions(model) + create(partitions_to_create) unless partitions_to_create.empty? - next if partitions_to_create.empty? - - create(model, partitions_to_create) + if Feature.enabled?(:partition_pruning_dry_run) + partitions_to_detach = extra_partitions(model) + detach(partitions_to_detach) unless partitions_to_detach.empty? + end end rescue StandardError => e - Gitlab::AppLogger.error("Failed to create partition(s) for #{model.table_name}: #{e.class}: #{e.message}") + Gitlab::AppLogger.error("Failed to create / detach partition(s) for #{model.table_name}: #{e.class}: #{e.message}") end end @@ -51,15 +53,22 @@ module Gitlab model.partitioning_strategy.missing_partitions end - def only_with_exclusive_lease(model) - lease = Gitlab::ExclusiveLease.new(LEASE_KEY % model.table_name, timeout: LEASE_TIMEOUT) + def extra_partitions(model) + return [] unless Feature.enabled?(:partition_pruning_dry_run) + return [] unless connection.table_exists?(model.table_name) + + model.partitioning_strategy.extra_partitions + end + + def only_with_exclusive_lease(model, lease_key:) + lease = Gitlab::ExclusiveLease.new(lease_key % model.table_name, timeout: LEASE_TIMEOUT) yield if lease.try_obtain ensure lease&.cancel end - def create(model, partitions) + def create(partitions) connection.transaction do with_lock_retries do partitions.each do |partition| @@ -71,6 +80,18 @@ module Gitlab end end + def detach(partitions) + connection.transaction do + with_lock_retries do + partitions.each { |p| detach_one_partition(p) } + end + end + end + + def detach_one_partition(partition) + Gitlab::AppLogger.info("Planning to detach #{partition.partition_name} for table #{partition.table}") + end + def with_lock_retries(&block) Gitlab::Database::WithLockRetries.new( klass: self.class, diff --git a/lib/gitlab/database/partitioning/partition_monitoring.rb b/lib/gitlab/database/partitioning/partition_monitoring.rb index 9ec9ae684a5..ad122fd47fe 100644 --- a/lib/gitlab/database/partitioning/partition_monitoring.rb +++ b/lib/gitlab/database/partitioning/partition_monitoring.rb @@ -6,7 +6,7 @@ module Gitlab class PartitionMonitoring attr_reader :models - def initialize(models = PartitionCreator.models) + def initialize(models = PartitionManager.models) @models = models end diff --git a/lib/gitlab/database/postgres_index.rb b/lib/gitlab/database/postgres_index.rb index 2f9e26ad451..b6c57528e9f 100644 --- a/lib/gitlab/database/postgres_index.rb +++ b/lib/gitlab/database/postgres_index.rb @@ -38,7 +38,10 @@ module Gitlab where('NOT EXISTS (?)', recent_actions) end - alias_method :reset, :reload + def reset + reload # rubocop:disable Cop/ActiveRecordAssociationReload + clear_memoization(:bloat_size) + end def bloat_size strong_memoize(:bloat_size) { bloat_estimate&.bloat_size || 0 } diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb index 4183c886c10..e4835504c2d 100644 --- a/lib/gitlab/github_import/object_counter.rb +++ b/lib/gitlab/github_import/object_counter.rb @@ -1,24 +1,24 @@ # frozen_string_literal: true -# Count objects fetched or imported from Github in the context of the -# project being imported. +# Count objects fetched or imported from Github. module Gitlab module GithubImport class ObjectCounter OPERATIONS = %w[fetched imported].freeze - COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}' - COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}' + PROJECT_COUNTER_LIST_KEY = 'github-importer/object-counters-list/%{project}/%{operation}' + PROJECT_COUNTER_KEY = 'github-importer/object-counter/%{project}/%{operation}/%{object_type}' + + GLOBAL_COUNTER_KEY = 'github_importer_%{operation}_%{object_type}' + GLOBAL_COUNTER_DESCRIPTION = 'The number of %{operation} Github %{object_type}' + CACHING = Gitlab::Cache::Import::Caching class << self def increment(project, object_type, operation) validate_operation!(operation) - counter_key = COUNTER_KEY % { project: project.id, operation: operation, object_type: object_type } - - add_counter_to_list(project, operation, counter_key) - - CACHING.increment(counter_key) + increment_project_counter(project, object_type, operation) + increment_global_counter(object_type, operation) end def summary(project) @@ -37,12 +37,40 @@ module Gitlab private + # Global counters are long lived, in Prometheus, + # and it's used to report the health of the Github Importer + # in the Grafana Dashboard + # https://dashboards.gitlab.net/d/2zgM_rImz/github-importer?orgId=1 + def increment_global_counter(object_type, operation) + key = GLOBAL_COUNTER_KEY % { + operation: operation, + object_type: object_type + } + description = GLOBAL_COUNTER_DESCRIPTION % { + operation: operation, + object_type: object_type.to_s.humanize + } + + Gitlab::Metrics.counter(key.to_sym, description).increment + end + + # Project counters are short lived, in Redis, + # and it's used to report how successful a project + # import was with the #summary method. + def increment_project_counter(project, object_type, operation) + counter_key = PROJECT_COUNTER_KEY % { project: project.id, operation: operation, object_type: object_type } + + add_counter_to_list(project, operation, counter_key) + + CACHING.increment(counter_key) + end + def add_counter_to_list(project, operation, key) CACHING.set_add(counter_list_key(project, operation), key) end def counter_list_key(project, operation) - COUNTER_LIST_KEY % { project: project.id, operation: operation } + PROJECT_COUNTER_LIST_KEY % { project: project.id, operation: operation } end def validate_operation!(operation) diff --git a/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb new file mode 100644 index 00000000000..dd1f9948815 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CollectedDataCategoriesMetric < GenericMetric + def value + ::ServicePing::PermitDataCategoriesService.new.execute + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 80540257b6c..aabc706901e 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -256,7 +256,8 @@ module Gitlab settings: { ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, operating_system: alt_usage_data(fallback: nil) { operating_system }, - gitaly_apdex: alt_usage_data { gitaly_apdex } + gitaly_apdex: alt_usage_data { gitaly_apdex }, + collected_data_categories: alt_usage_data(fallback: []) { Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric.new(time_frame: 'none').value } } } end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 963c7e486f6..82d0c83b311 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -118,7 +118,7 @@ namespace :gitlab do desc 'Create missing dynamic database partitions' task create_dynamic_partitions: :environment do - Gitlab::Database::Partitioning::PartitionCreator.new.create_partitions + Gitlab::Database::Partitioning::PartitionManager.new.sync_partitions end # This is targeted towards deploys and upgrades of GitLab. diff --git a/locale/gitlab.pot b/locale/gitlab.pot index d65b6817810..bd24e75512c 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -3855,6 +3855,9 @@ msgstr "" msgid "An error occurred while updating the notification settings. Please try again." msgstr "" +msgid "An error occurred while uploading the image. Please try again." +msgstr "" + msgid "An error occurred while validating group path" msgstr "" @@ -11683,6 +11686,9 @@ msgstr "" msgid "Download %{name} artifact" msgstr "" +msgid "Download (%{fileSizeReadable})" +msgstr "" + msgid "Download (%{size})" msgstr "" @@ -13075,9 +13081,6 @@ msgstr "" msgid "EscalationPolicies|Edit escalation policy" msgstr "" -msgid "EscalationPolicies|Elapsed time must be greater than or equal to zero." -msgstr "" - msgid "EscalationPolicies|Email on-call user in schedule" msgstr "" @@ -13096,6 +13099,9 @@ msgstr "" msgid "EscalationPolicies|IF alert is not %{alertStatus} in %{minutes} minutes" msgstr "" +msgid "EscalationPolicies|Minutes must be between 0 and 1440." +msgstr "" + msgid "EscalationPolicies|Remove escalation rule" msgstr "" @@ -16757,6 +16763,9 @@ msgstr[1] "" msgid "Importing..." msgstr "" +msgid "Import|A repository URL usually ends in a .git suffix, although this is not required. Double check to make sure your repository URL is correct." +msgstr "" + msgid "Improve customer support with Service Desk" msgstr "" @@ -33477,6 +33486,9 @@ msgstr "" msgid "This group can't be removed because it is linked to a subscription. To remove this group, %{linkStart}link the subscription%{linkEnd} with a different group." msgstr "" +msgid "This group can't be transfered because it is linked to a subscription. To transfer this group, %{linkStart}link the subscription%{linkEnd} with a different group." +msgstr "" + msgid "This group cannot be invited to a project inside a group with enforced SSO" msgstr "" diff --git a/package.json b/package.json index 31432dad363..1a74e453b53 100644 --- a/package.json +++ b/package.json @@ -157,6 +157,7 @@ "prosemirror-inputrules": "^1.1.3", "prosemirror-markdown": "^1.5.1", "prosemirror-model": "^1.13.3", + "prosemirror-state": "^1.3.4", "raphael": "^2.2.7", "raw-loader": "^4.0.2", "scrollparent": "^2.0.1", diff --git a/qa/Gemfile b/qa/Gemfile index cb8915f9c25..75d4e2607ee 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -24,7 +24,7 @@ gem 'parallel', '~> 1.19' gem 'rspec-parameterized', '~> 0.4.2' gem 'github_api', '~> 0.18.2' -gem 'chemlab', '~> 0.5' +gem 'chemlab', '~> 0.7' gem 'chemlab-library-www-gitlab-com', '~> 0.1' group :development do diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 31bd9ab9c54..5f12715d372 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -41,18 +41,21 @@ GEM capybara-screenshot (1.0.23) capybara (>= 1.0, < 4) launchy - chemlab (0.5.0) - rake (~> 12.3.0) - selenium-webdriver (~> 3.12) - watir (~> 6.17) + chemlab (0.7.2) + colorize (~> 0.8) + i18n (~> 1.8) + rake (>= 12, < 14) + selenium-webdriver (>= 3, < 5) + watir (>= 6, < 8) chemlab-library-www-gitlab-com (0.1.1) chemlab (~> 0.4) childprocess (4.1.0) coderay (1.1.2) + colorize (0.8.1) concord (0.1.5) adamantium (~> 0.2.0) equalizer (~> 0.0.9) - concurrent-ruby (1.1.8) + concurrent-ruby (1.1.9) descendants_tracker (0.0.4) thread_safe (~> 0.3, >= 0.3.1) diff-lcs (1.3) @@ -129,7 +132,7 @@ GEM rack-test (1.1.0) rack (>= 1.0, < 3) rake (12.3.3) - regexp_parser (1.6.0) + regexp_parser (1.8.2) require_all (3.0.0) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) @@ -163,7 +166,7 @@ GEM rspec-core (>= 2, < 4, != 2.12.0) ruby-debug-ide (0.7.2) rake (>= 0.8.1) - rubyzip (2.3.0) + rubyzip (2.3.2) selenium-webdriver (4.0.0.beta4) childprocess (>= 0.5, < 5.0) rexml (~> 3.2) @@ -186,9 +189,9 @@ GEM procto (~> 0.0.2) uuid (2.3.9) macaddr (~> 1.0) - watir (6.18.0) + watir (6.19.1) regexp_parser (>= 1.2, < 3) - selenium-webdriver (>= 3.8) + selenium-webdriver (>= 3.142.7) xpath (3.2.0) nokogiri (~> 1.8) zeitwerk (2.4.2) @@ -202,7 +205,7 @@ DEPENDENCIES allure-rspec (~> 2.14.1) capybara (~> 3.29.0) capybara-screenshot (~> 1.0.23) - chemlab (~> 0.5) + chemlab (~> 0.7) chemlab-library-www-gitlab-com (~> 0.1) faker (~> 1.6, >= 1.6.6) github_api (~> 0.18.2) @@ -224,4 +227,4 @@ DEPENDENCIES timecop (~> 0.9.1) BUNDLED WITH - 2.1.4 + 2.2.22 diff --git a/qa/qa/page/group/settings/general.rb b/qa/qa/page/group/settings/general.rb index 4977e5c7105..2e7ab131225 100644 --- a/qa/qa/page/group/settings/general.rb +++ b/qa/qa/page/group/settings/general.rb @@ -38,7 +38,7 @@ module QA element :project_creation_level_dropdown end - view 'app/views/groups/settings/_advanced.html.haml' do + view 'app/views/groups/settings/_transfer.html.haml' do element :select_group_dropdown element :transfer_group_button end diff --git a/qa/qa/runtime/browser.rb b/qa/qa/runtime/browser.rb index 0488103ceb3..9cc72ae64a5 100644 --- a/qa/qa/runtime/browser.rb +++ b/qa/qa/runtime/browser.rb @@ -179,6 +179,7 @@ module QA config.browser = Capybara.current_session.driver.browser # reuse Capybara session config.libraries = [GitlabHandbook] config.base_url = Runtime::Scenario.attributes[:gitlab_address] # reuse GitLab address + config.hide_banner = true end end # rubocop: enable Metrics/AbcSize diff --git a/scripts/review_apps/automated_cleanup.rb b/scripts/review_apps/automated_cleanup.rb index 0927481070b..5707f02d3f0 100755 --- a/scripts/review_apps/automated_cleanup.rb +++ b/scripts/review_apps/automated_cleanup.rb @@ -116,6 +116,12 @@ class AutomatedCleanup delete_helm_releases(releases_to_delete) end + def perform_stale_namespace_cleanup!(days:) + kubernetes_client = Tooling::KubernetesClient.new(namespace: nil) + + kubernetes_client.cleanup_review_app_namespaces(created_before: threshold_time(days: days), wait: false) + end + def perform_stale_pvc_cleanup!(days:) kubernetes.cleanup_by_created_at(resource_type: 'pvc', created_before: threshold_time(days: days), wait: false) end @@ -203,6 +209,10 @@ timed('Helm releases cleanup') do automated_cleanup.perform_helm_releases_cleanup!(days: 7) end +timed('Stale Namespace cleanup') do + automated_cleanup.perform_stale_namespace_cleanup!(days: 14) +end + timed('Stale PVC cleanup') do automated_cleanup.perform_stale_pvc_cleanup!(days: 30) end diff --git a/spec/factories/ci/pending_builds.rb b/spec/factories/ci/pending_builds.rb index 3e2544b0ce1..90779ae8ab9 100644 --- a/spec/factories/ci/pending_builds.rb +++ b/spec/factories/ci/pending_builds.rb @@ -5,5 +5,6 @@ FactoryBot.define do build factory: :ci_build project protected { build.protected } + instance_runners_enabled { true } end end diff --git a/spec/features/issues/filtered_search/dropdown_assignee_spec.rb b/spec/features/issues/filtered_search/dropdown_assignee_spec.rb index 381633b0fc9..e873ebb21c4 100644 --- a/spec/features/issues/filtered_search/dropdown_assignee_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_assignee_spec.rb @@ -5,15 +5,16 @@ require 'spec_helper' RSpec.describe 'Dropdown assignee', :js do include FilteredSearchHelpers - let!(:project) { create(:project) } - let!(:user) { create(:user, name: 'administrator', username: 'root') } + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user, name: 'administrator', username: 'root') } + let_it_be(:issue) { create(:issue, project: project) } + let(:js_dropdown_assignee) { '#js-dropdown-assignee' } let(:filter_dropdown) { find("#{js_dropdown_assignee} .filter-dropdown") } before do project.add_maintainer(user) sign_in(user) - create(:issue, project: project) visit project_issues_path(project) end diff --git a/spec/features/issues/filtered_search/dropdown_author_spec.rb b/spec/features/issues/filtered_search/dropdown_author_spec.rb index 91c85825a17..893ffc6575b 100644 --- a/spec/features/issues/filtered_search/dropdown_author_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_author_spec.rb @@ -5,15 +5,16 @@ require 'spec_helper' RSpec.describe 'Dropdown author', :js do include FilteredSearchHelpers - let!(:project) { create(:project) } - let!(:user) { create(:user, name: 'administrator', username: 'root') } + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user, name: 'administrator', username: 'root') } + let_it_be(:issue) { create(:issue, project: project) } + let(:js_dropdown_author) { '#js-dropdown-author' } let(:filter_dropdown) { find("#{js_dropdown_author} .filter-dropdown") } before do project.add_maintainer(user) sign_in(user) - create(:issue, project: project) visit project_issues_path(project) end diff --git a/spec/features/issues/filtered_search/dropdown_base_spec.rb b/spec/features/issues/filtered_search/dropdown_base_spec.rb index d730525cb8b..3a304515cab 100644 --- a/spec/features/issues/filtered_search/dropdown_base_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_base_spec.rb @@ -5,8 +5,10 @@ require 'spec_helper' RSpec.describe 'Dropdown base', :js do include FilteredSearchHelpers - let!(:project) { create(:project) } - let!(:user) { create(:user, name: 'administrator', username: 'root') } + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user, name: 'administrator', username: 'root') } + let_it_be(:issue) { create(:issue, project: project) } + let(:filtered_search) { find('.filtered-search') } let(:js_dropdown_assignee) { '#js-dropdown-assignee' } let(:filter_dropdown) { find("#{js_dropdown_assignee} .filter-dropdown") } @@ -18,7 +20,6 @@ RSpec.describe 'Dropdown base', :js do before do project.add_maintainer(user) sign_in(user) - create(:issue, project: project) visit project_issues_path(project) end diff --git a/spec/features/issues/filtered_search/dropdown_emoji_spec.rb b/spec/features/issues/filtered_search/dropdown_emoji_spec.rb index c2c933f8a86..f5ab53d5052 100644 --- a/spec/features/issues/filtered_search/dropdown_emoji_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_emoji_spec.rb @@ -5,10 +5,11 @@ require 'spec_helper' RSpec.describe 'Dropdown emoji', :js do include FilteredSearchHelpers - let!(:project) { create(:project, :public) } - let!(:user) { create(:user, name: 'administrator', username: 'root') } - let!(:issue) { create(:issue, project: project) } - let!(:award_emoji_star) { create(:award_emoji, name: 'star', user: user, awardable: issue) } + let_it_be(:project) { create(:project, :public) } + let_it_be(:user) { create(:user, name: 'administrator', username: 'root') } + let_it_be(:issue) { create(:issue, project: project) } + let_it_be(:award_emoji_star) { create(:award_emoji, name: 'star', user: user, awardable: issue) } + let(:filtered_search) { find('.filtered-search') } let(:js_dropdown_emoji) { '#js-dropdown-my-reaction' } let(:filter_dropdown) { find("#{js_dropdown_emoji} .filter-dropdown") } diff --git a/spec/features/issues/filtered_search/dropdown_hint_spec.rb b/spec/features/issues/filtered_search/dropdown_hint_spec.rb index 9edc6e0b593..9cc58a33bb7 100644 --- a/spec/features/issues/filtered_search/dropdown_hint_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_hint_spec.rb @@ -5,8 +5,10 @@ require 'spec_helper' RSpec.describe 'Dropdown hint', :js do include FilteredSearchHelpers - let!(:project) { create(:project, :public) } - let!(:user) { create(:user) } + let_it_be(:project) { create(:project, :public) } + let_it_be(:user) { create(:user) } + let_it_be(:issue) { create(:issue, project: project) } + let(:filtered_search) { find('.filtered-search') } let(:js_dropdown_hint) { '#js-dropdown-hint' } let(:js_dropdown_operator) { '#js-dropdown-operator' } @@ -21,8 +23,6 @@ RSpec.describe 'Dropdown hint', :js do before do project.add_maintainer(user) - create(:issue, project: project) - create(:merge_request, source_project: project, target_project: project) end context 'when user not logged in' do diff --git a/spec/features/issues/filtered_search/dropdown_label_spec.rb b/spec/features/issues/filtered_search/dropdown_label_spec.rb index c0d5fe0d860..1b48810f716 100644 --- a/spec/features/issues/filtered_search/dropdown_label_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_label_spec.rb @@ -5,22 +5,23 @@ require 'spec_helper' RSpec.describe 'Dropdown label', :js do include FilteredSearchHelpers - let(:project) { create(:project) } - let(:user) { create(:user) } + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user) } + let_it_be(:issue) { create(:issue, project: project) } + let_it_be(:label) { create(:label, project: project, title: 'bug-label') } + let(:filtered_search) { find('.filtered-search') } let(:filter_dropdown) { find('#js-dropdown-label .filter-dropdown') } before do project.add_maintainer(user) sign_in(user) - create(:issue, project: project) visit project_issues_path(project) end describe 'behavior' do it 'loads all the labels when opened' do - create(:label, project: project, title: 'bug-label') filtered_search.set('label:=') expect_filtered_search_dropdown_results(filter_dropdown, 1) diff --git a/spec/features/issues/filtered_search/dropdown_milestone_spec.rb b/spec/features/issues/filtered_search/dropdown_milestone_spec.rb index 68afd973f1d..859d1e4a5e5 100644 --- a/spec/features/issues/filtered_search/dropdown_milestone_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_milestone_spec.rb @@ -5,10 +5,11 @@ require 'spec_helper' RSpec.describe 'Dropdown milestone', :js do include FilteredSearchHelpers - let!(:project) { create(:project) } - let!(:user) { create(:user) } - let!(:milestone) { create(:milestone, title: 'v1.0', project: project) } - let!(:uppercase_milestone) { create(:milestone, title: 'CAP_MILESTONE', project: project) } + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user) } + let_it_be(:milestone) { create(:milestone, title: 'v1.0', project: project) } + let_it_be(:uppercase_milestone) { create(:milestone, title: 'CAP_MILESTONE', project: project) } + let_it_be(:issue) { create(:issue, project: project) } let(:filtered_search) { find('.filtered-search') } let(:filter_dropdown) { find('#js-dropdown-milestone .filter-dropdown') } @@ -16,7 +17,6 @@ RSpec.describe 'Dropdown milestone', :js do before do project.add_maintainer(user) sign_in(user) - create(:issue, project: project) visit project_issues_path(project) end diff --git a/spec/features/issues/filtered_search/dropdown_release_spec.rb b/spec/features/issues/filtered_search/dropdown_release_spec.rb index daf686c2850..2210a26c251 100644 --- a/spec/features/issues/filtered_search/dropdown_release_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_release_spec.rb @@ -5,10 +5,11 @@ require 'spec_helper' RSpec.describe 'Dropdown release', :js do include FilteredSearchHelpers - let!(:project) { create(:project, :repository) } - let!(:user) { create(:user) } - let!(:release) { create(:release, tag: 'v1.0', project: project) } - let!(:crazy_release) { create(:release, tag: '☺!/"#%&\'{}+,-.<>;=@]_`{|}🚀', project: project) } + let_it_be(:project) { create(:project, :repository) } + let_it_be(:user) { create(:user) } + let_it_be(:release) { create(:release, tag: 'v1.0', project: project) } + let_it_be(:crazy_release) { create(:release, tag: '☺!/"#%&\'{}+,-.<>;=@]_`{|}🚀', project: project) } + let_it_be(:issue) { create(:issue, project: project) } let(:filtered_search) { find('.filtered-search') } let(:filter_dropdown) { find('#js-dropdown-release .filter-dropdown') } @@ -16,7 +17,6 @@ RSpec.describe 'Dropdown release', :js do before do project.add_maintainer(user) sign_in(user) - create(:issue, project: project) visit project_issues_path(project) end diff --git a/spec/features/issues/filtered_search/recent_searches_spec.rb b/spec/features/issues/filtered_search/recent_searches_spec.rb index 61c1e35f3c8..3ddcbf1bd01 100644 --- a/spec/features/issues/filtered_search/recent_searches_spec.rb +++ b/spec/features/issues/filtered_search/recent_searches_spec.rb @@ -6,14 +6,15 @@ RSpec.describe 'Recent searches', :js do include FilteredSearchHelpers include MobileHelpers - let(:project_1) { create(:project, :public) } - let(:project_2) { create(:project, :public) } + let_it_be(:project_1) { create(:project, :public) } + let_it_be(:project_2) { create(:project, :public) } + let_it_be(:issue_1) { create(:issue, project: project_1) } + let_it_be(:issue_2) { create(:issue, project: project_2) } + let(:project_1_local_storage_key) { "#{project_1.full_path}-issue-recent-searches" } before do Capybara.ignore_hidden_elements = false - create(:issue, project: project_1) - create(:issue, project: project_2) # Visit any fast-loading page so we can clear local storage without a DOM exception visit '/404' diff --git a/spec/features/issues/filtered_search/search_bar_spec.rb b/spec/features/issues/filtered_search/search_bar_spec.rb index 2a094281133..1efcc329e32 100644 --- a/spec/features/issues/filtered_search/search_bar_spec.rb +++ b/spec/features/issues/filtered_search/search_bar_spec.rb @@ -5,14 +5,15 @@ require 'spec_helper' RSpec.describe 'Search bar', :js do include FilteredSearchHelpers - let!(:project) { create(:project) } - let!(:user) { create(:user) } + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user) } + let_it_be(:issue) { create(:issue, project: project) } + let(:filtered_search) { find('.filtered-search') } before do project.add_maintainer(user) sign_in(user) - create(:issue, project: project) visit project_issues_path(project) end diff --git a/spec/features/issues/filtered_search/visual_tokens_spec.rb b/spec/features/issues/filtered_search/visual_tokens_spec.rb index c585d7f6194..644d7cc4611 100644 --- a/spec/features/issues/filtered_search/visual_tokens_spec.rb +++ b/spec/features/issues/filtered_search/visual_tokens_spec.rb @@ -5,13 +5,14 @@ require 'spec_helper' RSpec.describe 'Visual tokens', :js do include FilteredSearchHelpers - let!(:project) { create(:project) } - let!(:user) { create(:user, name: 'administrator', username: 'root') } - let!(:user_rock) { create(:user, name: 'The Rock', username: 'rock') } - let!(:milestone_nine) { create(:milestone, title: '9.0', project: project) } - let!(:milestone_ten) { create(:milestone, title: '10.0', project: project) } - let!(:label) { create(:label, project: project, title: 'abc') } - let!(:cc_label) { create(:label, project: project, title: 'Community Contribution') } + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user, name: 'administrator', username: 'root') } + let_it_be(:user_rock) { create(:user, name: 'The Rock', username: 'rock') } + let_it_be(:milestone_nine) { create(:milestone, title: '9.0', project: project) } + let_it_be(:milestone_ten) { create(:milestone, title: '10.0', project: project) } + let_it_be(:label) { create(:label, project: project, title: 'abc') } + let_it_be(:cc_label) { create(:label, project: project, title: 'Community Contribution') } + let_it_be(:issue) { create(:issue, project: project) } let(:filtered_search) { find('.filtered-search') } let(:filter_author_dropdown) { find("#js-dropdown-author .filter-dropdown") } @@ -27,7 +28,6 @@ RSpec.describe 'Visual tokens', :js do project.add_user(user, :maintainer) project.add_user(user_rock, :maintainer) sign_in(user) - create(:issue, project: project) set_cookie('sidebar_collapsed', 'true') diff --git a/spec/features/projects/new_project_spec.rb b/spec/features/projects/new_project_spec.rb index 70a7465a63d..ef28979798f 100644 --- a/spec/features/projects/new_project_spec.rb +++ b/spec/features/projects/new_project_spec.rb @@ -293,6 +293,14 @@ RSpec.describe 'New project', :js do expect(git_import_instructions).to have_content 'Git repository URL' end + it 'reports error if repo URL does not end with .git' do + fill_in 'project_import_url', with: 'http://foo/bar' + # simulate blur event + find('body').click + + expect(page).to have_text('A repository URL usually ends in a .git suffix') + end + it 'keeps "Import project" tab open after form validation error' do collision_project = create(:project, name: 'test-name-collision', namespace: user.namespace) diff --git a/spec/features/search/user_searches_for_code_spec.rb b/spec/features/search/user_searches_for_code_spec.rb index f55dccce4f0..ef7af0ba138 100644 --- a/spec/features/search/user_searches_for_code_spec.rb +++ b/spec/features/search/user_searches_for_code_spec.rb @@ -22,6 +22,7 @@ RSpec.describe 'User searches for code' do expect(page).to have_selector('.file-content .code') expect(page).to have_selector("span.line[lang='javascript']") expect(page).to have_link('application.js', href: %r{master/files/js/application.js}) + expect(page).to have_button('Copy file path') end context 'when on a project page', :js do diff --git a/spec/frontend/boards/components/board_column_spec.js b/spec/frontend/boards/components/board_column_spec.js index 4e523d636cd..f1964daa8b2 100644 --- a/spec/frontend/boards/components/board_column_spec.js +++ b/spec/frontend/boards/components/board_column_spec.js @@ -15,6 +15,10 @@ describe('Board Column Component', () => { wrapper = null; }); + const initStore = () => { + store = createStore(); + }; + const createComponent = ({ listType = ListType.backlog, collapsed = false } = {}) => { const boardId = '1'; @@ -29,8 +33,6 @@ describe('Board Column Component', () => { listMock.assignee = {}; } - store = createStore(); - wrapper = shallowMount(BoardColumn, { store, propsData: { @@ -47,6 +49,10 @@ describe('Board Column Component', () => { const isCollapsed = () => wrapper.classes('is-collapsed'); describe('Given different list types', () => { + beforeEach(() => { + initStore(); + }); + it('is expandable when List Type is `backlog`', () => { createComponent({ listType: ListType.backlog }); @@ -79,4 +85,31 @@ describe('Board Column Component', () => { expect(wrapper.element.scrollIntoView).toHaveBeenCalled(); }); }); + + describe('on mount', () => { + beforeEach(async () => { + initStore(); + jest.spyOn(store, 'dispatch').mockImplementation(); + }); + + describe('when list is collapsed', () => { + it('does not call fetchItemsForList when', async () => { + createComponent({ collapsed: true }); + + await nextTick(); + + expect(store.dispatch).toHaveBeenCalledTimes(0); + }); + }); + + describe('when the list is not collapsed', () => { + it('calls fetchItemsForList when', async () => { + createComponent({ collapsed: false }); + + await nextTick(); + + expect(store.dispatch).toHaveBeenCalledWith('fetchItemsForList', { listId: 300 }); + }); + }); + }); }); diff --git a/spec/frontend/boards/stores/actions_spec.js b/spec/frontend/boards/stores/actions_spec.js index 5f8af7cdc60..ebed33e5b34 100644 --- a/spec/frontend/boards/stores/actions_spec.js +++ b/spec/frontend/boards/stores/actions_spec.js @@ -492,6 +492,63 @@ describe('moveList', () => { }); describe('updateList', () => { + const listId = 'gid://gitlab/List/1'; + const createState = (boardItemsByListId = {}) => ({ + fullPath: 'gitlab-org', + fullBoardId: 'gid://gitlab/Board/1', + boardType: 'group', + disabled: false, + boardLists: [{ type: 'closed' }], + issuableType: issuableTypes.issue, + boardItemsByListId, + }); + + describe('when state doesnt have list items', () => { + it('calls fetchItemsByList', async () => { + const dispatch = jest.fn(); + + jest.spyOn(gqlClient, 'mutate').mockResolvedValue({ + data: { + updateBoardList: { + errors: [], + list: { + id: listId, + }, + }, + }, + }); + + await actions.updateList({ commit: () => {}, state: createState(), dispatch }, { listId }); + + expect(dispatch.mock.calls).toEqual([['fetchItemsForList', { listId }]]); + }); + }); + + describe('when state has list items', () => { + it('doesnt call fetchItemsByList', async () => { + const commit = jest.fn(); + const dispatch = jest.fn(); + + jest.spyOn(gqlClient, 'mutate').mockResolvedValue({ + data: { + updateBoardList: { + errors: [], + list: { + id: listId, + }, + }, + }, + }); + + await actions.updateList( + { commit, state: createState({ [listId]: [] }), dispatch }, + { listId }, + ); + + expect(dispatch.mock.calls).toEqual([]); + }); + }); + it('should commit UPDATE_LIST_FAILURE mutation when API returns an error', (done) => { jest.spyOn(gqlClient, 'mutate').mockResolvedValue({ data: { @@ -502,19 +559,10 @@ describe('updateList', () => { }, }); - const state = { - fullPath: 'gitlab-org', - fullBoardId: 'gid://gitlab/Board/1', - boardType: 'group', - disabled: false, - boardLists: [{ type: 'closed' }], - issuableType: issuableTypes.issue, - }; - testAction( actions.updateList, { listId: 'gid://gitlab/List/1', position: 1 }, - state, + createState(), [{ type: types.UPDATE_LIST_FAILURE }], [], done, diff --git a/spec/frontend/content_editor/extensions/image_spec.js b/spec/frontend/content_editor/extensions/image_spec.js new file mode 100644 index 00000000000..922966b813a --- /dev/null +++ b/spec/frontend/content_editor/extensions/image_spec.js @@ -0,0 +1,193 @@ +import axios from 'axios'; +import MockAdapter from 'axios-mock-adapter'; +import { once } from 'lodash'; +import waitForPromises from 'helpers/wait_for_promises'; +import * as Image from '~/content_editor/extensions/image'; +import httpStatus from '~/lib/utils/http_status'; +import { loadMarkdownApiResult } from '../markdown_processing_examples'; +import { createTestEditor, createDocBuilder } from '../test_utils'; + +describe('content_editor/extensions/image', () => { + let tiptapEditor; + let eq; + let doc; + let p; + let image; + let renderMarkdown; + let mock; + const uploadsPath = '/uploads/'; + const validFile = new File(['foo'], 'foo.png', { type: 'image/png' }); + const invalidFile = new File(['foo'], 'bar.html', { type: 'text/html' }); + + beforeEach(() => { + renderMarkdown = jest + .fn() + .mockResolvedValue(loadMarkdownApiResult('project_wiki_attachment_image').body); + + const { tiptapExtension } = Image.configure({ renderMarkdown, uploadsPath }); + + tiptapEditor = createTestEditor({ extensions: [tiptapExtension] }); + + ({ + builders: { doc, p, image }, + eq, + } = createDocBuilder({ + tiptapEditor, + names: { image: { nodeType: tiptapExtension.name } }, + })); + + mock = new MockAdapter(axios); + }); + + afterEach(() => { + mock.reset(); + }); + + it.each` + file | valid | description + ${validFile} | ${true} | ${'handles paste event when mime type is valid'} + ${invalidFile} | ${false} | ${'does not handle paste event when mime type is invalid'} + `('$description', ({ file, valid }) => { + const pasteEvent = Object.assign(new Event('paste'), { + clipboardData: { + files: [file], + }, + }); + let handled; + + tiptapEditor.view.someProp('handlePaste', (eventHandler) => { + handled = eventHandler(tiptapEditor.view, pasteEvent); + }); + + expect(handled).toBe(valid); + }); + + it.each` + file | valid | description + ${validFile} | ${true} | ${'handles drop event when mime type is valid'} + ${invalidFile} | ${false} | ${'does not handle drop event when mime type is invalid'} + `('$description', ({ file, valid }) => { + const dropEvent = Object.assign(new Event('drop'), { + dataTransfer: { + files: [file], + }, + }); + let handled; + + tiptapEditor.view.someProp('handleDrop', (eventHandler) => { + handled = eventHandler(tiptapEditor.view, dropEvent); + }); + + expect(handled).toBe(valid); + }); + + it('handles paste event when mime type is correct', () => { + const pasteEvent = Object.assign(new Event('paste'), { + clipboardData: { + files: [new File(['foo'], 'foo.png', { type: 'image/png' })], + }, + }); + const handled = tiptapEditor.view.someProp('handlePaste', (eventHandler) => { + return eventHandler(tiptapEditor.view, pasteEvent); + }); + + expect(handled).toBe(true); + }); + + describe('uploadImage command', () => { + describe('when file has correct mime type', () => { + let initialDoc; + const base64EncodedFile = 'data:image/png;base64,Zm9v'; + + beforeEach(() => { + initialDoc = doc(p('')); + tiptapEditor.commands.setContent(initialDoc.toJSON()); + }); + + describe('when uploading image succeeds', () => { + const successResponse = { + link: { + markdown: '[image](/uploads/25265/image.png)', + }, + }; + + beforeEach(() => { + mock.onPost().reply(httpStatus.OK, successResponse); + }); + + it('inserts an image with src set to the encoded image file and uploading true', (done) => { + const expectedDoc = doc(p(image({ uploading: true, src: base64EncodedFile }))); + + tiptapEditor.on( + 'update', + once(() => { + expect(eq(tiptapEditor.state.doc, expectedDoc)).toBe(true); + done(); + }), + ); + + tiptapEditor.commands.uploadImage({ file: validFile }); + }); + + it('updates the inserted image with canonicalSrc when upload is successful', async () => { + const expectedDoc = doc( + p( + image({ + canonicalSrc: 'test-file.png', + src: base64EncodedFile, + alt: 'test file', + uploading: false, + }), + ), + ); + + tiptapEditor.commands.uploadImage({ file: validFile }); + + await waitForPromises(); + + expect(eq(tiptapEditor.state.doc, expectedDoc)).toBe(true); + }); + }); + + describe('when uploading image request fails', () => { + beforeEach(() => { + mock.onPost().reply(httpStatus.INTERNAL_SERVER_ERROR); + }); + + it('resets the doc to orginal state', async () => { + const expectedDoc = doc(p('')); + + tiptapEditor.commands.uploadImage({ file: validFile }); + + await waitForPromises(); + + expect(eq(tiptapEditor.state.doc, expectedDoc)).toBe(true); + }); + + it('emits an error event that includes an error message', (done) => { + tiptapEditor.commands.uploadImage({ file: validFile }); + + tiptapEditor.on('error', (message) => { + expect(message).toBe('An error occurred while uploading the image. Please try again.'); + done(); + }); + }); + }); + }); + + describe('when file does not have correct mime type', () => { + let initialDoc; + + beforeEach(() => { + initialDoc = doc(p('')); + tiptapEditor.commands.setContent(initialDoc.toJSON()); + }); + + it('does not start the upload image process', () => { + tiptapEditor.commands.uploadImage({ file: invalidFile }); + + expect(eq(tiptapEditor.state.doc, initialDoc)).toBe(true); + }); + }); + }); +}); diff --git a/spec/frontend/design_management/components/design_notes/__snapshots__/design_note_spec.js.snap b/spec/frontend/design_management/components/design_notes/__snapshots__/design_note_spec.js.snap index 084a7e5d712..4ecf82a4714 100644 --- a/spec/frontend/design_management/components/design_notes/__snapshots__/design_note_spec.js.snap +++ b/spec/frontend/design_management/components/design_notes/__snapshots__/design_note_spec.js.snap @@ -6,7 +6,7 @@ exports[`Design note component should match the snapshot 1`] = ` id="note_123" > <user-avatar-link-stub - imgalt="" + imgalt="foo-bar" imgcssclasses="" imgsize="40" imgsrc="" @@ -22,7 +22,8 @@ exports[`Design note component should match the snapshot 1`] = ` <div> <gl-link-stub class="js-user-link" - data-user-id="author-id" + data-user-id="1" + data-username="foo-bar" > <span class="note-header-author-name gl-font-weight-bold" @@ -35,7 +36,7 @@ exports[`Design note component should match the snapshot 1`] = ` <span class="note-headline-light" > - @ + @foo-bar </span> </gl-link-stub> diff --git a/spec/frontend/design_management/components/design_notes/design_note_spec.js b/spec/frontend/design_management/components/design_notes/design_note_spec.js index 1cd556eabb4..3f5f5bcdfa7 100644 --- a/spec/frontend/design_management/components/design_notes/design_note_spec.js +++ b/spec/frontend/design_management/components/design_notes/design_note_spec.js @@ -9,7 +9,8 @@ const scrollIntoViewMock = jest.fn(); const note = { id: 'gid://gitlab/DiffNote/123', author: { - id: 'author-id', + id: 'gid://gitlab/User/1', + username: 'foo-bar', }, body: 'test', userPermissions: { diff --git a/spec/frontend/fixtures/api_markdown.yml b/spec/frontend/fixtures/api_markdown.yml index e660a99a2e7..906d8492c48 100644 --- a/spec/frontend/fixtures/api_markdown.yml +++ b/spec/frontend/fixtures/api_markdown.yml @@ -23,6 +23,15 @@ - name: attachment_link context: group markdown: '[test-file](/uploads/aa45a38ec2cfe97433281b10bbff042c/test-file.zip)' +- name: attachment_image + context: project_wiki + markdown: '' +- name: attachment_image + context: project + markdown: '' +- name: attachment_image + context: group + markdown: '' - name: code_block markdown: |- ```javascript diff --git a/spec/frontend/repository/components/blob_content_viewer_spec.js b/spec/frontend/repository/components/blob_content_viewer_spec.js index 6041b963e38..c9fe6cafba0 100644 --- a/spec/frontend/repository/components/blob_content_viewer_spec.js +++ b/spec/frontend/repository/components/blob_content_viewer_spec.js @@ -12,6 +12,7 @@ import BlobButtonGroup from '~/repository/components/blob_button_group.vue'; import BlobContentViewer from '~/repository/components/blob_content_viewer.vue'; import BlobEdit from '~/repository/components/blob_edit.vue'; import { loadViewer, viewerProps } from '~/repository/components/blob_viewers'; +import DownloadViewer from '~/repository/components/blob_viewers/download_viewer.vue'; import TextViewer from '~/repository/components/blob_viewers/text_viewer.vue'; import blobInfoQuery from '~/repository/queries/blob_info.query.graphql'; @@ -124,6 +125,7 @@ describe('Blob content viewer component', () => { const findBlobContent = () => wrapper.findComponent(BlobContent); const findBlobButtonGroup = () => wrapper.findComponent(BlobButtonGroup); const findTextViewer = () => wrapper.findComponent(TextViewer); + const findDownloadViewer = () => wrapper.findComponent(DownloadViewer); afterEach(() => { wrapper.destroy(); @@ -225,6 +227,7 @@ describe('Blob content viewer component', () => { describe('Blob viewer', () => { beforeEach(() => { loadViewer.mockClear(); + viewerProps.mockClear(); }); it('does not render a BlobContent component if a Blob viewer is available', () => { @@ -242,6 +245,31 @@ describe('Blob content viewer component', () => { expect(findTextViewer().exists()).toBe(true); }); + + it('renders a DownloadViewer for download files', async () => { + loadViewer.mockReturnValue(DownloadViewer); + viewerProps.mockReturnValue({ + filePath: '/some/file/path', + fileName: 'test.js', + fileSize: 100, + }); + + const downloadSimpleMockData = { + ...simpleMockData, + fileType: null, + simpleViewer: { + ...simpleMockData.simpleViewer, + fileType: 'download', + }, + }; + + factory({ mockData: { blobInfo: downloadSimpleMockData } }); + + await nextTick(); + + expect(loadViewer).toHaveBeenCalledWith('download'); + expect(findDownloadViewer().exists()).toBe(true); + }); }); describe('BlobHeader action slot', () => { diff --git a/spec/frontend/repository/components/blob_viewers/download_viewer_spec.js b/spec/frontend/repository/components/blob_viewers/download_viewer_spec.js new file mode 100644 index 00000000000..c71b2b3c55c --- /dev/null +++ b/spec/frontend/repository/components/blob_viewers/download_viewer_spec.js @@ -0,0 +1,70 @@ +import { GlLink, GlIcon } from '@gitlab/ui'; +import { shallowMount } from '@vue/test-utils'; +import { numberToHumanSize } from '~/lib/utils/number_utils'; +import DownloadViewer from '~/repository/components/blob_viewers/download_viewer.vue'; + +describe('Text Viewer', () => { + let wrapper; + + const DEFAULT_PROPS = { + fileName: 'file_name.js', + filePath: '/some/file/path', + fileSize: 2269674, + }; + + const createComponent = (props = {}) => { + wrapper = shallowMount(DownloadViewer, { + propsData: { + ...DEFAULT_PROPS, + ...props, + }, + }); + }; + + it('renders component', () => { + createComponent(); + + const { fileName, filePath, fileSize } = DEFAULT_PROPS; + expect(wrapper.props()).toMatchObject({ + fileName, + filePath, + fileSize, + }); + }); + + it('renders download human readable file size text', () => { + createComponent(); + + const downloadText = `Download (${numberToHumanSize(DEFAULT_PROPS.fileSize)})`; + expect(wrapper.text()).toBe(downloadText); + }); + + it('renders download text', () => { + createComponent({ + fileSize: 0, + }); + + expect(wrapper.text()).toBe('Download'); + }); + + it('renders download link', () => { + createComponent(); + const { filePath, fileName } = DEFAULT_PROPS; + + expect(wrapper.findComponent(GlLink).attributes()).toMatchObject({ + rel: 'nofollow', + target: '_blank', + href: filePath, + download: fileName, + }); + }); + + it('renders download icon', () => { + createComponent(); + + expect(wrapper.findComponent(GlIcon).props()).toMatchObject({ + name: 'download', + size: 16, + }); + }); +}); diff --git a/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb b/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb index 885eef5723e..f9dca371398 100644 --- a/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb +++ b/spec/lib/gitlab/database/partitioning/monthly_strategy_spec.rb @@ -71,6 +71,18 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do model.create!(created_at: Date.parse('2020-06-15')) end + context 'when pruning partitions before June 2020' do + subject { described_class.new(model, partitioning_key, retain_for: 1.month).missing_partitions } + + it 'does not include the missing partition from May 2020 because it would be dropped' do + expect(subject).not_to include(Gitlab::Database::Partitioning::TimePartition.new(model.table_name, '2020-05-01', '2020-06-01')) + end + + it 'detects the missing partition for 1 month ago (July 2020)' do + expect(subject).to include(Gitlab::Database::Partitioning::TimePartition.new(model.table_name, '2020-07-01', '2020-08-01')) + end + end + it 'detects the gap and the missing partition in May 2020' do expect(subject).to include(Gitlab::Database::Partitioning::TimePartition.new(model.table_name, '2020-05-01', '2020-06-01')) end @@ -108,6 +120,19 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do SQL end + context 'when pruning partitions before June 2020' do + subject { described_class.new(model, partitioning_key, retain_for: 1.month).missing_partitions } + + it 'detects exactly the set of partitions from June 2020 to March 2021' do + months = %w[2020-07-01 2020-08-01 2020-09-01 2020-10-01 2020-11-01 2020-12-01 2021-01-01 2021-02-01 2021-03-01] + expected = months[..-2].zip(months.drop(1)).map do |(from, to)| + Gitlab::Database::Partitioning::TimePartition.new(model.table_name, from, to) + end + + expect(subject).to match_array(expected) + end + end + it 'detects the missing catch-all partition at the beginning' do expect(subject).to include(Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-08-01')) end @@ -150,4 +175,100 @@ RSpec.describe Gitlab::Database::Partitioning::MonthlyStrategy do end end end + + describe '#extra_partitions' do + let(:model) do + Class.new(ActiveRecord::Base) do + self.table_name = 'partitioned_test' + self.primary_key = :id + end + end + + let(:partitioning_key) { :created_at } + let(:table_name) { :partitioned_test } + + around do |example| + travel_to(Date.parse('2020-08-22')) { example.run } + end + + describe 'with existing partitions' do + before do + ActiveRecord::Base.connection.execute(<<~SQL) + CREATE TABLE #{table_name} + (id serial not null, created_at timestamptz not null, PRIMARY KEY (id, created_at)) + PARTITION BY RANGE (created_at); + + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_000000 + PARTITION OF #{table_name} + FOR VALUES FROM (MINVALUE) TO ('2020-05-01'); + + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_202005 + PARTITION OF #{table_name} + FOR VALUES FROM ('2020-05-01') TO ('2020-06-01'); + + CREATE TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.partitioned_test_202006 + PARTITION OF #{table_name} + FOR VALUES FROM ('2020-06-01') TO ('2020-07-01') + SQL + end + + context 'without a time retention policy' do + subject { described_class.new(model, partitioning_key).extra_partitions } + + it 'has no extra partitions to prune' do + expect(subject).to eq([]) + end + end + + context 'with a time retention policy that excludes no partitions' do + subject { described_class.new(model, partitioning_key, retain_for: 4.months).extra_partitions } + + it 'has no extra partitions to prune' do + expect(subject).to eq([]) + end + end + + context 'with a time retention policy of 3 months' do + subject { described_class.new(model, partitioning_key, retain_for: 3.months).extra_partitions } + + it 'prunes the unbounded partition ending 2020-05-01' do + min_value_to_may = Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-05-01', + partition_name: 'partitioned_test_000000') + + expect(subject).to contain_exactly(min_value_to_may) + end + + context 'when the feature flag is toggled off' do + before do + stub_feature_flags(partition_pruning_dry_run: false) + end + + it 'is empty' do + expect(subject).to eq([]) + end + end + end + + context 'with a time retention policy of 2 months' do + subject { described_class.new(model, partitioning_key, retain_for: 2.months).extra_partitions } + + it 'prunes the unbounded partition and the partition for May-June' do + expect(subject).to contain_exactly( + Gitlab::Database::Partitioning::TimePartition.new(model.table_name, nil, '2020-05-01', partition_name: 'partitioned_test_000000'), + Gitlab::Database::Partitioning::TimePartition.new(model.table_name, '2020-05-01', '2020-06-01', partition_name: 'partitioned_test_202005') + ) + end + + context 'when the feature flag is toggled off' do + before do + stub_feature_flags(partition_pruning_dry_run: false) + end + + it 'is empty' do + expect(subject).to eq([]) + end + end + end + end + end end diff --git a/spec/lib/gitlab/database/partitioning/partition_creator_spec.rb b/spec/lib/gitlab/database/partitioning/partition_creator_spec.rb deleted file mode 100644 index ec89f2ed61c..00000000000 --- a/spec/lib/gitlab/database/partitioning/partition_creator_spec.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::Partitioning::PartitionCreator do - include Database::PartitioningHelpers - include ExclusiveLeaseHelpers - - describe '.register' do - let(:model) { double(partitioning_strategy: nil) } - - it 'remembers registered models' do - expect { described_class.register(model) }.to change { described_class.models }.to include(model) - end - end - - describe '#create_partitions (mocked)' do - subject { described_class.new(models).create_partitions } - - let(:models) { [model] } - let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table) } - let(:partitioning_strategy) { double(missing_partitions: partitions) } - let(:table) { "some_table" } - - before do - allow(ActiveRecord::Base.connection).to receive(:table_exists?).and_call_original - allow(ActiveRecord::Base.connection).to receive(:table_exists?).with(table).and_return(true) - allow(ActiveRecord::Base.connection).to receive(:execute).and_call_original - - stub_exclusive_lease(described_class::LEASE_KEY % table, timeout: described_class::LEASE_TIMEOUT) - end - - let(:partitions) do - [ - instance_double(Gitlab::Database::Partitioning::TimePartition, table: 'bar', partition_name: 'foo', to_sql: "SELECT 1"), - instance_double(Gitlab::Database::Partitioning::TimePartition, table: 'bar', partition_name: 'foo2', to_sql: "SELECT 2") - ] - end - - it 'creates the partition' do - expect(ActiveRecord::Base.connection).to receive(:execute).with(partitions.first.to_sql) - expect(ActiveRecord::Base.connection).to receive(:execute).with(partitions.second.to_sql) - - subject - end - - context 'error handling with 2 models' do - let(:models) do - [ - double(partitioning_strategy: strategy1, table_name: table), - double(partitioning_strategy: strategy2, table_name: table) - ] - end - - let(:strategy1) { double('strategy1', missing_partitions: nil) } - let(:strategy2) { double('strategy2', missing_partitions: partitions) } - - it 'still creates partitions for the second table' do - expect(strategy1).to receive(:missing_partitions).and_raise('this should never happen (tm)') - expect(ActiveRecord::Base.connection).to receive(:execute).with(partitions.first.to_sql) - expect(ActiveRecord::Base.connection).to receive(:execute).with(partitions.second.to_sql) - - subject - end - end - end - - describe '#create_partitions' do - subject { described_class.new([my_model]).create_partitions } - - let(:connection) { ActiveRecord::Base.connection } - let(:my_model) do - Class.new(ApplicationRecord) do - include PartitionedTable - - self.table_name = 'my_model_example_table' - - partitioned_by :created_at, strategy: :monthly - end - end - - before do - connection.execute(<<~SQL) - CREATE TABLE my_model_example_table - (id serial not null, created_at timestamptz not null, primary key (id, created_at)) - PARTITION BY RANGE (created_at); - SQL - end - - it 'creates partitions' do - expect { subject }.to change { find_partitions(my_model.table_name, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA).size }.from(0) - - subject - end - end -end diff --git a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb new file mode 100644 index 00000000000..903a41d6dd2 --- /dev/null +++ b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Partitioning::PartitionManager do + include Database::PartitioningHelpers + include Database::TableSchemaHelpers + include ExclusiveLeaseHelpers + + describe '.register' do + let(:model) { double(partitioning_strategy: nil) } + + it 'remembers registered models' do + expect { described_class.register(model) }.to change { described_class.models }.to include(model) + end + end + + context 'creating partitions (mocked)' do + subject(:sync_partitions) { described_class.new(models).sync_partitions } + + let(:models) { [model] } + let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table) } + let(:partitioning_strategy) { double(missing_partitions: partitions, extra_partitions: []) } + let(:table) { "some_table" } + + before do + allow(ActiveRecord::Base.connection).to receive(:table_exists?).and_call_original + allow(ActiveRecord::Base.connection).to receive(:table_exists?).with(table).and_return(true) + allow(ActiveRecord::Base.connection).to receive(:execute).and_call_original + + stub_exclusive_lease(described_class::MANAGEMENT_LEASE_KEY % table, timeout: described_class::LEASE_TIMEOUT) + end + + let(:partitions) do + [ + instance_double(Gitlab::Database::Partitioning::TimePartition, table: 'bar', partition_name: 'foo', to_sql: "SELECT 1"), + instance_double(Gitlab::Database::Partitioning::TimePartition, table: 'bar', partition_name: 'foo2', to_sql: "SELECT 2") + ] + end + + it 'creates the partition' do + expect(ActiveRecord::Base.connection).to receive(:execute).with(partitions.first.to_sql) + expect(ActiveRecord::Base.connection).to receive(:execute).with(partitions.second.to_sql) + + sync_partitions + end + + context 'error handling with 2 models' do + let(:models) do + [ + double(partitioning_strategy: strategy1, table_name: table), + double(partitioning_strategy: strategy2, table_name: table) + ] + end + + let(:strategy1) { double('strategy1', missing_partitions: nil, extra_partitions: []) } + let(:strategy2) { double('strategy2', missing_partitions: partitions, extra_partitions: []) } + + it 'still creates partitions for the second table' do + expect(strategy1).to receive(:missing_partitions).and_raise('this should never happen (tm)') + expect(ActiveRecord::Base.connection).to receive(:execute).with(partitions.first.to_sql) + expect(ActiveRecord::Base.connection).to receive(:execute).with(partitions.second.to_sql) + + sync_partitions + end + end + end + + context 'creating partitions' do + subject(:sync_partitions) { described_class.new([my_model]).sync_partitions } + + let(:connection) { ActiveRecord::Base.connection } + let(:my_model) do + Class.new(ApplicationRecord) do + include PartitionedTable + + self.table_name = 'my_model_example_table' + + partitioned_by :created_at, strategy: :monthly + end + end + + before do + connection.execute(<<~SQL) + CREATE TABLE my_model_example_table + (id serial not null, created_at timestamptz not null, primary key (id, created_at)) + PARTITION BY RANGE (created_at); + SQL + end + + it 'creates partitions' do + expect { sync_partitions }.to change { find_partitions(my_model.table_name, schema: Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA).size }.from(0) + end + end + + context 'detaching partitions (mocked)' do + subject(:sync_partitions) { manager.sync_partitions } + + let(:manager) { described_class.new(models) } + let(:models) { [model] } + let(:model) { double(partitioning_strategy: partitioning_strategy, table_name: table)} + let(:partitioning_strategy) { double(extra_partitions: extra_partitions, missing_partitions: []) } + let(:table) { "foo" } + + before do + allow(ActiveRecord::Base.connection).to receive(:table_exists?).and_call_original + allow(ActiveRecord::Base.connection).to receive(:table_exists?).with(table).and_return(true) + + stub_exclusive_lease(described_class::MANAGEMENT_LEASE_KEY % table, timeout: described_class::LEASE_TIMEOUT) + end + + let(:extra_partitions) do + [ + instance_double(Gitlab::Database::Partitioning::TimePartition, table: table, partition_name: 'foo1'), + instance_double(Gitlab::Database::Partitioning::TimePartition, table: table, partition_name: 'foo2') + ] + end + + context 'with the partition_pruning_dry_run feature flag enabled' do + before do + stub_feature_flags(partition_pruning_dry_run: true) + end + + it 'detaches each extra partition' do + extra_partitions.each { |p| expect(manager).to receive(:detach_one_partition).with(p) } + + sync_partitions + end + + context 'error handling' do + let(:models) do + [ + double(partitioning_strategy: error_strategy, table_name: table), + model + ] + end + + let(:error_strategy) { double(extra_partitions: nil, missing_partitions: []) } + + it 'still drops partitions for the other model' do + expect(error_strategy).to receive(:extra_partitions).and_raise('injected error!') + extra_partitions.each { |p| expect(manager).to receive(:detach_one_partition).with(p) } + + sync_partitions + end + end + end + + context 'with the partition_pruning_dry_run feature flag disabled' do + before do + stub_feature_flags(partition_pruning_dry_run: false) + end + + it 'returns immediately' do + expect(manager).not_to receive(:detach) + + sync_partitions + end + end + end +end diff --git a/spec/lib/gitlab/database/postgres_index_spec.rb b/spec/lib/gitlab/database/postgres_index_spec.rb index 0762d2a94ae..da047632985 100644 --- a/spec/lib/gitlab/database/postgres_index_spec.rb +++ b/spec/lib/gitlab/database/postgres_index_spec.rb @@ -107,6 +107,24 @@ RSpec.describe Gitlab::Database::PostgresIndex do end end + describe '#reset' do + subject { index.reset } + + let(:index) { described_class.by_identifier(identifier) } + + it 'calls #reload' do + expect(index).to receive(:reload).once.and_call_original + + subject + end + + it 'resets the bloat estimation' do + expect(index).to receive(:clear_memoization).with(:bloat_size).and_call_original + + subject + end + end + describe '#unique?' do it 'returns true for a unique index' do expect(find('public.bar_key')).to be_unique diff --git a/spec/lib/gitlab/github_import/object_counter_spec.rb b/spec/lib/gitlab/github_import/object_counter_spec.rb index be76159fdb3..668c11667b5 100644 --- a/spec/lib/gitlab/github_import/object_counter_spec.rb +++ b/spec/lib/gitlab/github_import/object_counter_spec.rb @@ -11,6 +11,18 @@ RSpec.describe Gitlab::GithubImport::ObjectCounter, :clean_gitlab_redis_cache do end it 'increments the counter and saves the key to be listed in the summary later' do + expect(Gitlab::Metrics) + .to receive(:counter) + .twice + .with(:github_importer_fetched_issue, 'The number of fetched Github Issue') + .and_return(double(increment: true)) + + expect(Gitlab::Metrics) + .to receive(:counter) + .twice + .with(:github_importer_imported_issue, 'The number of imported Github Issue') + .and_return(double(increment: true)) + described_class.increment(project, :issue, :fetched) described_class.increment(project, :issue, :fetched) described_class.increment(project, :issue, :imported) diff --git a/spec/lib/gitlab/sidekiq_config_spec.rb b/spec/lib/gitlab/sidekiq_config_spec.rb index 8f12af9edae..d2a53185acd 100644 --- a/spec/lib/gitlab/sidekiq_config_spec.rb +++ b/spec/lib/gitlab/sidekiq_config_spec.rb @@ -134,7 +134,7 @@ RSpec.describe Gitlab::SidekiqConfig do .to receive(:global).and_return(::Gitlab::SidekiqConfig::WorkerRouter.new(test_routes)) expect(described_class.worker_queue_mappings).to include('MergeWorker' => 'default', - 'BuildFinishedWorker' => 'default', + 'Ci::BuildFinishedWorker' => 'default', 'BackgroundMigrationWorker' => 'background_migration', 'AdminEmailWorker' => 'cronjob:admin_email') end @@ -155,7 +155,7 @@ RSpec.describe Gitlab::SidekiqConfig do mappings = described_class.current_worker_queue_mappings expect(mappings).to include('MergeWorker' => 'default', - 'BuildFinishedWorker' => 'default', + 'Ci::BuildFinishedWorker' => 'default', 'BackgroundMigrationWorker' => 'background_migration') expect(mappings).not_to include('AdminEmailWorker' => 'cronjob:admin_email') diff --git a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb index 8a9767f9012..3ec8d404bf0 100644 --- a/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb +++ b/spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb @@ -18,7 +18,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do it 'initializes sidekiq_jobs_completion_seconds for the workers in the current Sidekiq process' do allow(Gitlab::SidekiqConfig) .to receive(:current_worker_queue_mappings) - .and_return('MergeWorker' => 'merge', 'BuildFinishedWorker' => 'default') + .and_return('MergeWorker' => 'merge', 'Ci::BuildFinishedWorker' => 'default') expect(completion_seconds_metric) .to receive(:get).with(queue: 'merge', @@ -40,7 +40,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do expect(completion_seconds_metric) .to receive(:get).with(queue: 'default', - worker: 'BuildFinishedWorker', + worker: 'Ci::BuildFinishedWorker', urgency: 'high', external_dependencies: 'no', feature_category: 'continuous_integration', @@ -49,7 +49,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do expect(completion_seconds_metric) .to receive(:get).with(queue: 'default', - worker: 'BuildFinishedWorker', + worker: 'Ci::BuildFinishedWorker', urgency: 'high', external_dependencies: 'no', feature_category: 'continuous_integration', @@ -73,7 +73,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do it 'does not initialize sidekiq_jobs_completion_seconds' do allow(Gitlab::SidekiqConfig) .to receive(:current_worker_queue_mappings) - .and_return('MergeWorker' => 'merge', 'BuildFinishedWorker' => 'default') + .and_return('MergeWorker' => 'merge', 'Ci::BuildFinishedWorker' => 'default') expect(completion_seconds_metric).not_to receive(:get) @@ -118,7 +118,7 @@ RSpec.describe Gitlab::SidekiqMiddleware::ServerMetrics do allow(Gitlab::SidekiqConfig) .to receive(:current_worker_queue_mappings) - .and_return('MergeWorker' => 'merge', 'BuildFinishedWorker' => 'default') + .and_return('MergeWorker' => 'merge', 'Ci::BuildFinishedWorker' => 'default') allow(completion_seconds_metric).to receive(:get) do |labels| expect { label_validator.validate(labels) }.not_to raise_error diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb new file mode 100644 index 00000000000..8f52d550e5c --- /dev/null +++ b/spec/lib/gitlab/usage/metrics/instrumentations/collected_data_categories_metric_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Usage::Metrics::Instrumentations::CollectedDataCategoriesMetric do + it_behaves_like 'a correct instrumented metric value', {} do + let(:expected_value) { %w[Standard Subscription Operational Optional] } + + before do + allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance| + expect(instance).to receive(:execute).and_return(expected_value) + end + end + end +end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 145ca750ad6..d84974e562a 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -1078,6 +1078,16 @@ RSpec.describe Gitlab::UsageData, :aggregate_failures do it 'gathers gitaly apdex', :aggregate_failures do expect(subject[:settings][:gitaly_apdex]).to be_within(0.001).of(0.95) end + + it 'reports collected data categories' do + expected_value = %w[Standard Subscription Operational Optional] + + allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |instance| + expect(instance).to receive(:execute).and_return(expected_value) + end + + expect(subject[:settings][:collected_data_categories]).to eq(expected_value) + end end end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 1fbfae891af..0c344270e0b 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -39,6 +39,34 @@ RSpec.describe Ci::Build do it { is_expected.to delegate_method(:merge_request_ref?).to(:pipeline) } it { is_expected.to delegate_method(:legacy_detached_merge_request_pipeline?).to(:pipeline) } + shared_examples 'calling proper BuildFinishedWorker' do + context 'when ci_build_finished_worker_namespace_changed feature flag enabled' do + before do + stub_feature_flags(ci_build_finished_worker_namespace_changed: build.project) + end + + it 'calls Ci::BuildFinishedWorker' do + expect(Ci::BuildFinishedWorker).to receive(:perform_async) + expect(::BuildFinishedWorker).not_to receive(:perform_async) + + subject + end + end + + context 'when ci_build_finished_worker_namespace_changed feature flag disabled' do + before do + stub_feature_flags(ci_build_finished_worker_namespace_changed: false) + end + + it 'calls ::BuildFinishedWorker' do + expect(::BuildFinishedWorker).to receive(:perform_async) + expect(Ci::BuildFinishedWorker).not_to receive(:perform_async) + + subject + end + end + end + describe 'associations' do it 'has a bidirectional relationship with projects' do expect(described_class.reflect_on_association(:project).has_inverse?).to eq(:builds) @@ -1323,6 +1351,7 @@ RSpec.describe Ci::Build do end it_behaves_like 'avoid deadlock' + it_behaves_like 'calling proper BuildFinishedWorker' it 'transits deployment status to success' do subject @@ -1335,6 +1364,7 @@ RSpec.describe Ci::Build do let(:event) { :drop! } it_behaves_like 'avoid deadlock' + it_behaves_like 'calling proper BuildFinishedWorker' it 'transits deployment status to failed' do subject @@ -1359,6 +1389,7 @@ RSpec.describe Ci::Build do let(:event) { :cancel! } it_behaves_like 'avoid deadlock' + it_behaves_like 'calling proper BuildFinishedWorker' it 'transits deployment status to canceled' do subject diff --git a/spec/models/ci/pending_build_spec.rb b/spec/models/ci/pending_build_spec.rb index a0b0b92fee9..b64f3999232 100644 --- a/spec/models/ci/pending_build_spec.rb +++ b/spec/models/ci/pending_build_spec.rb @@ -29,5 +29,61 @@ RSpec.describe Ci::PendingBuild do expect(result.rows.dig(0, 0)).to eq build.id end end + + context 'when project does not have shared runner' do + it 'sets instance_runners_enabled to false' do + described_class.upsert_from_build!(build) + + expect(described_class.last.instance_runners_enabled).to be_falsey + end + end + + context 'when project has shared runner' do + let_it_be(:runner) { create(:ci_runner, :instance) } + + context 'when ci_pending_builds_maintain_shared_runners_data is enabled' do + it 'sets instance_runners_enabled to true' do + described_class.upsert_from_build!(build) + + expect(described_class.last.instance_runners_enabled).to be_truthy + end + + context 'when project is about to be deleted' do + before do + build.project.update!(pending_delete: true) + end + + it 'sets instance_runners_enabled to false' do + described_class.upsert_from_build!(build) + + expect(described_class.last.instance_runners_enabled).to be_falsey + end + end + + context 'when builds are disabled' do + before do + build.project.project_feature.update!(builds_access_level: false) + end + + it 'sets instance_runners_enabled to false' do + described_class.upsert_from_build!(build) + + expect(described_class.last.instance_runners_enabled).to be_falsey + end + end + end + + context 'when ci_pending_builds_maintain_shared_runners_data is disabled' do + before do + stub_feature_flags(ci_pending_builds_maintain_shared_runners_data: false) + end + + it 'sets instance_runners_enabled to false' do + described_class.upsert_from_build!(build) + + expect(described_class.last.instance_runners_enabled).to be_falsey + end + end + end end end diff --git a/spec/models/concerns/partitioned_table_spec.rb b/spec/models/concerns/partitioned_table_spec.rb index 3343b273ba2..c37fb81a1cf 100644 --- a/spec/models/concerns/partitioned_table_spec.rb +++ b/spec/models/concerns/partitioned_table_spec.rb @@ -14,6 +14,16 @@ RSpec.describe PartitionedTable do end end + context 'with keyword arguments passed to the strategy' do + subject { my_class.partitioned_by(key, strategy: :monthly, retain_for: 3.months) } + + it 'passes the keyword arguments to the strategy' do + expect(Gitlab::Database::Partitioning::MonthlyStrategy).to receive(:new).with(my_class, key, retain_for: 3.months).and_call_original + + subject + end + end + it 'assigns the MonthlyStrategy as the partitioning strategy' do subject @@ -27,7 +37,7 @@ RSpec.describe PartitionedTable do end it 'registers itself with the PartitionCreator' do - expect(Gitlab::Database::Partitioning::PartitionCreator).to receive(:register).with(my_class) + expect(Gitlab::Database::Partitioning::PartitionManager).to receive(:register).with(my_class) subject end diff --git a/spec/services/ci/archive_trace_service_spec.rb b/spec/services/ci/archive_trace_service_spec.rb index 3ec671d6add..12804efc28c 100644 --- a/spec/services/ci/archive_trace_service_spec.rb +++ b/spec/services/ci/archive_trace_service_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Ci::ArchiveTraceService, '#execute' do - subject { described_class.new.execute(job, worker_name: ArchiveTraceWorker.name) } + subject { described_class.new.execute(job, worker_name: Ci::ArchiveTraceWorker.name) } context 'when job is finished' do let(:job) { create(:ci_build, :success, :trace_live) } @@ -51,7 +51,7 @@ RSpec.describe Ci::ArchiveTraceService, '#execute' do it 'leaves a warning message in sidekiq log' do expect(Sidekiq.logger).to receive(:warn).with( - class: ArchiveTraceWorker.name, + class: Ci::ArchiveTraceWorker.name, message: 'The job does not have live trace but going to be archived.', job_id: job.id) @@ -68,7 +68,7 @@ RSpec.describe Ci::ArchiveTraceService, '#execute' do it 'leaves a warning message in sidekiq log' do expect(Sidekiq.logger).to receive(:warn).with( - class: ArchiveTraceWorker.name, + class: Ci::ArchiveTraceWorker.name, message: 'The job does not have archived trace after archiving.', job_id: job.id) @@ -88,7 +88,7 @@ RSpec.describe Ci::ArchiveTraceService, '#execute' do job_id: job.id).once expect(Sidekiq.logger).to receive(:warn).with( - class: ArchiveTraceWorker.name, + class: Ci::ArchiveTraceWorker.name, message: "Failed to archive trace. message: Job is not finished yet.", job_id: job.id).and_call_original diff --git a/spec/services/service_ping/permit_data_categories_service_spec.rb b/spec/services/service_ping/permit_data_categories_service_spec.rb new file mode 100644 index 00000000000..4fd5c6f9ccb --- /dev/null +++ b/spec/services/service_ping/permit_data_categories_service_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ServicePing::PermitDataCategoriesService do + using RSpec::Parameterized::TableSyntax + + describe '#execute', :without_license do + subject(:permitted_categories) { described_class.new.execute } + + context 'when usage ping setting is set to true' do + before do + allow(User).to receive(:single_user).and_return(double(:user, requires_usage_stats_consent?: false)) + stub_config_setting(usage_ping_enabled: true) + end + + it 'returns all categories' do + expect(permitted_categories).to match_array(%w[Standard Subscription Operational Optional]) + end + end + + context 'when usage ping setting is set to false' do + before do + allow(User).to receive(:single_user).and_return(double(:user, requires_usage_stats_consent?: false)) + stub_config_setting(usage_ping_enabled: false) + end + + it 'returns no categories' do + expect(permitted_categories).to match_array([]) + end + end + + context 'when User.single_user&.requires_usage_stats_consent? is required' do + before do + allow(User).to receive(:single_user).and_return(double(:user, requires_usage_stats_consent?: true)) + stub_config_setting(usage_ping_enabled: true) + end + + it 'returns no categories' do + expect(permitted_categories).to match_array([]) + end + end + end + + describe '#product_intelligence_enabled?' do + where(:usage_ping_enabled, :requires_usage_stats_consent, :expected_product_intelligence_enabled) do + # Usage ping enabled + true | false | true + true | true | false + + # Usage ping disabled + false | false | false + false | true | false + end + + with_them do + before do + allow(User).to receive(:single_user).and_return(double(:user, requires_usage_stats_consent?: requires_usage_stats_consent)) + stub_config_setting(usage_ping_enabled: usage_ping_enabled) + end + + it 'has the correct product_intelligence_enabled?' do + expect(described_class.new.product_intelligence_enabled?).to eq(expected_product_intelligence_enabled) + end + end + end +end diff --git a/spec/services/service_ping/submit_service_ping_service_spec.rb b/spec/services/service_ping/submit_service_ping_service_spec.rb index a9d9cf3b7ba..ded6194a586 100644 --- a/spec/services/service_ping/submit_service_ping_service_spec.rb +++ b/spec/services/service_ping/submit_service_ping_service_spec.rb @@ -98,6 +98,34 @@ RSpec.describe ServicePing::SubmitService do it_behaves_like 'does not run' end + context 'when product_intelligence_enabled is false' do + before do + allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |service| + allow(service).to receive(:product_intelligence_enabled?).and_return(false) + end + end + + it_behaves_like 'does not run' + end + + context 'when product_intelligence_enabled is true' do + before do + stub_usage_data_connections + + allow_next_instance_of(ServicePing::PermitDataCategoriesService) do |service| + allow(service).to receive(:product_intelligence_enabled?).and_return(true) + end + end + + it 'generates service ping' do + stub_response(body: with_dev_ops_score_params) + + expect(Gitlab::UsageData).to receive(:data).with(force_refresh: true).and_call_original + + subject.execute + end + end + context 'when usage ping is enabled' do before do stub_usage_data_connections diff --git a/spec/tooling/lib/tooling/kubernetes_client_spec.rb b/spec/tooling/lib/tooling/kubernetes_client_spec.rb index 636727401af..a7f50b0bb50 100644 --- a/spec/tooling/lib/tooling/kubernetes_client_spec.rb +++ b/spec/tooling/lib/tooling/kubernetes_client_spec.rb @@ -135,6 +135,52 @@ RSpec.describe Tooling::KubernetesClient do end end + describe '#cleanup_review_app_namespaces' do + let(:two_days_ago) { Time.now - 3600 * 24 * 2 } + let(:namespaces) { %w[review-abc-123 review-xyz-789] } + + subject { described_class.new(namespace: nil) } + + before do + allow(subject).to receive(:review_app_namespaces_created_before).with(created_before: two_days_ago).and_return(namespaces) + end + + shared_examples 'a kubectl command to delete namespaces older than given creation time' do + let(:wait) { true } + + specify do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with(["kubectl delete namespace " + + %(--now --ignore-not-found --wait=#{wait} #{namespaces.join(' ')})]) + .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: true))) + + # We're not verifying the output here, just silencing it + expect { subject.cleanup_review_app_namespaces(created_before: two_days_ago) }.to output.to_stdout + end + end + + it_behaves_like 'a kubectl command to delete namespaces older than given creation time' + + it 'raises an error if the Kubernetes command fails' do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with(["kubectl delete namespace " + + %(--now --ignore-not-found --wait=true #{namespaces.join(' ')})]) + .and_return(Gitlab::Popen::Result.new([], '', '', double(success?: false))) + + expect { subject.cleanup_review_app_namespaces(created_before: two_days_ago) }.to raise_error(described_class::CommandFailedError) + end + + context 'with no namespaces found' do + let(:namespaces) { [] } + + it 'does not call #delete_namespaces_by_exact_names' do + expect(subject).not_to receive(:delete_namespaces_by_exact_names) + + subject.cleanup_review_app_namespaces(created_before: two_days_ago) + end + end + end + describe '#raw_resource_names' do it 'calls kubectl to retrieve the resource names' do expect(Gitlab::Popen).to receive(:popen_with_detail) @@ -200,4 +246,49 @@ RSpec.describe Tooling::KubernetesClient do it_behaves_like 'a kubectl command to retrieve resource names sorted by creationTimestamp' end end + + describe '#review_app_namespaces_created_before' do + let(:three_days_ago) { Time.now - 3600 * 24 * 3 } + let(:two_days_ago) { Time.now - 3600 * 24 * 2 } + let(:namespace_created_three_days_ago) { 'namespace-created-three-days-ago' } + let(:resource_type) { 'namespace' } + let(:raw_resources) do + { + items: [ + { + apiVersion: "v1", + kind: "Namespace", + metadata: { + creationTimestamp: three_days_ago, + name: namespace_created_three_days_ago, + labels: { + tls: 'review-apps-tls' + } + } + }, + { + apiVersion: "v1", + kind: "Namespace", + metadata: { + creationTimestamp: Time.now, + name: 'another-pvc', + labels: { + tls: 'review-apps-tls' + } + } + } + ] + }.to_json + end + + specify do + expect(Gitlab::Popen).to receive(:popen_with_detail) + .with(["kubectl get namespace " \ + "-l tls=review-apps-tls " \ + "--sort-by='{.metadata.creationTimestamp}' -o json"]) + .and_return(Gitlab::Popen::Result.new([], raw_resources, '', double(success?: true))) + + expect(subject.__send__(:review_app_namespaces_created_before, created_before: two_days_ago)).to contain_exactly(namespace_created_three_days_ago) + end + end end diff --git a/spec/views/groups/settings/_transfer.html.haml_spec.rb b/spec/views/groups/settings/_transfer.html.haml_spec.rb new file mode 100644 index 00000000000..aeb70251a62 --- /dev/null +++ b/spec/views/groups/settings/_transfer.html.haml_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'groups/settings/_transfer.html.haml' do + describe 'render' do + it 'enables the Select parent group dropdown and does not show an alert for a group' do + group = build(:group) + + render 'groups/settings/transfer', group: group + + expect(rendered).to have_selector '[data-qa-selector="select_group_dropdown"]' + expect(rendered).not_to have_selector '[data-qa-selector="select_group_dropdown"][disabled]' + expect(rendered).not_to have_selector '[data-testid="group-to-transfer-has-linked-subscription-alert"]' + end + end +end diff --git a/spec/workers/build_finished_worker_spec.rb b/spec/workers/build_finished_worker_spec.rb index 3434980341b..6b7162ee886 100644 --- a/spec/workers/build_finished_worker_spec.rb +++ b/spec/workers/build_finished_worker_spec.rb @@ -10,6 +10,7 @@ RSpec.describe BuildFinishedWorker do let_it_be(:build) { create(:ci_build, :success, pipeline: create(:ci_pipeline)) } before do + stub_feature_flags(ci_build_finished_worker_namespace_changed: build.project) expect(Ci::Build).to receive(:find_by).with(id: build.id).and_return(build) end @@ -23,11 +24,23 @@ RSpec.describe BuildFinishedWorker do expect(BuildHooksWorker).to receive(:perform_async) expect(ChatNotificationWorker).not_to receive(:perform_async) - expect(ArchiveTraceWorker).to receive(:perform_in) + expect(Ci::ArchiveTraceWorker).to receive(:perform_in) subject end + context 'with ci_build_finished_worker_namespace_changed feature flag disabled' do + before do + stub_feature_flags(ci_build_finished_worker_namespace_changed: false) + end + + it 'calls deprecated worker' do + expect(ArchiveTraceWorker).to receive(:perform_in) + + subject + end + end + context 'when build is failed' do before do build.update!(status: :failed) diff --git a/spec/workers/ci/archive_trace_worker_spec.rb b/spec/workers/ci/archive_trace_worker_spec.rb new file mode 100644 index 00000000000..889e0c92042 --- /dev/null +++ b/spec/workers/ci/archive_trace_worker_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ArchiveTraceWorker do + describe '#perform' do + subject { described_class.new.perform(job&.id) } + + context 'when job is found' do + let(:job) { create(:ci_build, :trace_live) } + + it 'executes service' do + allow_next_instance_of(Ci::ArchiveTraceService) do |instance| + allow(instance).to receive(:execute).with(job, anything) + end + + subject + end + end + + context 'when job is not found' do + let(:job) { nil } + + it 'does not execute service' do + allow_next_instance_of(Ci::ArchiveTraceService) do |instance| + allow(instance).not_to receive(:execute) + end + + subject + end + end + end +end diff --git a/spec/workers/ci/build_finished_worker_spec.rb b/spec/workers/ci/build_finished_worker_spec.rb new file mode 100644 index 00000000000..374ecd8619f --- /dev/null +++ b/spec/workers/ci/build_finished_worker_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::BuildFinishedWorker do + subject { described_class.new.perform(build.id) } + + describe '#perform' do + context 'when build exists' do + let_it_be(:build) { create(:ci_build, :success, pipeline: create(:ci_pipeline)) } + + before do + stub_feature_flags(ci_build_finished_worker_namespace_changed: build.project) + expect(Ci::Build).to receive(:find_by).with(id: build.id).and_return(build) + end + + it 'calculates coverage and calls hooks', :aggregate_failures do + expect(build).to receive(:parse_trace_sections!).ordered + expect(build).to receive(:update_coverage).ordered + + expect_next_instance_of(Ci::BuildReportResultService) do |build_report_result_service| + expect(build_report_result_service).to receive(:execute).with(build) + end + + expect(BuildHooksWorker).to receive(:perform_async) + expect(ChatNotificationWorker).not_to receive(:perform_async) + expect(Ci::ArchiveTraceWorker).to receive(:perform_in) + + subject + end + + context 'with ci_build_finished_worker_namespace_changed feature flag disabled' do + before do + stub_feature_flags(ci_build_finished_worker_namespace_changed: false) + end + + it 'calls deprecated worker' do + expect(ArchiveTraceWorker).to receive(:perform_in) + + subject + end + end + + context 'when build is failed' do + before do + build.update!(status: :failed) + end + + it 'adds a todo' do + expect(::Ci::MergeRequests::AddTodoWhenBuildFailsWorker).to receive(:perform_async) + + subject + end + end + + context 'when build has a chat' do + before do + build.pipeline.update!(source: :chat) + end + + it 'schedules a ChatNotification job' do + expect(ChatNotificationWorker).to receive(:perform_async).with(build.id) + + subject + end + end + end + + context 'when build does not exist' do + it 'does not raise exception' do + expect { described_class.new.perform(non_existing_record_id) } + .not_to raise_error + end + end + end +end diff --git a/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb b/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb index 643a26490bd..4c96daea7b3 100644 --- a/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb +++ b/spec/workers/concerns/gitlab/github_import/object_importer_spec.rb @@ -11,14 +11,6 @@ RSpec.describe Gitlab::GithubImport::ObjectImporter do include(Gitlab::GithubImport::ObjectImporter) - def counter_name - :dummy_counter - end - - def counter_description - 'This is a counter' - end - def object_type :dummy end @@ -68,10 +60,6 @@ RSpec.describe Gitlab::GithubImport::ObjectImporter do expect(importer_instance) .to receive(:execute) - expect(worker.counter) - .to receive(:increment) - .and_call_original - expect_next_instance_of(Gitlab::Import::Logger) do |logger| expect(logger) .to receive(:info) @@ -185,18 +173,4 @@ RSpec.describe Gitlab::GithubImport::ObjectImporter do .to raise_error(KeyError, 'key not found: :github_id') end end - - describe '#counter' do - it 'returns a Prometheus counter' do - expect(worker) - .to receive(:counter_name) - .and_call_original - - expect(worker) - .to receive(:counter_description) - .and_call_original - - worker.counter - end - end end diff --git a/spec/workers/database/partition_management_worker_spec.rb b/spec/workers/database/partition_management_worker_spec.rb new file mode 100644 index 00000000000..01b7f209b2d --- /dev/null +++ b/spec/workers/database/partition_management_worker_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Database::PartitionManagementWorker do + describe '#perform' do + subject { described_class.new.perform } + + let(:manager) { instance_double('PartitionManager', sync_partitions: nil) } + let(:monitoring) { instance_double('PartitionMonitoring', report_metrics: nil) } + + before do + allow(Gitlab::Database::Partitioning::PartitionManager).to receive(:new).and_return(manager) + allow(Gitlab::Database::Partitioning::PartitionMonitoring).to receive(:new).and_return(monitoring) + end + + it 'delegates to PartitionManager' do + expect(manager).to receive(:sync_partitions) + + subject + end + + it 'reports partition metrics' do + expect(monitoring).to receive(:report_metrics) + + subject + end + end +end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index 5ab5c60065a..8c1667e5b4d 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -148,7 +148,9 @@ RSpec.describe 'Every Sidekiq worker' do 'Chaos::LeakMemWorker' => 3, 'Chaos::SleepWorker' => 3, 'ChatNotificationWorker' => false, + 'Ci::ArchiveTraceWorker' => 3, 'Ci::BatchResetMinutesWorker' => 10, + 'Ci::BuildFinishedWorker' => 3, 'Ci::BuildPrepareWorker' => 3, 'Ci::BuildScheduleWorker' => 3, 'Ci::BuildTraceChunkFlushWorker' => 3, diff --git a/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb b/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb index 6476d82eb85..34073d0ea39 100644 --- a/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb @@ -34,7 +34,7 @@ RSpec.describe Gitlab::GithubImport::ImportDiffNoteWorker do expect(importer) .to receive(:execute) - expect(worker.counter) + expect(Gitlab::GithubImport::ObjectCounter) .to receive(:increment) .and_call_original diff --git a/spec/workers/gitlab/github_import/import_issue_worker_spec.rb b/spec/workers/gitlab/github_import/import_issue_worker_spec.rb index 9f5bd1d9e5e..dc0338eccad 100644 --- a/spec/workers/gitlab/github_import/import_issue_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_issue_worker_spec.rb @@ -37,7 +37,7 @@ RSpec.describe Gitlab::GithubImport::ImportIssueWorker do expect(importer) .to receive(:execute) - expect(worker.counter) + expect(Gitlab::GithubImport::ObjectCounter) .to receive(:increment) .and_call_original diff --git a/spec/workers/gitlab/github_import/import_note_worker_spec.rb b/spec/workers/gitlab/github_import/import_note_worker_spec.rb index 94bc8e26e4a..bc254e6246d 100644 --- a/spec/workers/gitlab/github_import/import_note_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_note_worker_spec.rb @@ -32,7 +32,7 @@ RSpec.describe Gitlab::GithubImport::ImportNoteWorker do expect(importer) .to receive(:execute) - expect(worker.counter) + expect(Gitlab::GithubImport::ObjectCounter) .to receive(:increment) .and_call_original diff --git a/spec/workers/gitlab/github_import/import_pull_request_merged_by_worker_spec.rb b/spec/workers/gitlab/github_import/import_pull_request_merged_by_worker_spec.rb index c799c676300..728b4c6b440 100644 --- a/spec/workers/gitlab/github_import/import_pull_request_merged_by_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_pull_request_merged_by_worker_spec.rb @@ -12,12 +12,4 @@ RSpec.describe Gitlab::GithubImport::ImportPullRequestMergedByWorker do describe '#importer_class' do it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::PullRequestMergedByImporter) } end - - describe '#counter_name' do - it { expect(subject.counter_name).to eq(:github_importer_imported_pull_requests_merged_by) } - end - - describe '#counter_description' do - it { expect(subject.counter_description).to eq('The number of imported GitHub pull requests merged by') } - end end diff --git a/spec/workers/gitlab/github_import/import_pull_request_review_worker_spec.rb b/spec/workers/gitlab/github_import/import_pull_request_review_worker_spec.rb index cd14d6631d5..0607add52cd 100644 --- a/spec/workers/gitlab/github_import/import_pull_request_review_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_pull_request_review_worker_spec.rb @@ -12,12 +12,4 @@ RSpec.describe Gitlab::GithubImport::ImportPullRequestReviewWorker do describe '#importer_class' do it { expect(subject.importer_class).to eq(Gitlab::GithubImport::Importer::PullRequestReviewImporter) } end - - describe '#counter_name' do - it { expect(subject.counter_name).to eq(:github_importer_imported_pull_request_reviews) } - end - - describe '#counter_description' do - it { expect(subject.counter_description).to eq('The number of imported GitHub pull request reviews') } - end end diff --git a/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb b/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb index 1238929fbcb..6fe9741075f 100644 --- a/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb +++ b/spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb @@ -43,7 +43,7 @@ RSpec.describe Gitlab::GithubImport::ImportPullRequestWorker do expect(importer) .to receive(:execute) - expect(worker.counter) + expect(Gitlab::GithubImport::ObjectCounter) .to receive(:increment) .and_call_original diff --git a/spec/workers/partition_creation_worker_spec.rb b/spec/workers/partition_creation_worker_spec.rb index 37225cc1f79..5d15870b7f6 100644 --- a/spec/workers/partition_creation_worker_spec.rb +++ b/spec/workers/partition_creation_worker_spec.rb @@ -1,27 +1,16 @@ # frozen_string_literal: true - -require "spec_helper" +# +require 'spec_helper' RSpec.describe PartitionCreationWorker do - describe '#perform' do - subject { described_class.new.perform } - - let(:creator) { instance_double('PartitionCreator', create_partitions: nil) } - let(:monitoring) { instance_double('PartitionMonitoring', report_metrics: nil) } - - before do - allow(Gitlab::Database::Partitioning::PartitionCreator).to receive(:new).and_return(creator) - allow(Gitlab::Database::Partitioning::PartitionMonitoring).to receive(:new).and_return(monitoring) - end + subject { described_class.new.perform } - it 'delegates to PartitionCreator' do - expect(creator).to receive(:create_partitions) + let(:management_worker) { double } - subject - end - - it 'reports partition metrics' do - expect(monitoring).to receive(:report_metrics) + describe '#perform' do + it 'forwards to the Database::PartitionManagementWorker' do + expect(Database::PartitionManagementWorker).to receive(:new).and_return(management_worker) + expect(management_worker).to receive(:perform) subject end diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb index 9bc5626db6b..12c4ee0039f 100644 --- a/tooling/lib/tooling/kubernetes_client.rb +++ b/tooling/lib/tooling/kubernetes_client.rb @@ -27,6 +27,13 @@ module Tooling delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait) end + def cleanup_review_app_namespaces(created_before:, wait: true) + namespaces = review_app_namespaces_created_before(created_before: created_before) + return if namespaces.empty? + + delete_namespaces_by_exact_names(resource_names: namespaces, wait: wait) + end + private def delete_by_selector(release_name:, wait:) @@ -66,6 +73,19 @@ module Tooling run_command(command) end + def delete_namespaces_by_exact_names(resource_names:, wait:) + command = [ + 'delete', + 'namespace', + '--now', + '--ignore-not-found', + %(--wait=#{wait}), + resource_names.join(' ') + ] + + run_command(command) + end + def delete_by_matching_name(release_name:) resource_names = raw_resource_names command = [ @@ -101,9 +121,32 @@ module Tooling ] response = run_command(command) - JSON.parse(response)['items'] # rubocop:disable Gitlab/Json - .map { |resource| resource.dig('metadata', 'name') if Time.parse(resource.dig('metadata', 'creationTimestamp')) < created_before } - .compact + + resources_created_before_date(response, created_before) + end + + def review_app_namespaces_created_before(created_before:) + command = [ + 'get', + 'namespace', + "-l tls=review-apps-tls", # Get only namespaces used for review-apps + "--sort-by='{.metadata.creationTimestamp}'", + '-o json' + ] + + response = run_command(command) + + resources_created_before_date(response, created_before) + end + + def resources_created_before_date(response, date) + items = JSON.parse(response)['items'] # rubocop:disable Gitlab/Json + + items.filter_map do |item| + item_created_at = Time.parse(item.dig('metadata', 'creationTimestamp')) + + item.dig('metadata', 'name') if item_created_at < date + end rescue ::JSON::ParserError => ex puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output [] |