diff options
123 files changed, 3773 insertions, 1101 deletions
diff --git a/.markdownlint.json b/.markdownlint.json index 5d81905d056..e548e03fe28 100644 --- a/.markdownlint.json +++ b/.markdownlint.json @@ -49,6 +49,7 @@ "Elasticsearch", "Facebook", "fastlane", + "fluent-plugin-redis-slowlog", "GDK", "Geo", "Git LFS", diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 0987715dbcd..b29d496a18a 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -ab2f2386ab69575cd0a58f7279be707a17d7a6c8 +b670554eae8643f2072d3b4f6f7c5cd2b9ec8776 diff --git a/app/assets/javascripts/design_management/components/design_notes/design_note.vue b/app/assets/javascripts/design_management/components/design_notes/design_note.vue index 18444a2cc2f..8498ba51644 100644 --- a/app/assets/javascripts/design_management/components/design_notes/design_note.vue +++ b/app/assets/javascripts/design_management/components/design_notes/design_note.vue @@ -123,7 +123,7 @@ export default { </div> <template v-if="!isEditing"> <div - class="note-text js-note-text" + class="note-text js-note-text md" data-qa-selector="note_content" v-html="note.bodyHtml" ></div> diff --git a/app/assets/javascripts/repository/components/web_ide_link.vue b/app/assets/javascripts/repository/components/web_ide_link.vue deleted file mode 100644 index 6549d5a3878..00000000000 --- a/app/assets/javascripts/repository/components/web_ide_link.vue +++ /dev/null @@ -1,47 +0,0 @@ -<script> -import TreeActionLink from './tree_action_link.vue'; -import { __ } from '~/locale'; -import { webIDEUrl } from '~/lib/utils/url_utility'; - -export default { - components: { - TreeActionLink, - }, - props: { - projectPath: { - type: String, - required: true, - }, - refSha: { - type: String, - required: true, - }, - canPushCode: { - type: Boolean, - required: false, - default: true, - }, - forkPath: { - type: String, - required: false, - default: '', - }, - }, - computed: { - showLinkToFork() { - return !this.canPushCode && this.forkPath; - }, - text() { - return this.showLinkToFork ? __('Edit fork in Web IDE') : __('Web IDE'); - }, - path() { - const path = this.showLinkToFork ? this.forkPath : this.projectPath; - return webIDEUrl(`/${path}/edit/${this.refSha}/-/${this.$route.params.path || ''}`); - }, - }, -}; -</script> - -<template> - <tree-action-link :path="path" :text="text" data-qa-selector="web_ide_button" /> -</template> diff --git a/app/assets/javascripts/repository/index.js b/app/assets/javascripts/repository/index.js index 187bbfed125..7f72524b6fe 100644 --- a/app/assets/javascripts/repository/index.js +++ b/app/assets/javascripts/repository/index.js @@ -1,30 +1,22 @@ import Vue from 'vue'; -import { escapeFileUrl } from '../lib/utils/url_utility'; +import { escapeFileUrl, joinPaths, webIDEUrl } from '../lib/utils/url_utility'; import createRouter from './router'; import App from './components/app.vue'; import Breadcrumbs from './components/breadcrumbs.vue'; import LastCommit from './components/last_commit.vue'; import TreeActionLink from './components/tree_action_link.vue'; -import WebIdeLink from './components/web_ide_link.vue'; +import WebIdeLink from '~/vue_shared/components/web_ide_link.vue'; import DirectoryDownloadLinks from './components/directory_download_links.vue'; import apolloProvider from './graphql'; import { setTitle } from './utils/title'; import { updateFormAction } from './utils/dom'; -import { parseBoolean } from '../lib/utils/common_utils'; +import { convertObjectPropsToCamelCase, parseBoolean } from '../lib/utils/common_utils'; import { __ } from '../locale'; export default function setupVueRepositoryList() { const el = document.getElementById('js-tree-list'); const { dataset } = el; - const { - canPushCode, - projectPath, - projectShortPath, - forkPath, - ref, - escapedRef, - fullName, - } = dataset; + const { projectPath, projectShortPath, ref, escapedRef, fullName } = dataset; const router = createRouter(projectPath, escapedRef); apolloProvider.clients.defaultClient.cache.writeData({ @@ -121,6 +113,10 @@ export default function setupVueRepositoryList() { const webIdeLinkEl = document.getElementById('js-tree-web-ide-link'); if (webIdeLinkEl) { + const { ideBasePath, ...options } = convertObjectPropsToCamelCase( + JSON.parse(webIdeLinkEl.dataset.options), + ); + // eslint-disable-next-line no-new new Vue({ el: webIdeLinkEl, @@ -128,10 +124,10 @@ export default function setupVueRepositoryList() { render(h) { return h(WebIdeLink, { props: { - projectPath, - refSha: ref, - forkPath, - canPushCode: parseBoolean(canPushCode), + webIdeUrl: webIDEUrl( + joinPaths('/', ideBasePath, 'edit', ref, '-', this.$route.params.path || '', '/'), + ), + ...options, }, }); }, diff --git a/app/assets/javascripts/vue_shared/components/actions_button.vue b/app/assets/javascripts/vue_shared/components/actions_button.vue new file mode 100644 index 00000000000..f333ab49ead --- /dev/null +++ b/app/assets/javascripts/vue_shared/components/actions_button.vue @@ -0,0 +1,90 @@ +<script> +import { + GlDropdown, + GlDropdownItem, + GlDropdownDivider, + GlLink, + GlTooltipDirective, +} from '@gitlab/ui'; + +export default { + components: { + GlDropdown, + GlDropdownItem, + GlDropdownDivider, + GlLink, + }, + directives: { + GlTooltip: GlTooltipDirective, + }, + props: { + actions: { + type: Array, + required: true, + }, + selectedKey: { + type: String, + required: false, + default: '', + }, + }, + computed: { + hasMultipleActions() { + return this.actions.length > 1; + }, + selectedAction() { + return this.actions.find(x => x.key === this.selectedKey) || this.actions[0]; + }, + }, + methods: { + handleItemClick(action) { + this.$emit('select', action.key); + }, + handleClick(action, evt) { + return action.handle?.(evt); + }, + }, +}; +</script> + +<template> + <gl-dropdown + v-if="hasMultipleActions" + v-gl-tooltip="selectedAction.tooltip" + class="gl-button-deprecated-adapter" + :text="selectedAction.text" + :split-href="selectedAction.href" + split + @click="handleClick(selectedAction, $event)" + > + <template slot="button-content"> + <span class="gl-new-dropdown-button-text" v-bind="selectedAction.attrs"> + {{ selectedAction.text }} + </span> + </template> + <template v-for="(action, index) in actions"> + <gl-dropdown-item + :key="action.key" + class="gl-dropdown-item-deprecated-adapter" + :is-check-item="true" + :is-checked="action.key === selectedAction.key" + :secondary-text="action.secondaryText" + :data-testid="`action_${action.key}`" + @click="handleItemClick(action)" + > + {{ action.text }} + </gl-dropdown-item> + <gl-dropdown-divider v-if="index != actions.length - 1" :key="action.key + '_divider'" /> + </template> + </gl-dropdown> + <gl-link + v-else-if="selectedAction" + v-gl-tooltip="selectedAction.tooltip" + v-bind="selectedAction.attrs" + class="btn" + :href="selectedAction.href" + @click="handleClick(selectedAction, $event)" + > + {{ selectedAction.text }} + </gl-link> +</template> diff --git a/app/assets/javascripts/vue_shared/components/web_ide_link.vue b/app/assets/javascripts/vue_shared/components/web_ide_link.vue new file mode 100644 index 00000000000..8307c6d3b55 --- /dev/null +++ b/app/assets/javascripts/vue_shared/components/web_ide_link.vue @@ -0,0 +1,118 @@ +<script> +import $ from 'jquery'; +import { __ } from '~/locale'; +import LocalStorageSync from '~/vue_shared/components/local_storage_sync.vue'; +import ActionsButton from '~/vue_shared/components/actions_button.vue'; + +const KEY_WEB_IDE = 'webide'; +const KEY_GITPOD = 'gitpod'; + +export default { + components: { + ActionsButton, + LocalStorageSync, + }, + props: { + webIdeUrl: { + type: String, + required: true, + }, + needsToFork: { + type: Boolean, + required: false, + default: false, + }, + showWebIdeButton: { + type: Boolean, + required: false, + default: true, + }, + showGitpodButton: { + type: Boolean, + required: false, + default: false, + }, + gitpodUrl: { + type: String, + required: false, + default: '', + }, + gitpodEnabled: { + type: Boolean, + required: false, + default: false, + }, + }, + data() { + return { + selection: KEY_WEB_IDE, + }; + }, + computed: { + actions() { + return [this.webIdeAction, this.gitpodAction].filter(x => x); + }, + webIdeAction() { + if (!this.showWebIdeButton) { + return null; + } + + const handleOptions = this.needsToFork + ? { href: '#modal-confirm-fork', handle: () => this.showModal('#modal-confirm-fork') } + : { href: this.webIdeUrl }; + + return { + key: KEY_WEB_IDE, + text: __('Web IDE'), + secondaryText: __('Quickly and easily edit multiple files in your project.'), + tooltip: '', + attrs: { + 'data-qa-selector': 'web_ide_button', + }, + ...handleOptions, + }; + }, + gitpodAction() { + if (!this.showGitpodButton) { + return null; + } + + const handleOptions = this.gitpodEnabled + ? { href: this.gitpodUrl } + : { href: '#modal-enable-gitpod', handle: () => this.showModal('#modal-enable-gitpod') }; + + const secondaryText = __('Launch a ready-to-code development environment for your project.'); + + return { + key: KEY_GITPOD, + text: __('Gitpod'), + secondaryText, + tooltip: secondaryText, + attrs: { + 'data-qa-selector': 'gitpod_button', + }, + ...handleOptions, + }; + }, + }, + methods: { + select(key) { + this.selection = key; + }, + showModal(id) { + $(id).modal('show'); + }, + }, +}; +</script> + +<template> + <div> + <actions-button :actions="actions" :selected-key="selection" @select="select" /> + <local-storage-sync + storage-key="gl-web-ide-button-selected" + :value="selection" + @input="select" + /> + </div> +</template> diff --git a/app/assets/stylesheets/framework/buttons.scss b/app/assets/stylesheets/framework/buttons.scss index b6039b971a9..a9c1652d00d 100644 --- a/app/assets/stylesheets/framework/buttons.scss +++ b/app/assets/stylesheets/framework/buttons.scss @@ -542,3 +542,13 @@ fieldset[disabled] .btn, .btn-no-padding { padding: 0; } + +// This class helps convert `.gl-button` children so that they consistently +// match the style of `.btn` elements which might be around them. Ideally we +// wouldn't need this class. +// +// Remove by upgrading all buttons in a container to use the new `.gl-button` style. +.gl-button-deprecated-adapter .gl-button { + box-shadow: none; + border-width: 1px; +} diff --git a/app/assets/stylesheets/framework/dropdowns.scss b/app/assets/stylesheets/framework/dropdowns.scss index d41f10202a3..0f01f3b38aa 100644 --- a/app/assets/stylesheets/framework/dropdowns.scss +++ b/app/assets/stylesheets/framework/dropdowns.scss @@ -1135,3 +1135,17 @@ header.header-content .dropdown-menu.frequent-items-dropdown-menu { width: $gl-dropdown-width-wide; } } + +.gl-dropdown-item-deprecated-adapter { + .dropdown-item { + align-items: flex-start; + + .gl-new-dropdown-item-text-primary { + @include gl-font-weight-bold; + } + + .gl-new-dropdown-item-text-secondary { + color: inherit; + } + } +} diff --git a/app/controllers/profiles/preferences_controller.rb b/app/controllers/profiles/preferences_controller.rb index 8653fe3b6ed..ea4d3e861be 100644 --- a/app/controllers/profiles/preferences_controller.rb +++ b/app/controllers/profiles/preferences_controller.rb @@ -51,6 +51,7 @@ class Profiles::PreferencesController < Profiles::ApplicationController :view_diffs_file_by_file, :tab_width, :sourcegraph_enabled, + :gitpod_enabled, :render_whitespace_in_code ] end diff --git a/app/controllers/profiles_controller.rb b/app/controllers/profiles_controller.rb index 9a28088ccf6..248d5755d92 100644 --- a/app/controllers/profiles_controller.rb +++ b/app/controllers/profiles_controller.rb @@ -104,6 +104,7 @@ class ProfilesController < Profiles::ApplicationController :bio, :email, :role, + :gitpod_enabled, :hide_no_password, :hide_no_ssh_key, :hide_project_limit, diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index b56453d60e9..9245cc1cb1c 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -222,6 +222,8 @@ module ApplicationSettingsHelper :gitaly_timeout_default, :gitaly_timeout_medium, :gitaly_timeout_fast, + :gitpod_enabled, + :gitpod_url, :grafana_enabled, :grafana_url, :gravatar_enabled, diff --git a/app/helpers/preferences_helper.rb b/app/helpers/preferences_helper.rb index ff1f9d6d9fe..2c406641882 100644 --- a/app/helpers/preferences_helper.rb +++ b/app/helpers/preferences_helper.rb @@ -80,6 +80,13 @@ module PreferencesHelper ) end + def integration_views + [].tap do |views| + views << 'gitpod' if Gitlab::Gitpod.feature_and_settings_enabled? + views << 'sourcegraph' if Gitlab::Sourcegraph.feature_available? && Gitlab::CurrentSettings.sourcegraph_enabled + end + end + private # Ensure that anyone adding new options updates `DASHBOARD_CHOICES` too diff --git a/app/helpers/tree_helper.rb b/app/helpers/tree_helper.rb index 90a5b6da4c7..7644ed783eb 100644 --- a/app/helpers/tree_helper.rb +++ b/app/helpers/tree_helper.rb @@ -191,16 +191,46 @@ module TreeHelper def vue_file_list_data(project, ref) { - can_push_code: current_user&.can?(:push_code, project) && "true", project_path: project.full_path, project_short_path: project.path, - fork_path: current_user&.fork_of(project)&.full_path, ref: ref, escaped_ref: ActionDispatch::Journey::Router::Utils.escape_path(ref), full_name: project.name_with_namespace } end + def ide_base_path(project) + can_push_code = current_user&.can?(:push_code, project) + fork_path = current_user&.fork_of(project)&.full_path + + if can_push_code + project.full_path + else + fork_path || project.full_path + end + end + + def vue_ide_link_data(project, ref) + can_collaborate = can_collaborate_with_project?(project) + can_create_mr_from_fork = can?(current_user, :fork_project, project) && can?(current_user, :create_merge_request_in, project) + show_web_ide_button = (can_collaborate || current_user&.already_forked?(project) || can_create_mr_from_fork) + + { + ide_base_path: ide_base_path(project), + needs_to_fork: !can_collaborate && !current_user&.already_forked?(project), + show_web_ide_button: show_web_ide_button, + show_gitpod_button: show_web_ide_button && Gitlab::Gitpod.feature_and_settings_enabled?(project), + gitpod_url: full_gitpod_url(project, ref), + gitpod_enabled: current_user&.gitpod_enabled + } + end + + def full_gitpod_url(project, ref) + return "" unless Gitlab::Gitpod.feature_and_settings_enabled?(project) + + "#{Gitlab::CurrentSettings.gitpod_url}##{project_tree_url(project, tree_join(ref, @path || ''))}" + end + def directory_download_links(project, ref, archive_prefix) Gitlab::Workhorse::ARCHIVE_FORMATS.map do |fmt| { diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index b660c814881..e9a3dcf39df 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -132,6 +132,11 @@ class ApplicationSetting < ApplicationRecord presence: true, if: :sourcegraph_enabled + validates :gitpod_url, + presence: true, + addressable_url: { enforce_sanitization: true }, + if: :gitpod_enabled + validates :snowplow_collector_hostname, presence: true, hostname: true, diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 206123bb470..7a869d16a31 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -74,6 +74,8 @@ module ApplicationSettingImplementation gitaly_timeout_default: 55, gitaly_timeout_fast: 10, gitaly_timeout_medium: 30, + gitpod_enabled: false, + gitpod_url: 'https://gitpod.io/', gravatar_enabled: Settings.gravatar['enabled'], group_download_export_limit: 1, group_export_limit: 6, diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 4d06878ff6f..444742062d9 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -11,6 +11,8 @@ module Ci default_value_for :data_store, :redis + after_create { metrics.increment_trace_operation(operation: :chunked) } + CHUNK_SIZE = 128.kilobytes WRITE_LOCK_RETRY = 10 WRITE_LOCK_SLEEP = 0.01.seconds @@ -182,6 +184,8 @@ module Ci end current_store.append_data(self, value, offset).then do |stored| + metrics.increment_trace_operation(operation: :appended) + raise ArgumentError, 'Trace appended incorrectly' if stored != new_size end @@ -205,5 +209,9 @@ module Ci retries: WRITE_LOCK_RETRY, sleep_sec: WRITE_LOCK_SLEEP }] end + + def metrics + @metrics ||= ::Gitlab::Ci::Trace::Metrics.new + end end end diff --git a/app/models/concerns/relative_positioning.rb b/app/models/concerns/relative_positioning.rb index 3f65ebe3db7..3cbc174536c 100644 --- a/app/models/concerns/relative_positioning.rb +++ b/app/models/concerns/relative_positioning.rb @@ -27,18 +27,7 @@ # module RelativePositioning extend ActiveSupport::Concern - - STEPS = 10 - IDEAL_DISTANCE = 2**(STEPS - 1) + 1 - - MIN_POSITION = Gitlab::Database::MIN_INT_VALUE - START_POSITION = 0 - MAX_POSITION = Gitlab::Database::MAX_INT_VALUE - - MAX_GAP = IDEAL_DISTANCE * 2 - MIN_GAP = 2 - - NoSpaceLeft = Class.new(StandardError) + include ::Gitlab::RelativePositioning class_methods do def move_nulls_to_end(objects) @@ -49,56 +38,10 @@ module RelativePositioning move_nulls(objects, at_end: false) end - # This method takes two integer values (positions) and - # calculates the position between them. The range is huge as - # the maximum integer value is 2147483647. - # - # We avoid open ranges by clamping the range to [MIN_POSITION, MAX_POSITION]. - # - # Then we handle one of three cases: - # - If the gap is too small, we raise NoSpaceLeft - # - If the gap is larger than MAX_GAP, we place the new position at most - # IDEAL_DISTANCE from the edge of the gap. - # - otherwise we place the new position at the midpoint. - # - # The new position will always satisfy: pos_before <= midpoint <= pos_after - # - # As a precondition, the gap between pos_before and pos_after MUST be >= 2. - # If the gap is too small, NoSpaceLeft is raised. - # - # This class method should only be called by instance methods of this module, which - # include handling for minimum gap size. - # - # @raises NoSpaceLeft - # @api private - def position_between(pos_before, pos_after) - pos_before ||= MIN_POSITION - pos_after ||= MAX_POSITION - - pos_before, pos_after = [pos_before, pos_after].sort - - gap_width = pos_after - pos_before - midpoint = [pos_after - 1, pos_before + (gap_width / 2)].min - - if gap_width < MIN_GAP - raise NoSpaceLeft - elsif gap_width > MAX_GAP - if pos_before <= MIN_POSITION - pos_after - IDEAL_DISTANCE - elsif pos_after >= MAX_POSITION - pos_before + IDEAL_DISTANCE - else - midpoint - end - else - midpoint - end - end - private # @api private - def gap_size(object, gaps:, at_end:, starting_from:) + def gap_size(context, gaps:, at_end:, starting_from:) total_width = IDEAL_DISTANCE * gaps size = if at_end && starting_from + total_width >= MAX_POSITION (MAX_POSITION - starting_from) / gaps @@ -108,23 +51,17 @@ module RelativePositioning IDEAL_DISTANCE end - # Shift max elements leftwards if there isn't enough space return [size, starting_from] if size >= MIN_GAP - order = at_end ? :desc : :asc - terminus = object - .send(:relative_siblings) # rubocop:disable GitlabSecurity/PublicSend - .where('relative_position IS NOT NULL') - .order(relative_position: order) - .first - if at_end - terminus.move_sequence_before(true) - max_relative_position = terminus.reset.relative_position + terminus = context.max_sibling + terminus.shift_left + max_relative_position = terminus.relative_position [[(MAX_POSITION - max_relative_position) / gaps, IDEAL_DISTANCE].min, max_relative_position] else - terminus.move_sequence_after(true) - min_relative_position = terminus.reset.relative_position + terminus = context.min_sibling + terminus.shift_right + min_relative_position = terminus.relative_position [[(min_relative_position - MIN_POSITION) / gaps, IDEAL_DISTANCE].min, min_relative_position] end end @@ -142,8 +79,9 @@ module RelativePositioning objects = objects.reject(&:relative_position) return 0 if objects.empty? - representative = objects.first number_of_gaps = objects.size # 1 to the nearest neighbour, and one between each + representative = RelativePositioning.mover.context(objects.first) + position = if at_end representative.max_relative_position else @@ -198,306 +136,68 @@ module RelativePositioning end end - def min_relative_position(&block) - calculate_relative_position('MIN', &block) - end - - def max_relative_position(&block) - calculate_relative_position('MAX', &block) - end - - def prev_relative_position(ignoring: nil) - prev_pos = nil - - if self.relative_position - prev_pos = max_relative_position do |relation| - relation = relation.id_not_in(ignoring.id) if ignoring.present? - relation.where('relative_position < ?', self.relative_position) - end - end - - prev_pos - end - - def next_relative_position(ignoring: nil) - next_pos = nil - - if self.relative_position - next_pos = min_relative_position do |relation| - relation = relation.id_not_in(ignoring.id) if ignoring.present? - relation.where('relative_position > ?', self.relative_position) - end - end - - next_pos + def self.mover + ::Gitlab::RelativePositioning::Mover.new(START_POSITION, (MIN_POSITION..MAX_POSITION)) end def move_between(before, after) - return move_after(before) unless after - return move_before(after) unless before - - before, after = after, before if after.relative_position < before.relative_position - - pos_left = before.relative_position - pos_right = after.relative_position + before, after = [before, after].sort_by(&:relative_position) if before && after - if pos_right - pos_left < MIN_GAP - # Not enough room! Make space by shifting all previous elements to the left - # if there is enough space, else to the right - gap = after.send(:find_next_gap_before) # rubocop:disable GitlabSecurity/PublicSend - - if gap.present? - after.move_sequence_before(next_gap: gap) - pos_left -= optimum_delta_for_gap(gap) - else - before.move_sequence_after - pos_right = after.reset.relative_position - end - end - - new_position = self.class.position_between(pos_left, pos_right) - - self.relative_position = new_position + RelativePositioning.mover.move(self, before, after) + rescue ActiveRecord::QueryCanceled, NoSpaceLeft => e + could_not_move(e) + raise e end def move_after(before = self) - pos_before = before.relative_position - pos_after = before.next_relative_position(ignoring: self) - - if pos_before == MAX_POSITION || gap_too_small?(pos_after, pos_before) - gap = before.send(:find_next_gap_after) # rubocop:disable GitlabSecurity/PublicSend - - if gap.nil? - before.move_sequence_before(true) - pos_before = before.reset.relative_position - else - before.move_sequence_after(next_gap: gap) - pos_after += optimum_delta_for_gap(gap) - end - end - - self.relative_position = self.class.position_between(pos_before, pos_after) + RelativePositioning.mover.move(self, before, nil) + rescue ActiveRecord::QueryCanceled, NoSpaceLeft => e + could_not_move(e) + raise e end def move_before(after = self) - pos_after = after.relative_position - pos_before = after.prev_relative_position(ignoring: self) - - if pos_after == MIN_POSITION || gap_too_small?(pos_before, pos_after) - gap = after.send(:find_next_gap_before) # rubocop:disable GitlabSecurity/PublicSend - - if gap.nil? - after.move_sequence_after(true) - pos_after = after.reset.relative_position - else - after.move_sequence_before(next_gap: gap) - pos_before -= optimum_delta_for_gap(gap) - end - end - - self.relative_position = self.class.position_between(pos_before, pos_after) + RelativePositioning.mover.move(self, nil, after) + rescue ActiveRecord::QueryCanceled, NoSpaceLeft => e + could_not_move(e) + raise e end def move_to_end - max_pos = max_relative_position - - if max_pos.nil? - self.relative_position = START_POSITION - elsif gap_too_small?(max_pos, MAX_POSITION + 1) - max = relative_siblings.order(Gitlab::Database.nulls_last_order('relative_position', 'DESC')).first - max.move_sequence_before(true) - max.reset - self.relative_position = self.class.position_between(max.relative_position, MAX_POSITION + 1) - else - self.relative_position = self.class.position_between(max_pos, MAX_POSITION + 1) - end + RelativePositioning.mover.move_to_end(self) + rescue NoSpaceLeft => e + could_not_move(e) + self.relative_position = MAX_POSITION + rescue ActiveRecord::QueryCanceled => e + could_not_move(e) + raise e end def move_to_start - min_pos = min_relative_position - - if min_pos.nil? - self.relative_position = START_POSITION - elsif gap_too_small?(min_pos, MIN_POSITION - 1) - min = relative_siblings.order(Gitlab::Database.nulls_last_order('relative_position', 'ASC')).first - min.move_sequence_after(true) - min.reset - self.relative_position = self.class.position_between(MIN_POSITION - 1, min.relative_position) - else - self.relative_position = self.class.position_between(MIN_POSITION - 1, min_pos) - end - end - - # Moves the sequence before the current item to the middle of the next gap - # For example, we have - # - # 5 . . . . . 11 12 13 14 [15] 16 . 17 - # ----------- - # - # This moves the sequence [11 12 13 14] to [8 9 10 11], so we have: - # - # 5 . . 8 9 10 11 . . . [15] 16 . 17 - # --------- - # - # Creating a gap to the left of the current item. We can understand this as - # dividing the 5 spaces between 5 and 11 into two smaller gaps of 2 and 3. - # - # If `include_self` is true, the current item will also be moved, creating a - # gap to the right of the current item: - # - # 5 . . 8 9 10 11 [14] . . . 16 . 17 - # -------------- - # - # As an optimization, the gap can be precalculated and passed to this method. - # - # @api private - # @raises NoSpaceLeft if the sequence cannot be moved - def move_sequence_before(include_self = false, next_gap: find_next_gap_before) - raise NoSpaceLeft unless next_gap.present? - - delta = optimum_delta_for_gap(next_gap) - - move_sequence(next_gap[:start], relative_position, -delta, include_self) - end - - # Moves the sequence after the current item to the middle of the next gap - # For example, we have: - # - # 8 . 10 [11] 12 13 14 15 . . . . . 21 - # ----------- - # - # This moves the sequence [12 13 14 15] to [15 16 17 18], so we have: - # - # 8 . 10 [11] . . . 15 16 17 18 . . 21 - # ----------- - # - # Creating a gap to the right of the current item. We can understand this as - # dividing the 5 spaces between 15 and 21 into two smaller gaps of 3 and 2. - # - # If `include_self` is true, the current item will also be moved, creating a - # gap to the left of the current item: - # - # 8 . 10 . . . [14] 15 16 17 18 . . 21 - # ---------------- - # - # As an optimization, the gap can be precalculated and passed to this method. - # - # @api private - # @raises NoSpaceLeft if the sequence cannot be moved - def move_sequence_after(include_self = false, next_gap: find_next_gap_after) - raise NoSpaceLeft unless next_gap.present? - - delta = optimum_delta_for_gap(next_gap) - - move_sequence(relative_position, next_gap[:start], delta, include_self) - end - - private - - def gap_too_small?(pos_a, pos_b) - return false unless pos_a && pos_b - - (pos_a - pos_b).abs < MIN_GAP - end - - # Find the first suitable gap to the left of the current position. - # - # Satisfies the relations: - # - gap[:start] <= relative_position - # - abs(gap[:start] - gap[:end]) >= MIN_GAP - # - MIN_POSITION <= gap[:start] <= MAX_POSITION - # - MIN_POSITION <= gap[:end] <= MAX_POSITION - # - # Supposing that the current item is 13, and we have a sequence of items: - # - # 1 . . . 5 . . . . 11 12 [13] 14 . . 17 - # ^---------^ - # - # Then we return: `{ start: 11, end: 5 }` - # - # Here start refers to the end of the gap closest to the current item. - def find_next_gap_before - items_with_next_pos = scoped_items - .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position DESC) AS next_pos') - .where('relative_position <= ?', relative_position) - .order(relative_position: :desc) - - find_next_gap(items_with_next_pos, MIN_POSITION) - end - - # Find the first suitable gap to the right of the current position. - # - # Satisfies the relations: - # - gap[:start] >= relative_position - # - abs(gap[:start] - gap[:end]) >= MIN_GAP - # - MIN_POSITION <= gap[:start] <= MAX_POSITION - # - MIN_POSITION <= gap[:end] <= MAX_POSITION - # - # Supposing the current item is 13, and that we have a sequence of items: - # - # 9 . . . [13] 14 15 . . . . 20 . . . 24 - # ^---------^ - # - # Then we return: `{ start: 15, end: 20 }` - # - # Here start refers to the end of the gap closest to the current item. - def find_next_gap_after - items_with_next_pos = scoped_items - .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position ASC) AS next_pos') - .where('relative_position >= ?', relative_position) - .order(:relative_position) - - find_next_gap(items_with_next_pos, MAX_POSITION) - end - - def find_next_gap(items_with_next_pos, end_is_nil) - gap = self.class - .from(items_with_next_pos, :items) - .where('next_pos IS NULL OR ABS(pos::bigint - next_pos::bigint) >= ?', MIN_GAP) - .limit(1) - .pluck(:pos, :next_pos) - .first - - return if gap.nil? || gap.first == end_is_nil - - { start: gap.first, end: gap.second || end_is_nil } - end - - def optimum_delta_for_gap(gap) - delta = ((gap[:start] - gap[:end]) / 2.0).abs.ceil - - [delta, IDEAL_DISTANCE].min - end - - def move_sequence(start_pos, end_pos, delta, include_self = false) - relation = include_self ? scoped_items : relative_siblings - + RelativePositioning.mover.move_to_start(self) + rescue NoSpaceLeft => e + could_not_move(e) + self.relative_position = MIN_POSITION + rescue ActiveRecord::QueryCanceled => e + could_not_move(e) + raise e + end + + # This method is used during rebalancing - override it to customise the update + # logic: + def update_relative_siblings(relation, range, delta) relation - .where('relative_position BETWEEN ? AND ?', start_pos, end_pos) + .where(relative_position: range) .update_all("relative_position = relative_position + #{delta}") end - def calculate_relative_position(calculation) - # When calculating across projects, this is much more efficient than - # MAX(relative_position) without the GROUP BY, due to index usage: - # https://gitlab.com/gitlab-org/gitlab-foss/issues/54276#note_119340977 - relation = scoped_items - .order(Gitlab::Database.nulls_last_order('position', 'DESC')) - .group(self.class.relative_positioning_parent_column) - .limit(1) - - relation = yield relation if block_given? - - relation - .pluck(self.class.relative_positioning_parent_column, Arel.sql("#{calculation}(relative_position) AS position")) - .first&.last - end - - def relative_siblings(relation = scoped_items) - relation.id_not_in(id) + # This method is used to exclude the current self (or another object) + # from a relation. Customize this if `id <> :id` is not sufficient + def exclude_self(relation, excluded: self) + relation.id_not_in(excluded.id) end - def scoped_items - self.class.relative_positioning_query_base(self) + # Override if you want to be notified of failures to move + def could_not_move(exception) end end diff --git a/app/models/issue.rb b/app/models/issue.rb index b9c8c3cf9fe..73e22a00db0 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -444,20 +444,9 @@ class Issue < ApplicationRecord Gitlab::EtagCaching::Store.new.touch(key) end - def find_next_gap_before - super - rescue ActiveRecord::QueryCanceled => e - # Symptom of running out of space - schedule rebalancing - IssueRebalancingWorker.perform_async(nil, project_id) - raise e - end - - def find_next_gap_after - super - rescue ActiveRecord::QueryCanceled => e + def could_not_move(exception) # Symptom of running out of space - schedule rebalancing IssueRebalancingWorker.perform_async(nil, project_id) - raise e end end diff --git a/app/models/project.rb b/app/models/project.rb index b931f145260..38466c05bf4 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -41,10 +41,6 @@ class Project < ApplicationRecord STATISTICS_ATTRIBUTE = 'repositories_count' UNKNOWN_IMPORT_URL = 'http://unknown.git' - # Hashed Storage versions handle rolling out new storage to project and dependents models: - # nil: legacy - # 1: repository - # 2: attachments LATEST_STORAGE_VERSION = 2 HASHED_STORAGE_FEATURES = { repository: 1, diff --git a/app/models/user.rb b/app/models/user.rb index 9b8fe4881ad..0a784b30d8f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -279,6 +279,7 @@ class User < ApplicationRecord :view_diffs_file_by_file, :view_diffs_file_by_file=, :tab_width, :tab_width=, :sourcegraph_enabled, :sourcegraph_enabled=, + :gitpod_enabled, :gitpod_enabled=, :setup_for_company, :setup_for_company=, :render_whitespace_in_code, :render_whitespace_in_code=, :experience_level, :experience_level=, diff --git a/app/models/vulnerability.rb b/app/models/vulnerability.rb new file mode 100644 index 00000000000..71d0b1db410 --- /dev/null +++ b/app/models/vulnerability.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# Placeholder class for model that is implemented in EE +# It reserves '+' as a reference prefix, but the table does not exist in FOSS +class Vulnerability < ApplicationRecord + include IgnorableColumns + + def self.reference_prefix + '+' + end + + def self.reference_prefix_escaped + '+' + end +end + +Vulnerability.prepend_if_ee('EE::Vulnerability') diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb index 2cfde1a8bfe..81586332f51 100644 --- a/app/services/ci/create_job_artifacts_service.rb +++ b/app/services/ci/create_job_artifacts_service.rb @@ -2,6 +2,8 @@ module Ci class CreateJobArtifactsService < ::BaseService + include Gitlab::Utils::UsageData + ArtifactsExistError = Class.new(StandardError) LSIF_ARTIFACT_TYPE = 'lsif' @@ -22,7 +24,11 @@ module Ci return result unless result[:status] == :success headers = JobArtifactUploader.workhorse_authorize(has_length: false, maximum_size: max_size(artifact_type)) - headers[:ProcessLsif] = lsif?(artifact_type) + + if lsif?(artifact_type) + headers[:ProcessLsif] = true + track_usage_event('i_source_code_code_intelligence', project) + end success(headers: headers) end diff --git a/app/views/admin/application_settings/_gitpod.html.haml b/app/views/admin/application_settings/_gitpod.html.haml new file mode 100644 index 00000000000..bbad5155ada --- /dev/null +++ b/app/views/admin/application_settings/_gitpod.html.haml @@ -0,0 +1,30 @@ +- return unless Gitlab::Gitpod.feature_available? +- expanded = integration_expanded?('gitpod_') +- gitpod_link = link_to("Gitpod#{sprite_icon('external-link', size: 12, css_class: 'ml-1 vertical-align-center')}".html_safe, 'https://gitpod.io/', target: '_blank', rel: 'noopener noreferrer') + +%section.settings.no-animate#js-gitpod-settings{ class: ('expanded' if expanded) } + .settings-header + %h4 + = _('Gitpod') + %button.btn.btn-default.js-settings-toggle{ type: 'button' } + = expanded ? _('Collapse') : _('Expand') + %p + = s_('Enable %{gitpod_link} integration to launch a development environment in your browser directly from GitLab.').html_safe % { gitpod_link: gitpod_link } + = link_to sprite_icon('question-o'), help_page_path('integration/gitpod.md'), target: '_blank', class: 'has-tooltip', title: _('More information') + + + .settings-content + = form_for @application_setting, url: general_admin_application_settings_path(anchor: 'js-gitpod-settings'), html: { class: 'fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + .form-check + = f.check_box :gitpod_enabled, class: 'form-check-input' + = f.label :gitpod_enabled, s_('Gitpod|Enable Gitpod integration'), class: 'form-check-label' + .form-group + = f.label :gitpod_url, s_('Gitpod|Gitpod URL'), class: 'label-bold' + = f.text_field :gitpod_url, class: 'form-control', placeholder: s_('Gitpod|e.g. https://gitpod.example.com') + .form-text.text-muted + = s_('Gitpod|Add the URL to your Gitpod instance configured to read your GitLab projects.') + = f.submit s_('Save changes'), class: 'btn btn-success' diff --git a/app/views/admin/application_settings/general.html.haml b/app/views/admin/application_settings/general.html.haml index 788dc0b0f1b..823cee09d4b 100644 --- a/app/views/admin/application_settings/general.html.haml +++ b/app/views/admin/application_settings/general.html.haml @@ -117,6 +117,7 @@ #js-maintenance-mode-settings = render_if_exists 'admin/application_settings/elasticsearch_form' += render 'admin/application_settings/gitpod' = render 'admin/application_settings/plantuml' = render 'admin/application_settings/sourcegraph' = render_if_exists 'admin/application_settings/slack' diff --git a/app/views/notify/_failed_builds.html.haml b/app/views/notify/_failed_builds.html.haml index 1711c34a842..cde0ac21d6d 100644 --- a/app/views/notify/_failed_builds.html.haml +++ b/app/views/notify/_failed_builds.html.haml @@ -23,10 +23,3 @@ = build.stage %td{ align: "right", style: "font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; padding: 16px 0; color: #8c8c8c; font-weight: 500; font-size: 14px;" } = render "notify/links/#{build.to_partial_path}", pipeline: pipeline, build: build - %tr.build-log - - if build.has_trace? - %td{ colspan: "2", style: "font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; padding: 0 0 16px;" } - %pre{ style: "font-family: Monaco,'Lucida Console','Courier New',Courier,monospace; background-color: #fafafa; border-radius: 4px; overflow: hidden; white-space: pre-wrap; word-break: break-all; font-size:13px; line-height: 1.4; padding: 16px 8px; color: #333333; margin: 0;" } - = build.trace.html(last_lines: 30).html_safe - - else - %td{ colspan: "2" } diff --git a/app/views/notify/autodevops_disabled_email.text.erb b/app/views/notify/autodevops_disabled_email.text.erb index 91092060e74..f849c017265 100644 --- a/app/views/notify/autodevops_disabled_email.text.erb +++ b/app/views/notify/autodevops_disabled_email.text.erb @@ -14,7 +14,4 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>. <%= render "notify/links/#{build.to_partial_path}", pipeline: @pipeline, build: build %> Stage: <%= build.stage %> Name: <%= build.name %> - <% if build.has_trace? -%> - Trace: <%= build.trace.raw(last_lines: 30) %> - <% end -%> <% end -%> diff --git a/app/views/notify/pipeline_failed_email.text.erb b/app/views/notify/pipeline_failed_email.text.erb index 41b26842dbc..b388aad7048 100644 --- a/app/views/notify/pipeline_failed_email.text.erb +++ b/app/views/notify/pipeline_failed_email.text.erb @@ -34,8 +34,4 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>. <%= render "notify/links/#{build.to_partial_path}", pipeline: @pipeline, build: build %> Stage: <%= build.stage %> Name: <%= build.name %> -<% if build.has_trace? -%> -Trace: <%= build.trace.raw(last_lines: 30) %> -<% end -%> - <% end -%> diff --git a/app/views/profiles/preferences/_gitpod.html.haml b/app/views/profiles/preferences/_gitpod.html.haml new file mode 100644 index 00000000000..69c9443ebbb --- /dev/null +++ b/app/views/profiles/preferences/_gitpod.html.haml @@ -0,0 +1,11 @@ +- gitpod_link = link_to("Gitpod#{sprite_icon('external-link', size: 12, css_class: 'ml-1 vertical-align-center')}".html_safe, 'https://gitpod.io/', target: '_blank', rel: 'noopener noreferrer') + +%label.label-bold#gitpod + = s_('Gitpod') += link_to sprite_icon('question-o'), help_page_path('integration/gitpod.md'), target: '_blank', class: 'has-tooltip', title: _('More information') +.form-group.form-check + = f.check_box :gitpod_enabled, class: 'form-check-input' + = f.label :gitpod_enabled, class: 'form-check-label' do + = s_('Gitpod|Enable Gitpod integration').html_safe + .form-text.text-muted + = s_('Enable %{gitpod_link} integration to launch a development environment in your browser directly from GitLab.').html_safe % { gitpod_link: gitpod_link } diff --git a/app/views/profiles/preferences/_integrations.html.haml b/app/views/profiles/preferences/_integrations.html.haml new file mode 100644 index 00000000000..037fe5df263 --- /dev/null +++ b/app/views/profiles/preferences/_integrations.html.haml @@ -0,0 +1,18 @@ +- views = integration_views +- return unless views.any? + +.col-sm-12 + %hr + +.col-lg-4.profile-settings-sidebar#integrations + %h4.gl-mt-0 + = s_('Preferences|Integrations') + %p + = s_('Preferences|Customize integrations with third party services.') + = succeed '.' do + = link_to _('Learn more'), help_page_path('user/profile/preferences.md', anchor: 'integrations'), target: '_blank' + +.col-lg-8 + - views.each do |view| + = render view, f: f + diff --git a/app/views/profiles/preferences/_sourcegraph.html.haml b/app/views/profiles/preferences/_sourcegraph.html.haml index 595b70befcc..fdd0be22664 100644 --- a/app/views/profiles/preferences/_sourcegraph.html.haml +++ b/app/views/profiles/preferences/_sourcegraph.html.haml @@ -1,26 +1,10 @@ -- return unless Gitlab::Sourcegraph::feature_available? && Gitlab::CurrentSettings.sourcegraph_enabled -- sourcegraph_url = Gitlab::CurrentSettings.sourcegraph_url - -.col-sm-12 - %hr - -.col-lg-4.profile-settings-sidebar#integrations - %h4.gl-mt-0 - = s_('Preferences|Integrations') - %p - = s_('Preferences|Customize integrations with third party services.') - = succeed '.' do - = link_to _('Learn more'), help_page_path('user/profile/preferences.md', anchor: 'integrations'), target: '_blank' -.col-lg-8 - %label.label-bold - = s_('Preferences|Sourcegraph') - = link_to sprite_icon('question-o'), help_page_path('user/profile/preferences.md', anchor: 'sourcegraph'), target: '_blank', class: 'has-tooltip', title: _('More information') - .form-group.form-check - = f.check_box :sourcegraph_enabled, class: 'form-check-input' - = f.label :sourcegraph_enabled, class: 'form-check-label' do - - link_start = '<a href="%{url}">'.html_safe % { url: sourcegraph_url } - - link_end = '</a>'.html_safe - = s_('Preferences|Enable integrated code intelligence on code views').html_safe % { link_start: link_start, link_end: link_end } - .form-text.text-muted - = sourcegraph_url_message - = sourcegraph_experimental_message +%label.label-bold + = s_('Preferences|Sourcegraph') += link_to sprite_icon('question-o'), help_page_path('user/profile/preferences.md', anchor: 'sourcegraph'), target: '_blank', class: 'has-tooltip', title: _('More information') +.form-group.form-check + = f.check_box :sourcegraph_enabled, class: 'form-check-input' + = f.label :sourcegraph_enabled, class: 'form-check-label' do + = s_('Preferences|Enable integrated code intelligence on code views').html_safe + .form-text.text-muted + = sourcegraph_url_message + = sourcegraph_experimental_message diff --git a/app/views/profiles/preferences/show.html.haml b/app/views/profiles/preferences/show.html.haml index 472185f0b05..2c705886f47 100644 --- a/app/views/profiles/preferences/show.html.haml +++ b/app/views/profiles/preferences/show.html.haml @@ -138,7 +138,7 @@ .form-text.text-muted = s_('Preferences|For example: 30 mins ago.') - = render 'sourcegraph', f: f + = render 'integrations', f: f .col-lg-4.profile-settings-sidebar .col-lg-8 diff --git a/app/views/projects/tree/_tree_header.html.haml b/app/views/projects/tree/_tree_header.html.haml index eab6d750a02..268858f8ff8 100644 --- a/app/views/projects/tree/_tree_header.html.haml +++ b/app/views/projects/tree/_tree_header.html.haml @@ -1,5 +1,6 @@ - can_collaborate = can_collaborate_with_project?(@project) - can_create_mr_from_fork = can?(current_user, :fork_project, @project) && can?(current_user, :create_merge_request_in, @project) +- can_visit_ide = can_collaborate || current_user&.already_forked?(@project) .tree-ref-container .tree-ref-holder @@ -14,12 +15,12 @@ = render 'projects/find_file_link' - - if can_collaborate || current_user&.already_forked?(@project) - #js-tree-web-ide-link.d-inline-block - - elsif can_create_mr_from_fork - = link_to '#modal-confirm-fork', class: 'btn btn-default qa-web-ide-button', data: { target: '#modal-confirm-fork', toggle: 'modal'} do - = _('Web IDE') - = render 'shared/confirm_fork_modal', fork_path: ide_fork_and_edit_path(@project, @ref, @path) + - if can_visit_ide || can_create_mr_from_fork + #js-tree-web-ide-link.d-inline-block{ data: { options: vue_ide_link_data(@project, @ref).to_json } } + - if !can_visit_ide + = render 'shared/confirm_fork_modal', fork_path: ide_fork_and_edit_path(@project, @ref, @path) + - unless current_user&.gitpod_enabled + = render 'shared/gitpod/enable_gitpod_modal' - if show_xcode_link?(@project) .project-action-button.project-xcode.inline< diff --git a/app/views/shared/gitpod/_enable_gitpod_modal.html.haml b/app/views/shared/gitpod/_enable_gitpod_modal.html.haml new file mode 100644 index 00000000000..a6bd1d10e43 --- /dev/null +++ b/app/views/shared/gitpod/_enable_gitpod_modal.html.haml @@ -0,0 +1,12 @@ +#modal-enable-gitpod.modal.qa-enable-gitpod-modal + .modal-dialog + .modal-content + .modal-header + %h3.page-title= _('Enable Gitpod?') + %button.close{ type: "button", "data-dismiss": "modal", "aria-label" => _('Close') } + %span{ "aria-hidden": true } × + .modal-body.p-3 + %p= (_("To use Gitpod you must first enable the feature in the integrations section of your %{user_prefs}.") % { user_prefs: link_to(_('user preferences'), profile_preferences_path(anchor: 'gitpod')) }).html_safe + .modal-footer + = link_to _('Cancel'), '#', class: "btn btn-cancel", "data-dismiss" => "modal" + = link_to _('Enable Gitpod'), profile_path(user: { gitpod_enabled: true}), class: 'btn btn-success', method: :put diff --git a/bin/feature-flag b/bin/feature-flag index c01fb67131b..9a550dc8884 100755 --- a/bin/feature-flag +++ b/bin/feature-flag @@ -60,11 +60,11 @@ class FeatureFlagOptionParser options.force = value end - opts.on('-m', '--introduced-by-url [string]', String, 'URL to Merge Request introducing Feature Flag') do |value| + opts.on('-m', '--introduced-by-url [string]', String, 'URL of Merge Request introducing the Feature Flag') do |value| options.introduced_by_url = value end - opts.on('-i', '--rollout-issue-url [string]', String, 'URL to Issue rolling out Feature Flag') do |value| + opts.on('-i', '--rollout-issue-url [string]', String, 'URL of Issue rolling out the Feature Flag') do |value| options.rollout_issue_url = value end @@ -106,7 +106,7 @@ class FeatureFlagOptionParser def read_group $stdout.puts - $stdout.puts ">> Please specify the group introducing feature flag, like `group::apm`:" + $stdout.puts ">> Specify the group introducing the feature flag, like `group::apm`:" loop do $stdout.print "?> " @@ -114,7 +114,7 @@ class FeatureFlagOptionParser group = nil if group.empty? return group if group.nil? || group.start_with?('group::') - $stderr.puts "Group needs to include `group::`" + $stderr.puts "The group needs to include `group::`" end end @@ -123,7 +123,7 @@ class FeatureFlagOptionParser return TYPES.first.first if TYPES.one? $stdout.puts - $stdout.puts ">> Please specify the type of your feature flag:" + $stdout.puts ">> Specify the feature flag type:" $stdout.puts TYPES.each do |type, data| $stdout.puts "#{type.to_s.rjust(15)}#{' '*6}#{data[:description]}" @@ -141,7 +141,7 @@ class FeatureFlagOptionParser def read_introduced_by_url $stdout.puts - $stdout.puts ">> If you have MR open, can you paste the URL here? (or enter to skip)" + $stdout.puts ">> URL of the MR introducing the feature flag (enter to skip):" loop do $stdout.print "?> " @@ -166,11 +166,11 @@ class FeatureFlagOptionParser issue_new_url = url + "?" + URI.encode_www_form(params) $stdout.puts - $stdout.puts ">> Open this URL and fill the rest of details:" + $stdout.puts ">> Open this URL and fill in the rest of the details:" $stdout.puts issue_new_url $stdout.puts - $stdout.puts ">> Paste URL of `rollout issue` here, or enter to skip:" + $stdout.puts ">> URL of the rollout issue (enter to skip):" loop do $stdout.print "?> " diff --git a/changelogs/unreleased/222483-create-vulnerability-placeholder-model.yml b/changelogs/unreleased/222483-create-vulnerability-placeholder-model.yml new file mode 100644 index 00000000000..d97b1da2583 --- /dev/null +++ b/changelogs/unreleased/222483-create-vulnerability-placeholder-model.yml @@ -0,0 +1,5 @@ +--- +title: Create placeholder model for Vulnerability to reserve + as a reference prefix +merge_request: 42147 +author: +type: added diff --git a/changelogs/unreleased/237793-code-intelligence-usage-ping.yml b/changelogs/unreleased/237793-code-intelligence-usage-ping.yml new file mode 100644 index 00000000000..e770b3a938d --- /dev/null +++ b/changelogs/unreleased/237793-code-intelligence-usage-ping.yml @@ -0,0 +1,5 @@ +--- +title: Track projects using code intelligence +merge_request: 41881 +author: +type: added diff --git a/changelogs/unreleased/249519-fix-daemon-jobs-hash-thread-safe-issue.yml b/changelogs/unreleased/249519-fix-daemon-jobs-hash-thread-safe-issue.yml new file mode 100644 index 00000000000..b77a07c1df0 --- /dev/null +++ b/changelogs/unreleased/249519-fix-daemon-jobs-hash-thread-safe-issue.yml @@ -0,0 +1,5 @@ +--- +title: Fix daemon memory killer jobs hash thread safety issue +merge_request: 42468 +author: +type: fixed diff --git a/changelogs/unreleased/250350-design-comments-do-not-render-the-blockquotes-correctly.yml b/changelogs/unreleased/250350-design-comments-do-not-render-the-blockquotes-correctly.yml new file mode 100644 index 00000000000..2c4a0bd5992 --- /dev/null +++ b/changelogs/unreleased/250350-design-comments-do-not-render-the-blockquotes-correctly.yml @@ -0,0 +1,5 @@ +--- +title: Resolve Design comments do not render the blockquotes correctly +merge_request: 42498 +author: +type: fixed diff --git a/changelogs/unreleased/37985-add-gitpod-button-to-open-project-in-gitpod.yml b/changelogs/unreleased/37985-add-gitpod-button-to-open-project-in-gitpod.yml new file mode 100644 index 00000000000..4d4117cdca7 --- /dev/null +++ b/changelogs/unreleased/37985-add-gitpod-button-to-open-project-in-gitpod.yml @@ -0,0 +1,5 @@ +--- +title: Add Gitpod integration +merge_request: 37985 +author: Cornelius Ludmann @corneliusludmann +type: added diff --git a/changelogs/unreleased/ajk-relative-positioning-mover.yml b/changelogs/unreleased/ajk-relative-positioning-mover.yml new file mode 100644 index 00000000000..d4901e2a888 --- /dev/null +++ b/changelogs/unreleased/ajk-relative-positioning-mover.yml @@ -0,0 +1,5 @@ +--- +title: Refactor relative positioning to enable better testing +merge_request: 41967 +author: +type: other diff --git a/changelogs/unreleased/sh-remove-job-logs-from-notification-emails.yml b/changelogs/unreleased/sh-remove-job-logs-from-notification-emails.yml new file mode 100644 index 00000000000..f20a69cf741 --- /dev/null +++ b/changelogs/unreleased/sh-remove-job-logs-from-notification-emails.yml @@ -0,0 +1,5 @@ +--- +title: Remove job logs from notification e-mails +merge_request: 42395 +author: +type: changed diff --git a/config/feature_flags/development/gitpod.yml b/config/feature_flags/development/gitpod.yml new file mode 100644 index 00000000000..148ea7294ba --- /dev/null +++ b/config/feature_flags/development/gitpod.yml @@ -0,0 +1,7 @@ +--- +name: gitpod +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37985 +rollout_issue_url: +group: group::editor +type: development +default_enabled: false
\ No newline at end of file diff --git a/config/feature_flags/development/group_level_integrations.yml b/config/feature_flags/development/group_level_integrations.yml index 961b7cfcf11..8fc7da47ba1 100644 --- a/config/feature_flags/development/group_level_integrations.yml +++ b/config/feature_flags/development/group_level_integrations.yml @@ -1,7 +1,7 @@ --- name: group_level_integrations introduced_by_url: -rollout_issue_url: -group: +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/238575 +group: group::ecosystem type: development default_enabled: false diff --git a/db/migrate/20200811154630_add_gitpod_application_settings.rb b/db/migrate/20200811154630_add_gitpod_application_settings.rb new file mode 100644 index 00000000000..e4211d25d0b --- /dev/null +++ b/db/migrate/20200811154630_add_gitpod_application_settings.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddGitpodApplicationSettings < ActiveRecord::Migration[6.0] + DOWNTIME = false + + # rubocop:disable Migration/AddLimitToTextColumns + # limit is added in 20200727154631_add_gitpod_application_settings_text_limit + def change + add_column :application_settings, :gitpod_enabled, :boolean, default: false, null: false + add_column :application_settings, :gitpod_url, :text, default: 'https://gitpod.io/', null: true + end + # rubocop:enable Migration/AddLimitToTextColumns +end diff --git a/db/migrate/20200811154631_add_gitpod_application_settings_text_limit.rb b/db/migrate/20200811154631_add_gitpod_application_settings_text_limit.rb new file mode 100644 index 00000000000..1f43b5d88d5 --- /dev/null +++ b/db/migrate/20200811154631_add_gitpod_application_settings_text_limit.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddGitpodApplicationSettingsTextLimit < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_text_limit :application_settings, :gitpod_url, 255 + end + + def down + remove_text_limit :application_settings, :gitpod_url + end +end diff --git a/db/migrate/20200811154632_add_gitpod_user_preferences.rb b/db/migrate/20200811154632_add_gitpod_user_preferences.rb new file mode 100644 index 00000000000..0392c80d39c --- /dev/null +++ b/db/migrate/20200811154632_add_gitpod_user_preferences.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddGitpodUserPreferences < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + add_column :user_preferences, :gitpod_enabled, :boolean, default: false, null: false + end +end diff --git a/db/schema_migrations/20200811154630 b/db/schema_migrations/20200811154630 new file mode 100644 index 00000000000..0498382ef2e --- /dev/null +++ b/db/schema_migrations/20200811154630 @@ -0,0 +1 @@ +c04fe7e1a56bdcd41b5e1af346f9bfcae170d601954c4a0bcfcc9aea19d55528
\ No newline at end of file diff --git a/db/schema_migrations/20200811154631 b/db/schema_migrations/20200811154631 new file mode 100644 index 00000000000..1817460cd30 --- /dev/null +++ b/db/schema_migrations/20200811154631 @@ -0,0 +1 @@ +0ce17a8ad6c5ca5bba49ff522fede400fe6666490157af123ad98a7643f3ce01
\ No newline at end of file diff --git a/db/schema_migrations/20200811154632 b/db/schema_migrations/20200811154632 new file mode 100644 index 00000000000..bb1167e3182 --- /dev/null +++ b/db/schema_migrations/20200811154632 @@ -0,0 +1 @@ +523f200c635e37ee1ac52257ffd45443a3e17bfe993d22775a5377865e044a46
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 3d2e94311dd..1bf6b05da5e 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -9272,6 +9272,9 @@ CREATE TABLE public.application_settings ( enforce_namespace_storage_limit boolean DEFAULT false NOT NULL, container_registry_delete_tags_service_timeout integer DEFAULT 250 NOT NULL, elasticsearch_client_request_timeout integer DEFAULT 0 NOT NULL, + gitpod_enabled boolean DEFAULT false NOT NULL, + gitpod_url text DEFAULT 'https://gitpod.io/'::text, + CONSTRAINT check_2dba05b802 CHECK ((char_length(gitpod_url) <= 255)), CONSTRAINT check_51700b31b5 CHECK ((char_length(default_branch_name) <= 255)), CONSTRAINT check_9c6c447a13 CHECK ((char_length(maintenance_mode_message) <= 255)), CONSTRAINT check_d03919528d CHECK ((char_length(container_registry_vendor) <= 255)), @@ -16277,7 +16280,8 @@ CREATE TABLE public.user_preferences ( tab_width smallint, feature_filter_type bigint, experience_level smallint, - view_diffs_file_by_file boolean DEFAULT false NOT NULL + view_diffs_file_by_file boolean DEFAULT false NOT NULL, + gitpod_enabled boolean DEFAULT false NOT NULL ); CREATE SEQUENCE public.user_preferences_id_seq diff --git a/doc/administration/geo/disaster_recovery/promotion_runbook.md b/doc/administration/geo/disaster_recovery/promotion_runbook.md new file mode 100644 index 00000000000..fb2353513df --- /dev/null +++ b/doc/administration/geo/disaster_recovery/promotion_runbook.md @@ -0,0 +1,269 @@ +--- +stage: Enablement +group: Geo +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers +type: howto +--- + +CAUTION: **Caution:** +This runbook is in **alpha**. For complete, production-ready documentation, see the +[disaster recovery documentation](index.md). + +# Disaster Recovery (Geo) promotion runbooks **(PREMIUM ONLY)** + +## Geo planned failover runbook 1 + +| Component | Configuration | +| ----------- | --------------- | +| PostgreSQL | Omnibus-managed | +| Geo site | Single-node | +| Secondaries | One | + +This runbook will guide you through a planned failover of a single-node Geo site +with one secondary. The following general architecture is assumed: + +```mermaid +graph TD + subgraph main[Geo deployment] + subgraph Primary[Primary site] + Node_1[(GitLab node)] + end + subgraph Secondary1[Secondary site] + Node_2[(GitLab node)] + end + end +``` + +This guide will result in the following: + +1. An offline primary. +1. A promoted secondary that is now the new primary. + +What is not covered: + +1. Re-adding the old **primary** as a secondary. +1. Adding a new secondary. + +### Preparation + +NOTE: **Note:** +Before following any of those steps, make sure you have `root` access to the +**secondary** to promote it, since there isn't provided an automated way to +promote a Geo replica and perform a failover. + +On the **secondary** node, navigate to the **Admin Area > Geo** dashboard to +review its status. Replicated objects (shown in green) should be close to 100%, +and there should be no failures (shown in red). If a large proportion of +objects aren't yet replicated (shown in gray), consider giving the node more +time to complete. + + + +If any objects are failing to replicate, this should be investigated before +scheduling the maintenance window. After a planned failover, anything that +failed to replicate will be **lost**. + +You can use the +[Geo status API](../../../api/geo_nodes.md#retrieve-project-sync-or-verification-failures-that-occurred-on-the-current-node) +to review failed objects and the reasons for failure. +A common cause of replication failures is the data being missing on the +**primary** node - you can resolve these failures by restoring the data from backup, +or removing references to the missing data. + +The maintenance window won't end until Geo replication and verification is +completely finished. To keep the window as short as possible, you should +ensure these processes are close to 100% as possible during active use. + +If the **secondary** node is still replicating data from the **primary** node, +follow these steps to avoid unnecessary data loss: + +1. Until a [read-only mode](https://gitlab.com/gitlab-org/gitlab/-/issues/14609) + is implemented, updates must be prevented from happening manually to the + **primary**. Note that your **secondary** node still needs read-only + access to the **primary** node during the maintenance window: + + 1. At the scheduled time, using your cloud provider or your node's firewall, block + all HTTP, HTTPS and SSH traffic to/from the **primary** node, **except** for your IP and + the **secondary** node's IP. + + For instance, you can run the following commands on the **primary** node: + + ```shell + sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 22 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 22 -j ACCEPT + sudo iptables -A INPUT --destination-port 22 -j REJECT + + sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 80 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 80 -j ACCEPT + sudo iptables -A INPUT --tcp-dport 80 -j REJECT + + sudo iptables -A INPUT -p tcp -s <secondary_node_ip> --destination-port 443 -j ACCEPT + sudo iptables -A INPUT -p tcp -s <your_ip> --destination-port 443 -j ACCEPT + sudo iptables -A INPUT --tcp-dport 443 -j REJECT + ``` + + From this point, users will be unable to view their data or make changes on the + **primary** node. They will also be unable to log in to the **secondary** node. + However, existing sessions will work for the remainder of the maintenance period, and + public data will be accessible throughout. + + 1. Verify the **primary** node is blocked to HTTP traffic by visiting it in browser via + another IP. The server should refuse connection. + + 1. Verify the **primary** node is blocked to Git over SSH traffic by attempting to pull an + existing Git repository with an SSH remote URL. The server should refuse + connection. + + 1. On the **primary** node, disable non-Geo periodic background jobs by navigating + to **Admin Area > Monitoring > Background Jobs > Cron**, clicking `Disable All`, + and then clicking `Enable` for the `geo_sidekiq_cron_config_worker` cron job. + This job will re-enable several other cron jobs that are essential for planned + failover to complete successfully. + +1. Finish replicating and verifying all data: + + CAUTION: **Caution:** + Not all data is automatically replicated. Read more about + [what is excluded](planned_failover.md#not-all-data-is-automatically-replicated). + + 1. If you are manually replicating any + [data not managed by Geo](../replication/datatypes.md#limitations-on-replicationverification), + trigger the final replication process now. + 1. On the **primary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** + and wait for all queues except those with `geo` in the name to drop to 0. + These queues contain work that has been submitted by your users; failing over + before it is completed will cause the work to be lost. + 1. On the **primary** node, navigate to **Admin Area > Geo** and wait for the + following conditions to be true of the **secondary** node you are failing over to: + - All replication meters to each 100% replicated, 0% failures. + - All verification meters reach 100% verified, 0% failures. + - Database replication lag is 0ms. + - The Geo log cursor is up to date (0 events behind). + + 1. On the **secondary** node, navigate to **Admin Area > Monitoring > Background Jobs > Queues** + and wait for all the `geo` queues to drop to 0 queued and 0 running jobs. + 1. On the **secondary** node, use [these instructions](../../raketasks/check.md) + to verify the integrity of CI artifacts, LFS objects, and uploads in file + storage. + + At this point, your **secondary** node will contain an up-to-date copy of everything the + **primary** node has, meaning nothing will be lost when you fail over. + +1. In this final step, you need to permanently disable the **primary** node. + + CAUTION: **Caution:** + When the **primary** node goes offline, there may be data saved on the **primary** node + that has not been replicated to the **secondary** node. This data should be treated + as lost if you proceed. + + TIP: **Tip:** + If you plan to [update the **primary** domain DNS record](index.md#step-4-optional-updating-the-primary-domain-dns-record), + you may wish to lower the TTL now to speed up propagation. + + When performing a failover, we want to avoid a split-brain situation where + writes can occur in two different GitLab instances. So to prepare for the + failover, you must disable the **primary** node: + + - If you have SSH access to the **primary** node, stop and disable GitLab: + + ```shell + sudo gitlab-ctl stop + ``` + + Prevent GitLab from starting up again if the server unexpectedly reboots: + + ```shell + sudo systemctl disable gitlab-runsvdir + ``` + + NOTE: **Note:** + (**CentOS only**) In CentOS 6 or older, there is no easy way to prevent GitLab from being + started if the machine reboots isn't available (see [Omnibus GitLab issue #3058](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3058)). + It may be safest to uninstall the GitLab package completely with `sudo yum remove gitlab-ee`. + + NOTE: **Note:** + (**Ubuntu 14.04 LTS**) If you are using an older version of Ubuntu + or any other distribution based on the Upstart init system, you can prevent GitLab + from starting if the machine reboots as `root` with + `initctl stop gitlab-runsvvdir && echo 'manual' > /etc/init/gitlab-runsvdir.override && initctl reload-configuration`. + + - If you do not have SSH access to the **primary** node, take the machine offline and + prevent it from rebooting. Since there are many ways you may prefer to accomplish + this, we will avoid a single recommendation. You may need to: + + - Reconfigure the load balancers. + - Change DNS records (for example, point the **primary** DNS record to the **secondary** + node in order to stop usage of the **primary** node). + - Stop the virtual servers. + - Block traffic through a firewall. + - Revoke object storage permissions from the **primary** node. + - Physically disconnect a machine. + +### Promoting the **secondary** node + +Note the following when promoting a secondary: + +- A new **secondary** should not be added at this time. If you want to add a new + **secondary**, do this after you have completed the entire process of promoting + the **secondary** to the **primary**. +- If you encounter an `ActiveRecord::RecordInvalid: Validation failed: Name has already been taken` + error during this process, read + [the troubleshooting advice](../replication/troubleshooting.md#fixing-errors-during-a-failover-or-when-promoting-a-secondary-to-a-primary-node). + +To promote the secondary node: + +1. SSH in to your **secondary** node and login as root: + + ```shell + sudo -i + ``` + +1. Edit `/etc/gitlab/gitlab.rb` to reflect its new status as **primary** by + removing any lines that enabled the `geo_secondary_role`: + + ```ruby + ## In pre-11.5 documentation, the role was enabled as follows. Remove this line. + geo_secondary_role['enable'] = true + + ## In 11.5+ documentation, the role was enabled as follows. Remove this line. + roles ['geo_secondary_role'] + ``` + +1. Run the following command to list out all preflight checks and automatically + check if replication and verification are complete before scheduling a planned + failover to ensure the process will go smoothly: + + ```shell + gitlab-ctl promotion-preflight-checks + ``` + +1. Promote the **secondary**: + + ```shell + gitlab-ctl promote-to-primary-node + ``` + + If you have already run the [preflight checks](planned_failover.md#preflight-checks) + or don't want to run them, you can skip them: + + ```shell + gitlab-ctl promote-to-primary-node --skip-preflight-check + ``` + + You can also promote the secondary node to primary **without any further confirmation**, even when preflight checks fail: + + ```shell + sudo gitlab-ctl promote-to-primary-node --force + ``` + +1. Verify you can connect to the newly promoted **primary** node using the URL used + previously for the **secondary** node. + + If successful, the **secondary** node has now been promoted to the **primary** node. + +### Next steps + +To regain geographic redundancy as quickly as possible, you should +[add a new **secondary** node](../setup/index.md). To +do that, you can re-add the old **primary** as a new secondary and bring it back +online. diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index c2a60b958d4..876904a2093 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -1196,9 +1196,16 @@ CAUTION: **Caution:** ## Data recovery -If a Gitaly node fails replication jobs for any reason, it ends up hosting outdated versions of -the affected repositories. Praefect provides tools for automatically or manually reconciling -the outdated repositories in order to bring them fully up to date again. +If a Gitaly node fails replication jobs for any reason, it ends up hosting outdated versions of the +affected repositories. Praefect provides tools for: + +- [Automatic](#automatic-reconciliation) reconciliation, for GitLab 13.4 and later. +- [Manual](#manual-reconciliation) reconciliation, for: + - GitLab 13.3 and earlier. + - Repositories upgraded to GitLab 13.4 and later without entries in the `repositories` table. + A migration tool [is planned](https://gitlab.com/gitlab-org/gitaly/-/issues/3033). + +These tools reconcile the outdated repositories to bring them fully up to date again. ### Automatic reconciliation diff --git a/doc/api/users.md b/doc/api/users.md index 26d9a79a165..634e0bd0842 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -2,8 +2,6 @@ ## List users -Active users = Total accounts - Blocked users - Get a list of users. This function takes pagination parameters `page` and `per_page` to restrict the list of users. @@ -49,9 +47,9 @@ For example: GET /users?username=jack_smith ``` -In addition, you can filter users based on states eg. `blocked`, `active` -This works only to filter users who are `blocked` or `active`. -It does not support `active=false` or `blocked=false`. +In addition, you can filter users based on the states `blocked` and `active`. +It does not support `active=false` or `blocked=false`. The list of active users +is the total number of users minus the blocked users. ```plaintext GET /users?active=true diff --git a/doc/development/README.md b/doc/development/README.md index 398aea55e72..abdd5c662f3 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -57,6 +57,7 @@ Complementary reads: - [Generate a changelog entry with `bin/changelog`](changelog.md) - [Requesting access to Chatops on GitLab.com](chatops_on_gitlabcom.md#requesting-access) (for GitLab team members) - [Patch release process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/patch/process.md#process-for-developers) +- [Adding a new service component to GitLab](adding_service_component.md) ## UX and Frontend guides diff --git a/doc/development/adding_service_component.md b/doc/development/adding_service_component.md new file mode 100644 index 00000000000..2801e27145d --- /dev/null +++ b/doc/development/adding_service_component.md @@ -0,0 +1,89 @@ +# Adding a new Service Component to GitLab + +The GitLab product is made up of several service components that run as independent system processes in communication with each other. These services can be run on the same instance, or spread across different instances. A list of the existing components can be found in the [GitLab architecture overview](architecture.md). + +## Integration phases + +The following outline re-uses the [maturity metric](https://about.gitlab.com/direction/maturity) naming as an example of the various phases of integrating a component. These are only loosely coupled to a components actual maturity, and are intended as a guide for implementation order (for example, a component does not need to be enabled by default to be Lovable, and being enabled by default does not on its own cause a component to be Lovable). + +- Proposed + - [Proposing a new component](#proposing-a-new-component) +- Minimal + - [Integrating a new service with GitLab](#integrating-a-new-service-with-gitlab) + - [Handling service dependencies](#handling-service-dependencies) +- Viable + - [Bundled with GitLab installations](#bundling-a-service-with-gitlab) + - [End-to-end testing in GitLab QA](testing_guide/end_to_end/beginners_guide.md) + - [Release management](#release-management) + - [Enabled on GitLab.com](feature_flags/controls.md#enabling-a-feature-for-gitlabcom) +- Complete + - [Configurable by the GitLab orchestrator](https://gitlab.com/gitlab-org/gitlab-orchestrator) +- Lovable + - Enabled by default for the majority of users + +## Proposing a new component + +The initial step for integrating a new component with GitLab starts with creating a [Feature proposal in the issue tracker](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Feature%20proposal). + +Identify the [product category](https://about.gitlab.com/handbook/product/categories/) the component falls under and assign the Engineering Manager and Product Manager responsible for that category. + +The general steps for getting any GitLab feature from proposal to release can be found in the [Product development flow](https://about.gitlab.com/handbook/product-development-flow/). + +## Integrating a new service with GitLab + +Adding a new service follows the same [merge request workflow](contributing/merge_request_workflow.md) as other contributions, and must meet the same [completion criteria](contributing/merge_request_workflow.md#definition-of-done) and in addition needs to cover the following: + +- The [architecture component list](architecture.md#component-list) has been updated to include the service. +- Features provided by the component have been accepted into the [GitLab Product Direction](https://about.gitlab.com/direction/). +- Documentation is available and the support team has been made aware of the new component. + +**For services that can operate completely separate from GitLab:** + +The first iteration should be to add the ability to connect and use the service as an externally installed component. Often this involves providing settings in GitLab to connect to the service, or allow connections from it. And then shipping documentation on how to install and configure the service with GitLab. + +TIP: **Tip:** +[Elasticsearch](../integration/elasticsearch.md#installing-elasticsearch) is an example of a service that has been integrated this way. And many of the other services, including internal projects like Gitaly, started off as separately installed alternatives. + +**For services that depend on the existing GitLab codebase:** + +The first iteration should be opt-in, either through the `gitlab.yml` configuration or through [feature flags](feature_flags.md). For these types of services it is often necessary to [bundle the service and its dependencies with GitLab](#bundling-a-service-with-gitlab) as part of the initial integration. + +TIP: **Tip:** +[ActionCable](https://docs.gitlab.com/omnibus/settings/actioncable.html) is an example of a service that has been added this way. + +## Bundling a service with GitLab + +NOTE: **Note:** +Code shipped with GitLab needs to use a license approved by the Legal team. See the list of [existing approved licenses](https://about.gitlab.com/handbook/engineering/open-source/#using-open-source-libraries). + +NOTE: **Note:** +Notify the [Distribution team](https://about.gitlab.com/handbook/engineering/development/enablement/distribution/) when adding a new dependency that must be compiled. We must be able to compile the dependency on all supported platforms. + +New services to be bundled with GitLab need to be available in the following environments. + +**Dev environment** + +The first step of bundling a new service is to provide it in the development environment to engage in collaboration and feedback. + +- [Include in the GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) +- [Include in the source install instructions](../install/installation.md) + +**Standard install methods** + +In order for a service to be bundled for end-users or GitLab.com, it needs to be included in the standard install methods: + +- [Included in the Omnibus package](https://gitlab.com/gitlab-org/omnibus-gitlab) +- [Included in the GitLab Helm charts](https://gitlab.com/gitlab-org/charts/gitlab) + +## Handling service dependencies + +Dependencies should be kept up to date and be tracked for security updates. For the Rails codebase, the JavaScript and Ruby dependencies are +scanned for vulnerabilities using GitLab [dependency scanning](../user/application_security/dependency_scanning/index.md). + +In addition, any system dependencies used in Omnibus packages or the Cloud Native images should be added to the [dependency update automation](https://about.gitlab.com/handbook/engineering/development/enablement/distribution/maintenance/dependencies.io.html#adding-new-dependencies). + +## Release management + +If the service component needs to be updated or released with the monthly GitLab release, then the component should be added to the [release tools automation](https://gitlab.com/gitlab-org/release-tools). This project is maintained by the [Delivery team](https://about.gitlab.com/handbook/engineering/infrastructure/team/delivery/). A list of the projects managed this way can be found in the [release tools project directory](https://about.gitlab.com/handbook/engineering/infrastructure/team/delivery/). + +For example, during the monthly GitLab release, the desired version of Gitaly, GitLab Workhorse, GitLab Shell, etc., need to synchronized through the various release pipelines. diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index 14f987ca87d..d88b159b666 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -242,6 +242,7 @@ request: 1. The [GitLab Development Kit](https://gitlab.com/gitlab-org/gitlab-development-kit). 1. The [CI environment preparation](https://gitlab.com/gitlab-org/gitlab/blob/master/scripts/prepare_build.sh). 1. The [Omnibus package creator](https://gitlab.com/gitlab-org/omnibus-gitlab). +1. The [Cloud Native GitLab Dockerfiles](https://gitlab.com/gitlab-org/build/CNG) ## Incremental improvements diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 09bb6b9da6a..984c64b9e9e 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -1120,16 +1120,14 @@ they need to interact with the application. When you take screenshots: - *Capture the most relevant area of the page.* Don't include unnecessary white - space or areas of the page that don't help illustrate your point. Also, don't - include the entire page if you don't have to, but also ensure the image - contains enough information to allow the user to determine where things are. -- *Be consistent.* Find a browser window size that works for you that also - displays all areas of the product, including the left navigation (usually > - 1200px wide). For consistency, use this browser window size for your - screenshots by installing a browser extension for setting a window to a - specific size (for example, - [Window Resizer](https://chrome.google.com/webstore/detail/window-resizer/kkelicaakdanhinjdeammmilcgefonfh/related?hl=en) - for Google Chrome). + space or areas of the page that don't help illustrate the point. The left + sidebar of the GitLab user interface can change, so don't include the sidebar + if it's not necessary. +- *Keep it small.* If you don't need to show the full width of the screen, don't. + A value of 1000 pixels is a good maximum width for your screenshot image. +- *Be consistent.* Coordinate screenshots with the other screenshots already on + a documentation page. For example, if other screenshots include the left + sidebar, include the sidebar in all screenshots. ### Save the image diff --git a/doc/development/feature_flags/development.md b/doc/development/feature_flags/development.md index b5a2da1b3c5..29bd0ca0a7e 100644 --- a/doc/development/feature_flags/development.md +++ b/doc/development/feature_flags/development.md @@ -110,7 +110,7 @@ Each feature flag is defined in a separate YAML file consisting of a number of f |---------------------|----------|----------------------------------------------------------------| | `name` | yes | Name of the feature flag. | | `type` | yes | Type of feature flag. | -| `default_enabled` | yes | The default state of the feature flag that is strongly validated, with `default_enabled:` passed as an argument. | +| `default_enabled` | yes | The default state of the feature flag that is strictly validated, with `default_enabled:` passed as an argument. | | `introduced_by_url` | no | The URL to the Merge Request that introduced the feature flag. | | `rollout_issue_url` | no | The URL to the Issue covering the feature flag rollout. | | `group` | no | The [group](https://about.gitlab.com/handbook/product/product-categories/#devops-stages) that owns the feature flag. | @@ -129,16 +129,16 @@ Only feature flags that have a YAML definition file can be used when running the ```shell $ bin/feature-flag my-feature-flag ->> Please specify the group introducing feature flag, like `group::apm`: +>> Specify the group introducing the feature flag, like `group::apm`: ?> group::memory ->> If you have MR open, can you paste the URL here? (or enter to skip) +>> URL of the MR introducing the feature flag (enter to skip): ?> https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38602 ->> Open this URL and fill the rest of details: +>> Open this URL and fill in the rest of the details: https://gitlab.com/gitlab-org/gitlab/-/issues/new?issue%5Btitle%5D=%5BFeature+flag%5D+Rollout+of+%60test-flag%60&issuable_template=Feature+Flag+Roll+Out ->> Paste URL of `rollout issue` here, or enter to skip: +>> URL of the rollout issue (enter to skip): ?> https://gitlab.com/gitlab-org/gitlab/-/issues/232533 create config/feature_flags/development/test-flag.yml --- @@ -305,7 +305,7 @@ used as an actor for `Feature.enabled?`. ### Feature flags for licensed features If a feature is license-gated, there's no need to add an additional -explicit feature flag check since the flag will be checked as part of the +explicit feature flag check since the flag is checked as part of the `License.feature_available?` call. Similarly, there's no need to "clean up" a feature flag once the feature has reached general availability. @@ -316,7 +316,7 @@ a by default enabled feature flag with the same name as the provided argument. **An important side-effect of the implicit feature flags mentioned above is that unless the feature is explicitly disabled or limited to a percentage of users, -the feature flag check will default to `true`.** +the feature flag check defaults to `true`.** NOTE: **Note:** Due to limitations with `feature_available?`, the YAML definition for `licensed` feature @@ -361,9 +361,9 @@ default_enabled: [false, true] Feature groups must be defined statically in `lib/feature.rb` (in the `.register_feature_groups` method), but their implementation can obviously be -dynamic (querying the DB etc.). +dynamic (querying the DB, for example). -Once defined in `lib/feature.rb`, you will be able to activate a +Once defined in `lib/feature.rb`, you can to activate a feature for a given feature group via the [`feature_group` parameter of the features API](../../api/features.md#set-or-create-a-feature) ### Enabling a feature flag locally (in development) @@ -374,7 +374,7 @@ In the rails console (`rails c`), enter the following command to enable a featur Feature.enable(:feature_flag_name) ``` -Similarly, the following command will disable a feature flag: +Similarly, the following command disables a feature flag: ```ruby Feature.disable(:feature_flag_name) @@ -388,7 +388,7 @@ Feature.enable(:feature_flag_name, Project.find_by_full_path("root/my-project")) ## Feature flags in tests -Introducing a feature flag into the codebase creates an additional codepath that should be tested. +Introducing a feature flag into the codebase creates an additional code path that should be tested. It is strongly advised to test all code affected by a feature flag, both when **enabled** and **disabled** to ensure the feature works properly. @@ -423,10 +423,10 @@ Feature.enabled?(:ci_live_trace, project2) # => false The behavior of FlipperGate is as follows: -1. You can enable an override for a specified actor to be enabled +1. You can enable an override for a specified actor to be enabled. 1. You can disable (remove) an override for a specified actor, - falling back to default state -1. There's no way to model that you explicitly disable a specified actor + falling back to the default state. +1. There's no way to model that you explicitly disabled a specified actor. ```ruby Feature.enable(:my_feature) @@ -467,7 +467,7 @@ Feature.enable_percentage_of_time(:my_feature_3, 50) Feature.enable_percentage_of_actors(:my_feature_4, 50) ``` -Each feature flag that has a defined state will be persisted +Each feature flag that has a defined state is persisted during test execution time: ```ruby diff --git a/doc/development/geo/framework.md b/doc/development/geo/framework.md index c999b19ab7d..b720a6ca47e 100644 --- a/doc/development/geo/framework.md +++ b/doc/development/geo/framework.md @@ -235,11 +235,10 @@ For example, to add support for files referenced by a `Widget` model with a `ee/lib/gitlab/geo.rb`: ```ruby - def self.replicator_classes - classes = [::Geo::PackageFileReplicator, - ::Geo::WidgetReplicator] - - classes.select(&:enabled?) + REPLICATOR_CLASSES = [ + ::Geo::PackageFileReplicator, + ::Geo::WidgetReplicator + ] end ``` @@ -315,10 +314,6 @@ For example, to add support for files referenced by a `Widget` model with a end ``` - The method `has_create_events?` should return `true` in most of the cases. - However, if the entity you add doesn't have the create event, don't add the - method at all. - 1. Update `REGISTRY_CLASSES` in `ee/app/workers/geo/secondary/registry_consistency_worker.rb`. 1. Add `widget_registry` to `ActiveSupport::Inflector.inflections` in `config/initializers_before_autoloader/000_inflections.rb`. @@ -435,7 +430,7 @@ for verification state to the widgets table: ``` 1. Add a partial index on `verification_failure` and `verification_checksum` to ensure - re-verification can be performed efficiently. Add a migration in `ee/db/geo/migrate/`: + re-verification can be performed efficiently: ```ruby # frozen_string_literal: true @@ -461,9 +456,9 @@ for verification state to the widgets table: ##### Option 2: Create a separate `widget_states` table with verification state fields -1. Add a migration in `ee/db/geo/migrate/` to create a `widget_states` table and add a - partial index on `verification_failure` and `verification_checksum` to ensure - re-verification can be performed efficiently. Order the columns according to [our guidelines](../ordering_table_columns.md): +1. Create a `widget_states` table and add a partial index on `verification_failure` and + `verification_checksum` to ensure re-verification can be performed efficiently. Order + the columns according to [our guidelines](../ordering_table_columns.md): ```ruby # frozen_string_literal: true diff --git a/doc/development/redis.md b/doc/development/redis.md index d5d42a3869e..d205082b9c6 100644 --- a/doc/development/redis.md +++ b/doc/development/redis.md @@ -1,10 +1,19 @@ # Redis guidelines -GitLab uses [Redis](https://redis.io) for three distinct purposes: +GitLab uses [Redis](https://redis.io) for the following distinct purposes: -- Caching via `Rails.cache`. +- Caching (mostly via `Rails.cache`). - As a job processing queue with [Sidekiq](sidekiq_style_guide.md). - To manage the shared application state. +- As a Pub/Sub queue backend for ActionCable. + +In most environments (including the GDK), all of these point to the same +Redis instance. + +On GitLab.com, we use [separate Redis +instances](../administration/redis/replication_and_failover.md#running-multiple-redis-clusters). +(We do not currently use [ActionCable on +GitLab.com](https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/228)). Every application process is configured to use the same Redis servers, so they can be used for inter-process communication in cases where [PostgreSQL](sql.md) @@ -21,11 +30,11 @@ to key names to avoid collisions. Typically we use colon-separated elements to provide a semblance of structure at application level. An example might be `projects:1:somekey`. -Although we split our Redis usage into three separate purposes, and those may -map to separate Redis servers in a [Highly Available](../administration/high_availability/redis.md) -configuration, the default Omnibus and GDK setups share a single Redis server. -This means that keys should **always** be globally unique across the three -purposes. +Although we split our Redis usage by purpose into distinct categories, and +those may map to separate Redis servers in a Highly Available +configuration like GitLab.com, the default Omnibus and GDK setups share +a single Redis server. This means that keys should **always** be +globally unique across all categories. It is usually better to use immutable identifiers - project ID rather than full path, for instance - in Redis key names. If full path is used, the key will @@ -56,3 +65,127 @@ Currently, we validate this in the development and test environments with the [`RedisClusterValidator`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/instrumentation/redis_cluster_validator.rb), which is enabled for the `cache` and `shared_state` [Redis instances](https://docs.gitlab.com/omnibus/settings/redis.html#running-with-multiple-redis-instances).. + +## Redis in structured logging + +Our [structured logging](logging.md#use-structured-json-logging) for web +requests and Sidekiq jobs contains fields for the duration, call count, +bytes written, and bytes read per Redis instance, along with a total for +all Redis instances. For a particular request, this might look like: + +| Field | Value | +| --- | --- | +| `json.queue_duration_s` | 0.01 | +| `json.redis_cache_calls` | 1 | +| `json.redis_cache_duration_s` | 0 | +| `json.redis_cache_read_bytes` | 109 | +| `json.redis_cache_write_bytes` | 49 | +| `json.redis_calls` | 2 | +| `json.redis_duration_s` | 0.001 | +| `json.redis_read_bytes` | 111 | +| `json.redis_shared_state_calls` | 1 | +| `json.redis_shared_state_duration_s` | 0 | +| `json.redis_shared_state_read_bytes` | 2 | +| `json.redis_shared_state_write_bytes` | 206 | +| `json.redis_write_bytes` | 255 | + +As all of these fields are indexed, it is then straightforward to +investigate Redis usage in production. For instance, to find the +requests that read the most data from the cache, we can just sort by +`redis_cache_read_bytes` in descending order. + +### The slow log + +On GitLab.com, entries from the [Redis +slow log](https://redis.io/commands/slowlog) are available in the +`pubsub-redis-inf-gprd*` index with the [`redis.slowlog` +tag](https://log.gprd.gitlab.net/app/kibana#/discover?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-1d,to:now))&_a=(columns:!(json.type,json.command,json.exec_time),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:AWSQX_Vf93rHTYrsexmk,key:json.tag,negate:!f,params:(query:redis.slowlog),type:phrase),query:(match:(json.tag:(query:redis.slowlog,type:phrase))))),index:AWSQX_Vf93rHTYrsexmk)). +This shows commands that have taken a long time and may be a performance +concern. + +The +[fluent-plugin-redis-slowlog](https://gitlab.com/gitlab-org/fluent-plugin-redis-slowlog) +project is responsible for taking the slowlog entries from Redis and +passing to fluentd (and ultimately Elasticsearch). + +## Analyzing the entire keyspace + +The [Redis Keyspace +Analyzer](https://gitlab.com/gitlab-com/gl-infra/redis-keyspace-analyzer) +project contains tools for dumping the full key list and memory usage of a Redis +instance, and then analyzing those lists while elimating potentially sensitive +data from the results. It can be used to find the most frequent key patterns, or +those that use the most memory. + +Currently this is not run automatically for the GitLab.com Redis instances, but +is run manually on an as-needed basis. + +## Utility classes + +We have some extra classes to help with specific use cases. These are +mostly for fine-grained control of Redis usage, so they wouldn't be used +in combination with the `Rails.cache` wrapper: we'd either use +`Rails.cache` or these classes and literal Redis commands. + +`Rails.cache` or these classes and literal Redis commands. We prefer +using `Rails.cache` so we can reap the benefits of future optimizations +done to Rails. It is worth noting that Ruby objects are +[marshalled](https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/cache/redis_cache_store.rb#L447) +when written to Redis, so we need to pay attention to not to store huge +objects, or untrusted user input. + +Typically we would only use these classes when at least one of the +following is true: + +1. We want to manipulate data on a non-cache Redis instance. +1. `Rails.cache` does not support the operations we want to perform. + +### `Gitlab::Redis::{Cache,SharedState,Queues}` + +These classes wrap the Redis instances (using +[`Gitlab::Redis::Wrapper`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/redis/wrapper.rb)) +to make it convenient to work with them directly. The typical use is to +call `.with` on the class, which takes a block that yields the Redis +connection. For example: + +```ruby +# Get the value of `key` from the shared state (persistent) Redis +Gitlab::Redis::SharedState.with { |redis| redis.get(key) } + +# Check if `value` is a member of the set `key` +Gitlab::Redis::Cache.with { |redis| redis.sismember(key, value) } +``` + +### `Gitlab::Redis::Boolean` + +In Redis, every value is a string. +[`Gitlab::Redis::Boolean`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/redis/boolean.rb) +makes sure that booleans are encoded and decoded consistently. + +### `Gitlab::Redis::HLL` + +The Redis [`PFCOUNT`](https://redis.io/commands/pfcount), +[`PFADD`](https://redis.io/commands/pfadd), and +[`PFMERGE`](https://redis.io/commands/pfmergge) commands operate on +HyperLogLogs, a data structure that allows estimating the number of unique +elements with low memory usage. (In addition to the `PFCOUNT` documentation, +Thoughtbot's article on [HyperLogLogs in +Redis](https://thoughtbot.com/blog/hyperloglogs-in-redis) provides a good +background here.) + +[`Gitlab::Redis::HLL`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/redis/hll.rb) +provides a convenient interface for adding and counting values in HyperLogLogs. + +### `Gitlab::SetCache` + +For cases where we need to efficiently check the whether an item is in a group +of items, we can use a Redis set. +[`Gitlab::SetCache`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/set_cache.rb) +provides an `#include?` method that will use the +[`SISMEMBER`](https://redis.io/commands/sismember) command, as well as `#read` +to fetch all entries in the set. + +This is used by the +[`RepositorySetCache`](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/repository_set_cache.rb) +to provide a convenient way to use sets to cache repository data like branch +names. diff --git a/doc/integration/gitpod.md b/doc/integration/gitpod.md new file mode 100644 index 00000000000..f26483e3b5e --- /dev/null +++ b/doc/integration/gitpod.md @@ -0,0 +1,74 @@ +--- +type: reference, how-to +stage: Create +group: Editor +info: "To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers" +--- + +# Gitpod Integration + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/228893) in GitLab 13.4. +> - It's [deployed behind a feature flag](#enable-or-disable-the-gitpod-integration), disabled by default. +> - It's enabled on GitLab.com. +> - It's recommended for production use. +> - To use it in GitLab self-managed instances, ask a GitLab administrator to [enable it](#configure-your-gitlab-instance-with-gitpod). **(CORE ONLY)** + +CAUTION: **Warning:** +This feature might not be available to you. Check the **version history** note above for details. + +With [Gitpod](https://gitpod.io/) you can describe your dev environment as code to get fully set +up, compiled, and tested dev environments for any GitLab project. The dev environments are not only +automated but also prebuilt which means that Gitpod continuously builds your Git branches like a CI +server. By that you don’t have to wait for dependencies to be downloaded and builds to finish, but +you can start coding immediately. + +In short: With Gitpod you can start coding instantly on any project, branch, and merge request from +any device, at any time. + + + +You can launch Gitpod directly from GitLab by clicking the **Gitpod** button from the **Web IDE** +dropdown on the project page: + + + +To learn more about Gitpod, see their [features](https://www.gitpod.io/features/) and +[documentation](https://www.gitpod.io/docs/). + +To use the GitLab-Gitpod integration, you need to enable it from your user preferences: + +1. From the GitLab UI, click your avatar in the top-right corner, then click **Settings**. +1. On the left-hand nav, click **Preferences**. +1. Under **Integrations**, find the **Gitpod** section. +1. Check **Enable Gitpod**. + +Users of GitLab.com can enable it and start using straightaway. Users of GitLab self-managed instances +can follow the same steps once the integration has been enabled and configured by a GitLab administrator. + +## Configure your GitLab instance with Gitpod **(CORE ONLY)** + +If you are new to Gitpod, head over to the [Gitpod documentation](https://www.gitpod.io/docs/self-hosted/latest/self-hosted/) +and get your instance up and running. + +1. In GitLab, go to **Admin Area > Settings > Integrations**. +1. Expand the **Gitpod** configuration section. +1. Check **Enable Gitpod**. +1. Add your Gitpod instance URL (for example, `https://gitpod.example.com`). + +## Enable or disable the Gitpod integration **(CORE ONLY)** + +The Gitpod integration is under development and not ready for production use. It is deployed behind a +feature flag that is **disabled by default**. +[GitLab administrators with access to the GitLab Rails console](../administration/feature_flags.md) +can enable it. + +To enable it: + +```ruby +Feature.enable(:gitpod) +``` + +To disable it: + +```ruby +Feature.disable(:gitpod) diff --git a/doc/integration/img/gitpod_button_project_page_v13_4.png b/doc/integration/img/gitpod_button_project_page_v13_4.png Binary files differnew file mode 100644 index 00000000000..55a70d89169 --- /dev/null +++ b/doc/integration/img/gitpod_button_project_page_v13_4.png diff --git a/doc/integration/img/gitpod_web_interface_v13_4.png b/doc/integration/img/gitpod_web_interface_v13_4.png Binary files differnew file mode 100644 index 00000000000..5cd9a6aad0f --- /dev/null +++ b/doc/integration/img/gitpod_web_interface_v13_4.png diff --git a/doc/user/application_security/offline_deployments/index.md b/doc/user/application_security/offline_deployments/index.md index ea7c5e23d2a..3a7c0148388 100644 --- a/doc/user/application_security/offline_deployments/index.md +++ b/doc/user/application_security/offline_deployments/index.md @@ -64,10 +64,10 @@ Once a vulnerability is found, you can interact with it. Read more on how to Please note that in some cases the reported vulnerabilities provide metadata that can contain external links exposed in the UI. These links might not be accessible within an offline environment. -### Suggested Solutions for vulnerabilities +### Automatic remediation for vulnerabilities -The [suggested solutions](../index.md#solutions-for-vulnerabilities-auto-remediation) feature -(auto-remediation) is available for Dependency Scanning and Container Scanning, but may not work +The [automatic remediation for vulnerabilities](../index.md#solutions-for-vulnerabilities-auto-remediation) feature +(auto-remediation) is available for offline Dependency Scanning and Container Scanning, but may not work depending on your instance's configuration. We can only suggest solutions, which are generally more current versions that have been patched, when we are able to access up-to-date registry services hosting the latest versions of that dependency or image. diff --git a/doc/user/profile/preferences.md b/doc/user/profile/preferences.md index 03bb1bf677e..f84fc1ae898 100644 --- a/doc/user/profile/preferences.md +++ b/doc/user/profile/preferences.md @@ -182,6 +182,12 @@ Manage the availability of integrated code intelligence features powered by Sourcegraph. View [the Sourcegraph feature documentation](../../integration/sourcegraph.md#enable-sourcegraph-in-user-preferences) for more information. +### Gitpod + +Enable and disable the [GitLab-Gitpod integration](../../integration/gitpod.md). This is only +visible after the integration is configured by a GitLab administrator. View +[the Gitpod feature documentation](../../integration/gitpod.md) for more information. + <!-- ## Troubleshooting Include any troubleshooting steps that you can foresee. If you know beforehand what issues diff --git a/doc/user/project/import/bitbucket_server.md b/doc/user/project/import/bitbucket_server.md index 8ecc0e1f9c4..d0499730bfe 100644 --- a/doc/user/project/import/bitbucket_server.md +++ b/doc/user/project/import/bitbucket_server.md @@ -37,7 +37,12 @@ Import your projects from Bitbucket Server to GitLab with minimal effort. empty changes. 1. Attachments in Markdown are currently not imported. 1. Task lists are not imported. -1. Emoji reactions are not imported +1. Emoji reactions are not imported. +1. [LFS objects](../../../topics/git/lfs/index.md) are not imported. + + NOTE: **Note:** + To import a repository including LFS objects from a Bitbucket server repository, use the [Repo by URL](../import/repo_by_url.md) importer. + 1. Project filtering does not support fuzzy search (only `starts with` or `full match strings` are currently supported) diff --git a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md index 403067ba0cd..1d0299637bd 100644 --- a/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md +++ b/doc/user/project/merge_requests/reviewing_and_managing_merge_requests.md @@ -289,6 +289,17 @@ the command line. NOTE: **Note:** This section might move in its own document in the future. +### Copy the branch name for local checkout + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/23767) in GitLab 13.4. + +The merge request sidebar contains the branch reference for the source branch +used to contribute changes for this merge request. + +To copy the branch reference into your clipboard, click the **Copy branch name** button +(**{copy-to-clipboard}**) in the right sidebar. You can then use it to checkout the branch locally +via command line by running `git checkout <branch-name>`. + ### Checkout merge requests locally through the `head` ref A merge request contains all the history from a repository, plus the additional diff --git a/doc/user/project/settings/project_access_tokens.md b/doc/user/project/settings/project_access_tokens.md index 597d8dfe3f1..ef32b0dbb18 100644 --- a/doc/user/project/settings/project_access_tokens.md +++ b/doc/user/project/settings/project_access_tokens.md @@ -15,10 +15,7 @@ type: reference, howto > - It's recommended for production use. > - For GitLab self-managed instances, GitLab administrators can [disable it](#enable-or-disable-project-access-tokens). -Project access tokens are scoped to a project and can be used to authenticate with the [GitLab API](../../../api/README.md#personalproject-access-tokens). - -<!-- Commented out until https://gitlab.com/gitlab-org/gitlab/-/issues/219551 is fixed --> -<!-- You can also use project access tokens with Git to authenticate over HTTP or SSH. --> +Project access tokens are scoped to a project and can be used to authenticate with the [GitLab API](../../../api/README.md#personalproject-access-tokens). You can also use project access tokens with Git to authenticate over HTTP or SSH. Project access tokens expire on the date you define, at midnight UTC. diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index d31e42e270e..821b42af049 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -53,21 +53,42 @@ If you are missing Syntax Highlighting support for any language, we prepared a s NOTE: **Note:** Single file editing is based on the [Ace Editor](https://ace.c9.io). -### Schema based validation +### Themes + +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2389) in GitLab in 13.0. +> - Full Solarized Dark Theme [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219228) in GitLab 13.1. + +All the themes GitLab supports for syntax highlighting are added to the Web IDE's code editor. +You can pick a theme from your [profile preferences](../../profile/preferences.md). + +The themes are available only in the Web IDE file editor, except for the [dark theme](https://gitlab.com/gitlab-org/gitlab/-/issues/209808) and +the [solarized dark theme](https://gitlab.com/gitlab-org/gitlab/-/issues/219228), +which apply to the entire Web IDE screen. + +| Solarized Light Theme | Solarized Dark Theme | Dark Theme | +|---------------------------------------------------------------|-------------------------------------------------------------|-----------------------------------------| +|  |  |  | + +## Schema based validation -> - Support for `.gitlab-ci.yml` validation [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218472) in GitLab 13.2. +> - Support for validation based on predefined schemas [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/218472) in GitLab 13.2. > - It was deployed behind a feature flag, disabled by default. > - It's enabled on GitLab.com. > - It cannot be enabled or disabled per-project. -> - For GitLab self-managed instances, GitLab administrators can opt to [enable it](#enable-or-disable-schema-based-validation). +> - For GitLab self-managed instances, GitLab administrators can opt to [enable it](#enable-or-disable-validation-based-on-predefined-schemas). +> - Support for validation based on custom schemas [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/226982) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. The Web IDE provides validation support for certain JSON and YAML files using schemas -based on the [JSON Schema Store](https://www.schemastore.org/json/). This feature is -only supported for the `.gitlab-ci.yml` file. +based on the [JSON Schema Store](https://www.schemastore.org/json/). -#### Enable or disable Schema based validation **(CORE ONLY)** +### Predefined schemas -Schema based validation is under development and not ready for production use. It is +The Web IDE has validation for certain files built in. This feature is only supported for +the `*.gitlab-ci.yml` files. + +#### Enable or disable validation based on predefined schemas **(CORE ONLY)** + +Validation based on predefined schemas is under development and not ready for production use. It is deployed behind a feature flag that is **disabled by default** for self-managed instances, [GitLab administrators with access to the GitLab Rails console](../../../administration/feature_flags.md) can enable it for your instance. @@ -84,21 +105,35 @@ To disable it: Feature.disable(:schema_linting) ``` -### Themes +### Custom schemas **(PREMIUM)** -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/2389) in GitLab in 13.0. -> - Full Solarized Dark Theme [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/219228) in GitLab 13.1. +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/226982) in [GitLab Premium](https://about.gitlab.com/pricing/) 13.4. -All the themes GitLab supports for syntax highlighting are added to the Web IDE's code editor. -You can pick a theme from your [profile preferences](../../profile/preferences.md). +The Web IDE also allows you to define custom schemas for certain JSON/YAML files in your project. +You can do so by defining a `schemas` entry in the `.gitlab/.gitlab-webide.yml` file inside the +repository's root. Here is an example configuration: -The themes are available only in the Web IDE file editor, except for the [dark theme](https://gitlab.com/gitlab-org/gitlab/-/issues/209808) and -the [solarized dark theme](https://gitlab.com/gitlab-org/gitlab/-/issues/219228), -which apply to the entire Web IDE screen. +```yaml +schemas: + - uri: https://json.schemastore.org/package + match: + - package.json + - uri: https://somewebsite.com/first/raw/url + match: + - data/release_posts/unreleased/*.{yml,yaml} + - uri: https://somewebsite.com/second/raw/url + match: + - "*.meta.json" +``` -| Solarized Light Theme | Solarized Dark Theme | Dark Theme | -|---------------------------------------------------------------|-------------------------------------------------------------|-----------------------------------------| -|  |  |  | +Each schema entry supports two properties: + +- `uri`: please provide an absolute URL for the schema definition file here. The schema from this URL +is loaded when a matching file is open. +- `match`: a list of matching paths or glob expressions. If a schema matches a particular path pattern, +it will be applied to that file. Please enclose the pattern in quotes if it begins with an asterisk (`*`), +it's be applied to that file. If a pattern begins with an asterisk (`*`), enclose it in quotation +marks. Otherwise, the configuration file is not valid YAML. ## Configure the Web IDE diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 0ccdd3936e4..6e5534d0c9a 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -61,6 +61,10 @@ module API end optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' + optional :gitpod_enabled, type: Boolean, desc: 'Enable Gitpod' + given gitpod_enabled: ->(val) { val } do + requires :gitpod_url, type: String, desc: 'The configured Gitpod instance URL' + end optional :gitaly_timeout_default, type: Integer, desc: 'Default Gitaly timeout, in seconds. Set to 0 to disable timeouts.' optional :gitaly_timeout_fast, type: Integer, desc: 'Gitaly fast operation timeout, in seconds. Set to 0 to disable timeouts.' optional :gitaly_timeout_medium, type: Integer, desc: 'Medium Gitaly timeout, in seconds. Set to 0 to disable timeouts.' @@ -150,6 +154,7 @@ module API end optional :issues_create_limit, type: Integer, desc: "Maximum number of issue creation requests allowed per minute per user. Set to 0 for unlimited requests per minute." optional :raw_blob_request_limit, type: Integer, desc: "Maximum number of requests per minute for each raw path. Set to 0 for unlimited requests per minute." + optional :wiki_page_max_content_bytes, type: Integer, desc: "Maximum wiki page content size in bytes" ApplicationSetting::SUPPORTED_KEY_TYPES.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb index 9ac92aab637..c485c23f3be 100644 --- a/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb +++ b/lib/gitlab/background_migration/set_null_package_files_file_store_to_local_value.rb @@ -11,9 +11,11 @@ module Gitlab class SetNullPackageFilesFileStoreToLocalValue LOCAL_STORE = 1 # equal to ObjectStorage::Store::LOCAL - # Temporary AR class for package files - class PackageFile < ActiveRecord::Base - self.table_name = 'packages_package_files' + module Packages + # Temporary AR class for package files + class PackageFile < ActiveRecord::Base + self.table_name = 'packages_package_files' + end end def perform(start_id, stop_id) diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index db6acb3f5aa..82a7d5fb83c 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -6,17 +6,21 @@ module Gitlab class Metrics extend Gitlab::Utils::StrongMemoize - OPERATIONS = [:appended, :mutated, :overwrite, :accepted, - :finalized, :discarded, :flaky].freeze + OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite, + :accepted, :finalized, :discarded, :conflict].freeze def increment_trace_operation(operation: :unknown) unless OPERATIONS.include?(operation) - raise ArgumentError, 'unknown trace operation' + raise ArgumentError, "unknown trace operation: #{operation}" end self.class.trace_operations.increment(operation: operation) end + def increment_trace_bytes(size) + self.class.trace_bytes.increment(by: size.to_i) + end + def self.trace_operations strong_memoize(:trace_operations) do name = :gitlab_ci_trace_operations_total @@ -25,6 +29,15 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def self.trace_bytes + strong_memoize(:trace_bytes) do + name = :gitlab_ci_trace_bytes_total + comment = 'Total amount of build trace bytes transferred' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index 20f5620dd64..618438c8887 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -8,7 +8,7 @@ module Gitlab BUFFER_SIZE = 4096 LIMIT_SIZE = 500.kilobytes - attr_reader :stream + attr_reader :stream, :metrics delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true @@ -16,9 +16,10 @@ module Gitlab alias_method :present?, :valid? - def initialize + def initialize(metrics = Trace::Metrics.new) @stream = yield @stream&.binmode + @metrics = metrics end def valid? @@ -43,6 +44,9 @@ module Gitlab def append(data, offset) data = data.force_encoding(Encoding::BINARY) + metrics.increment_trace_operation(operation: :streamed) + metrics.increment_trace_bytes(data.bytesize) + stream.seek(offset, IO::SEEK_SET) stream.write(data) stream.truncate(offset + data.bytesize) diff --git a/lib/gitlab/consul/internal.rb b/lib/gitlab/consul/internal.rb index 8d99ea9f3c0..3afc24ddab9 100644 --- a/lib/gitlab/consul/internal.rb +++ b/lib/gitlab/consul/internal.rb @@ -35,19 +35,12 @@ module Gitlab [service_address, service_port] end - def discover_prometheus_uri + def discover_prometheus_server_address service_address, service_port = discover_service(service_name: 'prometheus') return unless service_address && service_port - # There really is not a way to discover whether a Prometheus connection is using TLS or not - # Try TLS first because HTTPS will return fast if failed. - %w[https http].find do |scheme| - connection_url = "#{scheme}://#{service_address}:#{service_port}" - break connection_url if Gitlab::PrometheusClient.new(connection_url, allow_local_requests: true).healthy? - rescue - nil - end + "#{service_address}:#{service_port}" end private diff --git a/lib/gitlab/gitpod.rb b/lib/gitlab/gitpod.rb new file mode 100644 index 00000000000..11b54db72ea --- /dev/null +++ b/lib/gitlab/gitpod.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + class Gitpod + class << self + def feature_conditional? + feature.conditional? + end + + def feature_available? + # The gitpod_bundle feature could be conditionally applied, so check if `!off?` + !feature.off? + end + + def feature_enabled?(actor = nil) + feature.enabled?(actor) + end + + def feature_and_settings_enabled?(actor = nil) + feature_enabled?(actor) && Gitlab::CurrentSettings.gitpod_enabled + end + + private + + def feature + Feature.get(:gitpod) # rubocop:disable Gitlab/AvoidFeatureGet + end + end + end +end diff --git a/lib/gitlab/prometheus/internal.rb b/lib/gitlab/prometheus/internal.rb index d59352119ba..c2f4035821e 100644 --- a/lib/gitlab/prometheus/internal.rb +++ b/lib/gitlab/prometheus/internal.rb @@ -25,6 +25,10 @@ module Gitlab end end + def self.server_address + uri&.strip&.sub(/^http[s]?:\/\//, '') + end + def self.listen_address Gitlab.config.prometheus.listen_address.to_s if Gitlab.config.prometheus rescue Settingslogic::MissingSetting diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 56e1154a672..965349ad711 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -42,6 +42,15 @@ module Gitlab response_body == HEALTHY_RESPONSE end + def ready? + response = get(ready_url, {}) + + # From Prometheus docs: This endpoint returns 200 when Prometheus is ready to serve traffic (i.e. respond to queries). + response.code == 200 + rescue => e + raise PrometheusClient::UnexpectedResponseError, "#{e.message}" + end + def proxy(type, args) path = api_path(type) get(path, args) @@ -103,7 +112,11 @@ module Gitlab end def health_url - [api_url, '-/healthy'].join('/') + "#{api_url}/-/healthy" + end + + def ready_url + "#{api_url}/-/ready" end private diff --git a/lib/gitlab/relative_positioning.rb b/lib/gitlab/relative_positioning.rb new file mode 100644 index 00000000000..b5a923f0824 --- /dev/null +++ b/lib/gitlab/relative_positioning.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + STEPS = 10 + IDEAL_DISTANCE = 2**(STEPS - 1) + 1 + + MIN_POSITION = Gitlab::Database::MIN_INT_VALUE + START_POSITION = 0 + MAX_POSITION = Gitlab::Database::MAX_INT_VALUE + + MAX_GAP = IDEAL_DISTANCE * 2 + MIN_GAP = 2 + + NoSpaceLeft = Class.new(StandardError) + end +end diff --git a/lib/gitlab/relative_positioning/gap.rb b/lib/gitlab/relative_positioning/gap.rb new file mode 100644 index 00000000000..ab894141a60 --- /dev/null +++ b/lib/gitlab/relative_positioning/gap.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +# +module Gitlab + module RelativePositioning + class Gap + attr_reader :start_pos, :end_pos + + def initialize(start_pos, end_pos) + @start_pos, @end_pos = start_pos, end_pos + end + + def ==(other) + other.is_a?(self.class) && other.start_pos == start_pos && other.end_pos == end_pos + end + + def delta + ((start_pos - end_pos) / 2.0).abs.ceil.clamp(0, RelativePositioning::IDEAL_DISTANCE) + end + end + end +end diff --git a/lib/gitlab/relative_positioning/item_context.rb b/lib/gitlab/relative_positioning/item_context.rb new file mode 100644 index 00000000000..cd03a347355 --- /dev/null +++ b/lib/gitlab/relative_positioning/item_context.rb @@ -0,0 +1,259 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + # This class is API private - it should not be explicitly instantiated + # outside of tests + # rubocop: disable CodeReuse/ActiveRecord + class ItemContext + include Gitlab::Utils::StrongMemoize + + attr_reader :object, :model_class, :range + attr_accessor :ignoring + + def initialize(object, range, ignoring: nil) + @object = object + @range = range + @model_class = object.class + @ignoring = ignoring + end + + def ==(other) + other.is_a?(self.class) && other.object == object && other.range == range && other.ignoring == ignoring + end + + def positioned? + relative_position.present? + end + + def min_relative_position + strong_memoize(:min_relative_position) { calculate_relative_position('MIN') } + end + + def max_relative_position + strong_memoize(:max_relative_position) { calculate_relative_position('MAX') } + end + + def prev_relative_position + calculate_relative_position('MAX') { |r| nextify(r, false) } if object.relative_position + end + + def next_relative_position + calculate_relative_position('MIN') { |r| nextify(r) } if object.relative_position + end + + def nextify(relation, gt = true) + if gt + relation.where("relative_position > ?", relative_position) + else + relation.where("relative_position < ?", relative_position) + end + end + + def relative_siblings(relation = scoped_items) + object.exclude_self(relation) + end + + # Handles the possibility that the position is already occupied by a sibling + def place_at_position(position, lhs) + current_occupant = relative_siblings.find_by(relative_position: position) + + if current_occupant.present? + Mover.new(position, range).move(object, lhs.object, current_occupant) + else + object.relative_position = position + end + end + + def lhs_neighbour + scoped_items + .where('relative_position < ?', relative_position) + .reorder(relative_position: :desc) + .first + .then { |x| neighbour(x) } + end + + def rhs_neighbour + scoped_items + .where('relative_position > ?', relative_position) + .reorder(relative_position: :asc) + .first + .then { |x| neighbour(x) } + end + + def neighbour(item) + return unless item.present? + + self.class.new(item, range, ignoring: ignoring) + end + + def scoped_items + r = model_class.relative_positioning_query_base(object) + r = object.exclude_self(r, excluded: ignoring) if ignoring.present? + r + end + + def calculate_relative_position(calculation) + # When calculating across projects, this is much more efficient than + # MAX(relative_position) without the GROUP BY, due to index usage: + # https://gitlab.com/gitlab-org/gitlab-foss/issues/54276#note_119340977 + relation = scoped_items + .order(Gitlab::Database.nulls_last_order('position', 'DESC')) + .group(grouping_column) + .limit(1) + + relation = yield relation if block_given? + + relation + .pluck(grouping_column, Arel.sql("#{calculation}(relative_position) AS position")) + .first&.last + end + + def grouping_column + model_class.relative_positioning_parent_column + end + + def max_sibling + sib = relative_siblings + .order(Gitlab::Database.nulls_last_order('relative_position', 'DESC')) + .first + + neighbour(sib) + end + + def min_sibling + sib = relative_siblings + .order(Gitlab::Database.nulls_last_order('relative_position', 'ASC')) + .first + + neighbour(sib) + end + + def shift_left + move_sequence_before(true) + object.reset + end + + def shift_right + move_sequence_after(true) + object.reset + end + + def create_space_left + find_next_gap_before.tap { |gap| move_sequence_before(false, next_gap: gap) } + end + + def create_space_right + find_next_gap_after.tap { |gap| move_sequence_after(false, next_gap: gap) } + end + + def find_next_gap_before + items_with_next_pos = scoped_items + .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position DESC) AS next_pos') + .where('relative_position <= ?', relative_position) + .order(relative_position: :desc) + + find_next_gap(items_with_next_pos, range.first) + end + + def find_next_gap_after + items_with_next_pos = scoped_items + .select('relative_position AS pos, LEAD(relative_position) OVER (ORDER BY relative_position ASC) AS next_pos') + .where('relative_position >= ?', relative_position) + .order(:relative_position) + + find_next_gap(items_with_next_pos, range.last) + end + + def find_next_gap(items_with_next_pos, default_end) + gap = model_class + .from(items_with_next_pos, :items) + .where('next_pos IS NULL OR ABS(pos::bigint - next_pos::bigint) >= ?', MIN_GAP) + .limit(1) + .pluck(:pos, :next_pos) + .first + + return if gap.nil? || gap.first == default_end + + Gap.new(gap.first, gap.second || default_end) + end + + def relative_position + object.relative_position + end + + private + + # Moves the sequence before the current item to the middle of the next gap + # For example, we have + # + # 5 . . . . . 11 12 13 14 [15] 16 . 17 + # ----------- + # + # This moves the sequence [11 12 13 14] to [8 9 10 11], so we have: + # + # 5 . . 8 9 10 11 . . . [15] 16 . 17 + # --------- + # + # Creating a gap to the left of the current item. We can understand this as + # dividing the 5 spaces between 5 and 11 into two smaller gaps of 2 and 3. + # + # If `include_self` is true, the current item will also be moved, creating a + # gap to the right of the current item: + # + # 5 . . 8 9 10 11 [14] . . . 16 . 17 + # -------------- + # + # As an optimization, the gap can be precalculated and passed to this method. + # + # @api private + # @raises NoSpaceLeft if the sequence cannot be moved + def move_sequence_before(include_self = false, next_gap: find_next_gap_before) + raise NoSpaceLeft unless next_gap.present? + + delta = next_gap.delta + + move_sequence(next_gap.start_pos, relative_position, -delta, include_self) + end + + # Moves the sequence after the current item to the middle of the next gap + # For example, we have: + # + # 8 . 10 [11] 12 13 14 15 . . . . . 21 + # ----------- + # + # This moves the sequence [12 13 14 15] to [15 16 17 18], so we have: + # + # 8 . 10 [11] . . . 15 16 17 18 . . 21 + # ----------- + # + # Creating a gap to the right of the current item. We can understand this as + # dividing the 5 spaces between 15 and 21 into two smaller gaps of 3 and 2. + # + # If `include_self` is true, the current item will also be moved, creating a + # gap to the left of the current item: + # + # 8 . 10 . . . [14] 15 16 17 18 . . 21 + # ---------------- + # + # As an optimization, the gap can be precalculated and passed to this method. + # + # @api private + # @raises NoSpaceLeft if the sequence cannot be moved + def move_sequence_after(include_self = false, next_gap: find_next_gap_after) + raise NoSpaceLeft unless next_gap.present? + + delta = next_gap.delta + + move_sequence(relative_position, next_gap.start_pos, delta, include_self) + end + + def move_sequence(start_pos, end_pos, delta, include_self = false) + relation = include_self ? scoped_items : relative_siblings + + object.update_relative_siblings(relation, (start_pos..end_pos), delta) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/lib/gitlab/relative_positioning/mover.rb b/lib/gitlab/relative_positioning/mover.rb new file mode 100644 index 00000000000..9d891bfbe3b --- /dev/null +++ b/lib/gitlab/relative_positioning/mover.rb @@ -0,0 +1,155 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + class Mover + attr_reader :range, :start_position + + def initialize(start, range) + @range = range + @start_position = start + end + + def move_to_end(object) + focus = context(object, ignoring: object) + max_pos = focus.max_relative_position + + move_to_range_end(focus, max_pos) + end + + def move_to_start(object) + focus = context(object, ignoring: object) + min_pos = focus.min_relative_position + + move_to_range_start(focus, min_pos) + end + + def move(object, first, last) + raise ArgumentError, 'object is required' unless object + + lhs = context(first, ignoring: object) + rhs = context(last, ignoring: object) + focus = context(object) + range = RelativePositioning.range(lhs, rhs) + + if range.cover?(focus) + # Moving a object already within a range is a no-op + elsif range.open_on_left? + move_to_range_start(focus, range.rhs.relative_position) + elsif range.open_on_right? + move_to_range_end(focus, range.lhs.relative_position) + else + pos_left, pos_right = create_space_between(range) + desired_position = position_between(pos_left, pos_right) + focus.place_at_position(desired_position, range.lhs) + end + end + + def context(object, ignoring: nil) + return unless object + + ItemContext.new(object, range, ignoring: ignoring) + end + + private + + def gap_too_small?(pos_a, pos_b) + return false unless pos_a && pos_b + + (pos_a - pos_b).abs < MIN_GAP + end + + def move_to_range_end(context, max_pos) + range_end = range.last + 1 + + new_pos = if max_pos.nil? + start_position + elsif gap_too_small?(max_pos, range_end) + max = context.max_sibling + max.ignoring = context.object + max.shift_left + position_between(max.relative_position, range_end) + else + position_between(max_pos, range_end) + end + + context.object.relative_position = new_pos + end + + def move_to_range_start(context, min_pos) + range_end = range.first - 1 + + new_pos = if min_pos.nil? + start_position + elsif gap_too_small?(min_pos, range_end) + sib = context.min_sibling + sib.ignoring = context.object + sib.shift_right + position_between(sib.relative_position, range_end) + else + position_between(min_pos, range_end) + end + + context.object.relative_position = new_pos + end + + def create_space_between(range) + pos_left = range.lhs&.relative_position + pos_right = range.rhs&.relative_position + + return [pos_left, pos_right] unless gap_too_small?(pos_left, pos_right) + + gap = range.rhs.create_space_left + [pos_left - gap.delta, pos_right] + rescue NoSpaceLeft + gap = range.lhs.create_space_right + [pos_left, pos_right + gap.delta] + end + + # This method takes two integer values (positions) and + # calculates the position between them. The range is huge as + # the maximum integer value is 2147483647. + # + # We avoid open ranges by clamping the range to [MIN_POSITION, MAX_POSITION]. + # + # Then we handle one of three cases: + # - If the gap is too small, we raise NoSpaceLeft + # - If the gap is larger than MAX_GAP, we place the new position at most + # IDEAL_DISTANCE from the edge of the gap. + # - otherwise we place the new position at the midpoint. + # + # The new position will always satisfy: pos_before <= midpoint <= pos_after + # + # As a precondition, the gap between pos_before and pos_after MUST be >= 2. + # If the gap is too small, NoSpaceLeft is raised. + # + # @raises NoSpaceLeft + def position_between(pos_before, pos_after) + pos_before ||= range.first + pos_after ||= range.last + + pos_before, pos_after = [pos_before, pos_after].sort + + gap_width = pos_after - pos_before + + if gap_too_small?(pos_before, pos_after) + raise NoSpaceLeft + elsif gap_width > MAX_GAP + if pos_before <= range.first + pos_after - IDEAL_DISTANCE + elsif pos_after >= range.last + pos_before + IDEAL_DISTANCE + else + midpoint(pos_before, pos_after) + end + else + midpoint(pos_before, pos_after) + end + end + + def midpoint(lower_bound, upper_bound) + ((lower_bound + upper_bound) / 2.0).ceil.clamp(lower_bound, upper_bound - 1) + end + end + end +end diff --git a/lib/gitlab/relative_positioning/range.rb b/lib/gitlab/relative_positioning/range.rb new file mode 100644 index 00000000000..174d5ef4b35 --- /dev/null +++ b/lib/gitlab/relative_positioning/range.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module RelativePositioning + IllegalRange = Class.new(ArgumentError) + + class Range + attr_reader :lhs, :rhs + + def open_on_left? + lhs.nil? + end + + def open_on_right? + rhs.nil? + end + + def cover?(item_context) + return false unless item_context + return false unless item_context.positioned? + return true if item_context.object == lhs&.object + return true if item_context.object == rhs&.object + + pos = item_context.relative_position + + return lhs.relative_position < pos if open_on_right? + return pos < rhs.relative_position if open_on_left? + + lhs.relative_position < pos && pos < rhs.relative_position + end + + def ==(other) + other.is_a?(RelativePositioning::Range) && lhs == other.lhs && rhs == other.rhs + end + end + + def self.range(lhs, rhs) + if lhs && rhs + ClosedRange.new(lhs, rhs) + elsif lhs + StartingFrom.new(lhs) + elsif rhs + EndingAt.new(rhs) + else + raise IllegalRange, 'One of rhs or lhs must be provided' unless lhs && rhs + end + end + + class ClosedRange < RelativePositioning::Range + def initialize(lhs, rhs) + @lhs, @rhs = lhs, rhs + raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs + raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs + end + end + + class StartingFrom < RelativePositioning::Range + include Gitlab::Utils::StrongMemoize + + def initialize(lhs) + @lhs = lhs + raise IllegalRange, 'lhs is required' unless lhs + end + + def rhs + strong_memoize(:rhs) { lhs.rhs_neighbour } + end + end + + class EndingAt < RelativePositioning::Range + include Gitlab::Utils::StrongMemoize + + def initialize(rhs) + @rhs = rhs + raise IllegalRange, 'rhs is required' unless rhs + end + + def lhs + strong_memoize(:lhs) { rhs.lhs_neighbour } + end + end + end +end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index b8a4eedd620..e1a87a77f04 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -230,8 +230,10 @@ module Gitlab end def rss_increase_by_jobs - Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord - rss_increase_by_job(job) + Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do + Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord + rss_increase_by_job(job) + end end end diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb index ec404ebd309..231d5aea129 100644 --- a/lib/gitlab/sourcegraph.rb +++ b/lib/gitlab/sourcegraph.rb @@ -12,8 +12,8 @@ module Gitlab !feature.off? end - def feature_enabled?(thing = nil) - feature.enabled?(thing) + def feature_enabled?(actor = nil) + feature.enabled?(actor) end private diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b7d3ffb0232..89605ce5d07 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -815,11 +815,9 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord - # rubocop: disable UsageData/DistinctCountByLargeForeignKey def cluster_applications_user_distinct_count(applications, time_period) distinct_count(applications.where(time_period).available.joins(:cluster), 'clusters.user_id') end - # rubocop: enable UsageData/DistinctCountByLargeForeignKey def clusters_user_distinct_count(clusters, time_period) distinct_count(clusters.where(time_period), :user_id) diff --git a/lib/gitlab/usage_data/topology.rb b/lib/gitlab/usage_data/topology.rb index facecec547f..7f7854c3eb1 100644 --- a/lib/gitlab/usage_data/topology.rb +++ b/lib/gitlab/usage_data/topology.rb @@ -40,7 +40,7 @@ module Gitlab private def topology_fetch_all_data - with_prometheus_client(fallback: {}) do |client| + with_prometheus_client(fallback: {}, verify: false) do |client| { application_requests_per_hour: topology_app_requests_per_hour(client), query_apdex_weekly_average: topology_query_apdex_weekly_average(client), diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events.yml index 375195ed38d..3650ceda5be 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events.yml @@ -120,6 +120,10 @@ - name: merge_request_action category: source_code aggregation: daily +- name: i_source_code_code_intelligence + redis_slot: source_code + category: source_code + aggregation: daily # Incident management - name: incident_management_alert_status_changed redis_slot: incident_management diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 03ddb0eb5fb..3c998987427 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -83,11 +83,11 @@ module Gitlab end end - def with_prometheus_client(fallback: nil) - api_url = prometheus_api_url - return fallback unless api_url + def with_prometheus_client(fallback: nil, verify: true) + client = prometheus_client(verify: verify) + return fallback unless client - yield Gitlab::PrometheusClient.new(api_url, allow_local_requests: true) + yield client end def measure_duration @@ -111,11 +111,27 @@ module Gitlab private - def prometheus_api_url + def prometheus_client(verify:) + server_address = prometheus_server_address + + return unless server_address + + # There really is not a way to discover whether a Prometheus connection is using TLS or not + # Try TLS first because HTTPS will return fast if failed. + %w[https http].find do |scheme| + api_url = "#{scheme}://#{server_address}" + client = Gitlab::PrometheusClient.new(api_url, allow_local_requests: true, verify: verify) + break client if client.ready? + rescue + nil + end + end + + def prometheus_server_address if Gitlab::Prometheus::Internal.prometheus_enabled? - Gitlab::Prometheus::Internal.uri + Gitlab::Prometheus::Internal.server_address elsif Gitlab::Consul::Internal.api_url - Gitlab::Consul::Internal.discover_prometheus_uri + Gitlab::Consul::Internal.discover_prometheus_server_address end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 9082f355494..fb397729dec 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -9240,9 +9240,6 @@ msgstr "" msgid "Edit files in the editor and commit changes here" msgstr "" -msgid "Edit fork in Web IDE" -msgstr "" - msgid "Edit group: %{group_name}" msgstr "" @@ -9420,9 +9417,18 @@ msgstr "" msgid "Enable" msgstr "" +msgid "Enable %{gitpod_link} integration to launch a development environment in your browser directly from GitLab." +msgstr "" + msgid "Enable Auto DevOps" msgstr "" +msgid "Enable Gitpod" +msgstr "" + +msgid "Enable Gitpod?" +msgstr "" + msgid "Enable HTML emails" msgstr "" @@ -11996,6 +12002,21 @@ msgstr "" msgid "Gitlab Pages" msgstr "" +msgid "Gitpod" +msgstr "" + +msgid "Gitpod|Add the URL to your Gitpod instance configured to read your GitLab projects." +msgstr "" + +msgid "Gitpod|Enable Gitpod integration" +msgstr "" + +msgid "Gitpod|Gitpod URL" +msgstr "" + +msgid "Gitpod|e.g. https://gitpod.example.com" +msgstr "" + msgid "Given access %{time_ago}" msgstr "" @@ -14594,6 +14615,9 @@ msgstr "" msgid "Latest pipeline for the most recent commit on this branch" msgstr "" +msgid "Launch a ready-to-code development environment for your project." +msgstr "" + msgid "Lead" msgstr "" @@ -20679,6 +20703,9 @@ msgstr "" msgid "Quick range" msgstr "" +msgid "Quickly and easily edit multiple files in your project." +msgstr "" + msgid "README" msgstr "" @@ -26561,6 +26588,9 @@ msgstr "" msgid "To update Snippets with multiple files, you must use the `files` parameter" msgstr "" +msgid "To use Gitpod you must first enable the feature in the integrations section of your %{user_prefs}." +msgstr "" + msgid "To view all %{scannedResourcesCount} scanned URLs, please download the CSV file" msgstr "" @@ -30892,6 +30922,9 @@ msgstr "" msgid "user avatar" msgstr "" +msgid "user preferences" +msgstr "" + msgid "username" msgstr "" diff --git a/qa/Dockerfile b/qa/Dockerfile index 2b7c4a19122..6310e4b290d 100644 --- a/qa/Dockerfile +++ b/qa/Dockerfile @@ -3,8 +3,8 @@ LABEL maintainer="GitLab Quality Department <quality@gitlab.com>" ENV DEBIAN_FRONTEND="noninteractive" ENV DOCKER_VERSION="17.09.0-ce" -ENV CHROME_VERSION="85.0.4183.83-1" -ENV CHROME_DRIVER_VERSION="85.0.4183.87" +ENV CHROME_VERSION="84.0.4147.89-1" +ENV CHROME_DRIVER_VERSION="84.0.4147.30" ENV CHROME_DEB="google-chrome-stable_${CHROME_VERSION}_amd64.deb" ENV CHROME_URL="https://s3.amazonaws.com/gitlab-google-chrome-stable/${CHROME_DEB}" diff --git a/qa/qa/page/project/show.rb b/qa/qa/page/project/show.rb index 22c2ed2a0c2..c607b35005e 100644 --- a/qa/qa/page/project/show.rb +++ b/qa/qa/page/project/show.rb @@ -56,7 +56,7 @@ module QA element :new_file_option end - view 'app/assets/javascripts/repository/components/web_ide_link.vue' do + view 'app/assets/javascripts/vue_shared/components/web_ide_link.vue' do element :web_ide_button end diff --git a/rubocop/cop/usage_data/distinct_count_by_large_foreign_key.rb b/rubocop/cop/usage_data/distinct_count_by_large_foreign_key.rb index e28dabfe39f..36bcda527e8 100644 --- a/rubocop/cop/usage_data/distinct_count_by_large_foreign_key.rb +++ b/rubocop/cop/usage_data/distinct_count_by_large_foreign_key.rb @@ -31,11 +31,11 @@ module RuboCop private def allowed_foreign_key?(key) - [:sym, :str].include?(key.type) && allowed_foreign_keys.include?(key.value.to_sym) + [:sym, :str].include?(key.type) && allowed_foreign_keys.include?(key.value.to_s) end def allowed_foreign_keys - cop_config['AllowedForeignKeys'] || [] + (cop_config['AllowedForeignKeys'] || []).map(&:to_s) end end end diff --git a/rubocop/rubocop-usage-data.yml b/rubocop/rubocop-usage-data.yml index bcdb0631090..9f594bc5817 100644 --- a/rubocop/rubocop-usage-data.yml +++ b/rubocop/rubocop-usage-data.yml @@ -38,12 +38,13 @@ UsageData/DistinctCountByLargeForeignKey: - 'lib/gitlab/usage_data.rb' - 'ee/lib/ee/gitlab/usage_data.rb' AllowedForeignKeys: - - :user_id - - :author_id - - :creator_id - - :owner_id - - :project_id - - :issue_id - - :merge_request_id - - :merge_requests.target_project_id - - :agent_id + - 'agent_id' + - 'author_id' + - 'clusters.user_id' + - 'creator_id' + - 'issue_id' + - 'merge_request_id' + - 'merge_requests.target_project_id' + - 'owner_id' + - 'project_id' + - 'user_id' diff --git a/spec/bin/feature_flag_spec.rb b/spec/bin/feature_flag_spec.rb index 92d8fbddd44..41117880f95 100644 --- a/spec/bin/feature_flag_spec.rb +++ b/spec/bin/feature_flag_spec.rb @@ -135,7 +135,7 @@ RSpec.describe 'bin/feature-flag' do expect($stdin).to receive(:gets).and_return(type) expect do expect(described_class.read_type).to eq(:development) - end.to output(/specify the type/).to_stdout + end.to output(/Specify the feature flag type/).to_stdout end context 'when invalid type is given' do @@ -147,7 +147,7 @@ RSpec.describe 'bin/feature-flag' do expect do expect { described_class.read_type }.to raise_error(/EOF/) - end.to output(/specify the type/).to_stdout + end.to output(/Specify the feature flag type/).to_stdout .and output(/Invalid type specified/).to_stderr end end @@ -161,7 +161,7 @@ RSpec.describe 'bin/feature-flag' do expect($stdin).to receive(:gets).and_return(group) expect do expect(described_class.read_group).to eq('group::memory') - end.to output(/specify the group/).to_stdout + end.to output(/Specify the group introducing the feature flag/).to_stdout end context 'invalid group given' do @@ -173,8 +173,8 @@ RSpec.describe 'bin/feature-flag' do expect do expect { described_class.read_group }.to raise_error(/EOF/) - end.to output(/specify the group/).to_stdout - .and output(/Group needs to include/).to_stderr + end.to output(/Specify the group introducing the feature flag/).to_stdout + .and output(/The group needs to include/).to_stderr end end end @@ -186,7 +186,7 @@ RSpec.describe 'bin/feature-flag' do expect($stdin).to receive(:gets).and_return(url) expect do expect(described_class.read_introduced_by_url).to eq('https://merge-request') - end.to output(/can you paste the URL here/).to_stdout + end.to output(/URL of the MR introducing the feature flag/).to_stdout end context 'empty URL given' do @@ -196,7 +196,7 @@ RSpec.describe 'bin/feature-flag' do expect($stdin).to receive(:gets).and_return(url) expect do expect(described_class.read_introduced_by_url).to be_nil - end.to output(/can you paste the URL here/).to_stdout + end.to output(/URL of the MR introducing the feature flag/).to_stdout end end @@ -209,7 +209,7 @@ RSpec.describe 'bin/feature-flag' do expect do expect { described_class.read_introduced_by_url }.to raise_error(/EOF/) - end.to output(/can you paste the URL here/).to_stdout + end.to output(/URL of the MR introducing the feature flag/).to_stdout .and output(/URL needs to start with/).to_stderr end end @@ -223,7 +223,7 @@ RSpec.describe 'bin/feature-flag' do expect($stdin).to receive(:gets).and_return(url) expect do expect(described_class.read_rollout_issue_url(options)).to eq('https://issue') - end.to output(/Paste URL of `rollout issue` here/).to_stdout + end.to output(/URL of the rollout issue/).to_stdout end context 'invalid URL given' do @@ -235,7 +235,7 @@ RSpec.describe 'bin/feature-flag' do expect do expect { described_class.read_rollout_issue_url(options) }.to raise_error(/EOF/) - end.to output(/Paste URL of `rollout issue` here/).to_stdout + end.to output(/URL of the rollout issue/).to_stdout .and output(/URL needs to start/).to_stderr end end diff --git a/spec/features/admin/admin_settings_spec.rb b/spec/features/admin/admin_settings_spec.rb index dbe79e9a1b7..38f0b813183 100644 --- a/spec/features/admin/admin_settings_spec.rb +++ b/spec/features/admin/admin_settings_spec.rb @@ -17,7 +17,10 @@ RSpec.describe 'Admin updates settings', :clean_gitlab_redis_shared_state, :do_n end context 'General page' do + let(:gitpod_feature_enabled) { true } + before do + stub_feature_flags(gitpod: gitpod_feature_enabled) visit general_admin_application_settings_path end @@ -205,6 +208,32 @@ RSpec.describe 'Admin updates settings', :clean_gitlab_redis_shared_state, :do_n expect(page).to have_content "Application settings saved successfully" expect(current_settings.terminal_max_session_time).to eq(15) end + + context 'Configure Gitpod' do + context 'with feature disabled' do + let(:gitpod_feature_enabled) { false } + + it 'do not show settings' do + expect(page).not_to have_selector('#js-gitpod-settings') + end + end + + context 'with feature enabled' do + let(:gitpod_feature_enabled) { true } + + it 'changes gitpod settings' do + page.within('#js-gitpod-settings') do + check 'Enable Gitpod integration' + fill_in 'Gitpod URL', with: 'https://gitpod.test/' + click_button 'Save changes' + end + + expect(page).to have_content 'Application settings saved successfully' + expect(current_settings.gitpod_url).to eq('https://gitpod.test/') + expect(current_settings.gitpod_enabled).to be(true) + end + end + end end context 'Integrations page' do diff --git a/spec/frontend/design_management/components/design_notes/__snapshots__/design_note_spec.js.snap b/spec/frontend/design_management/components/design_notes/__snapshots__/design_note_spec.js.snap index e1db42aeb0b..8711083ad8c 100644 --- a/spec/frontend/design_management/components/design_notes/__snapshots__/design_note_spec.js.snap +++ b/spec/frontend/design_management/components/design_notes/__snapshots__/design_note_spec.js.snap @@ -68,7 +68,7 @@ exports[`Design note component should match the snapshot 1`] = ` </div> <div - class="note-text js-note-text" + class="note-text js-note-text md" data-qa-selector="note_content" /> diff --git a/spec/frontend/repository/components/web_ide_link_spec.js b/spec/frontend/repository/components/web_ide_link_spec.js deleted file mode 100644 index 877756db364..00000000000 --- a/spec/frontend/repository/components/web_ide_link_spec.js +++ /dev/null @@ -1,51 +0,0 @@ -import { mount } from '@vue/test-utils'; -import WebIdeLink from '~/repository/components/web_ide_link.vue'; - -describe('Web IDE link component', () => { - let wrapper; - - function createComponent(props) { - wrapper = mount(WebIdeLink, { - propsData: { ...props }, - mocks: { - $route: { - params: {}, - }, - }, - }); - } - - afterEach(() => { - wrapper.destroy(); - }); - - it('renders link to the Web IDE for a project if only projectPath is given', () => { - createComponent({ projectPath: 'gitlab-org/gitlab', refSha: 'master' }); - - expect(wrapper.attributes('href')).toBe('/-/ide/project/gitlab-org/gitlab/edit/master/-/'); - expect(wrapper.text()).toBe('Web IDE'); - }); - - it('renders link to the Web IDE for a project even if both projectPath and forkPath are given', () => { - createComponent({ - projectPath: 'gitlab-org/gitlab', - refSha: 'master', - forkPath: 'my-namespace/gitlab', - }); - - expect(wrapper.attributes('href')).toBe('/-/ide/project/gitlab-org/gitlab/edit/master/-/'); - expect(wrapper.text()).toBe('Web IDE'); - }); - - it('renders link to the forked project if it exists and cannot write to the repo', () => { - createComponent({ - projectPath: 'gitlab-org/gitlab', - refSha: 'master', - forkPath: 'my-namespace/gitlab', - canPushCode: false, - }); - - expect(wrapper.attributes('href')).toBe('/-/ide/project/my-namespace/gitlab/edit/master/-/'); - expect(wrapper.text()).toBe('Edit fork in Web IDE'); - }); -}); diff --git a/spec/frontend/vue_shared/components/actions_button_spec.js b/spec/frontend/vue_shared/components/actions_button_spec.js new file mode 100644 index 00000000000..4dde9d726d1 --- /dev/null +++ b/spec/frontend/vue_shared/components/actions_button_spec.js @@ -0,0 +1,203 @@ +import { shallowMount } from '@vue/test-utils'; +import { GlDropdown, GlLink } from '@gitlab/ui'; +import { createMockDirective, getBinding } from 'helpers/vue_mock_directive'; +import ActionsButton from '~/vue_shared/components/actions_button.vue'; + +const TEST_ACTION = { + key: 'action1', + text: 'Sample', + secondaryText: 'Lorem ipsum.', + tooltip: '', + href: '/sample', + attrs: { 'data-test': '123' }, +}; +const TEST_ACTION_2 = { + key: 'action2', + text: 'Sample 2', + secondaryText: 'Dolar sit amit.', + tooltip: 'Dolar sit amit.', + href: '#', + attrs: { 'data-test': '456' }, +}; +const TEST_TOOLTIP = 'Lorem ipsum dolar sit'; + +describe('Actions button component', () => { + let wrapper; + + function createComponent(props) { + wrapper = shallowMount(ActionsButton, { + propsData: { ...props }, + directives: { GlTooltip: createMockDirective() }, + }); + } + + afterEach(() => { + wrapper.destroy(); + }); + + const getTooltip = child => { + const directiveBinding = getBinding(child.element, 'gl-tooltip'); + + return directiveBinding.value; + }; + const findLink = () => wrapper.find(GlLink); + const findLinkTooltip = () => getTooltip(findLink()); + const findDropdown = () => wrapper.find(GlDropdown); + const findDropdownTooltip = () => getTooltip(findDropdown()); + const parseDropdownItems = () => + findDropdown() + .findAll('gl-dropdown-item-stub,gl-dropdown-divider-stub') + .wrappers.map(x => { + if (x.is('gl-dropdown-divider-stub')) { + return { type: 'divider' }; + } + + const { isCheckItem, isChecked, secondaryText } = x.props(); + + return { + type: 'item', + isCheckItem, + isChecked, + secondaryText, + text: x.text(), + }; + }); + const clickOn = (child, evt = new Event('click')) => child.vm.$emit('click', evt); + const clickLink = (...args) => clickOn(findLink(), ...args); + const clickDropdown = (...args) => clickOn(findDropdown(), ...args); + + describe('with 1 action', () => { + beforeEach(() => { + createComponent({ actions: [TEST_ACTION] }); + }); + + it('should not render dropdown', () => { + expect(findDropdown().exists()).toBe(false); + }); + + it('should render single button', () => { + const link = findLink(); + + expect(link.attributes()).toEqual({ + class: expect.any(String), + href: TEST_ACTION.href, + ...TEST_ACTION.attrs, + }); + expect(link.text()).toBe(TEST_ACTION.text); + }); + + it('should have tooltip', () => { + expect(findLinkTooltip()).toBe(TEST_ACTION.tooltip); + }); + + it('should have attrs', () => { + expect(findLink().attributes()).toMatchObject(TEST_ACTION.attrs); + }); + + it('can click', () => { + expect(clickLink).not.toThrow(); + }); + }); + + describe('with 1 action with tooltip', () => { + it('should have tooltip', () => { + createComponent({ actions: [{ ...TEST_ACTION, tooltip: TEST_TOOLTIP }] }); + + expect(findLinkTooltip()).toBe(TEST_TOOLTIP); + }); + }); + + describe('with 1 action with handle', () => { + it('can click and trigger handle', () => { + const handleClick = jest.fn(); + createComponent({ actions: [{ ...TEST_ACTION, handle: handleClick }] }); + + const event = new Event('click'); + clickLink(event); + + expect(handleClick).toHaveBeenCalledWith(event); + }); + }); + + describe('with multiple actions', () => { + let handleAction; + + beforeEach(() => { + handleAction = jest.fn(); + + createComponent({ actions: [{ ...TEST_ACTION, handle: handleAction }, TEST_ACTION_2] }); + }); + + it('should default to selecting first action', () => { + expect(findDropdown().attributes()).toMatchObject({ + text: TEST_ACTION.text, + 'split-href': TEST_ACTION.href, + }); + }); + + it('should handle first action click', () => { + const event = new Event('click'); + + clickDropdown(event); + + expect(handleAction).toHaveBeenCalledWith(event); + }); + + it('should render dropdown items', () => { + expect(parseDropdownItems()).toEqual([ + { + type: 'item', + isCheckItem: true, + isChecked: true, + secondaryText: TEST_ACTION.secondaryText, + text: TEST_ACTION.text, + }, + { type: 'divider' }, + { + type: 'item', + isCheckItem: true, + isChecked: false, + secondaryText: TEST_ACTION_2.secondaryText, + text: TEST_ACTION_2.text, + }, + ]); + }); + + it('should select action 2 when clicked', () => { + expect(wrapper.emitted('select')).toBeUndefined(); + + const action2 = wrapper.find(`[data-testid="action_${TEST_ACTION_2.key}"]`); + action2.vm.$emit('click'); + + expect(wrapper.emitted('select')).toEqual([[TEST_ACTION_2.key]]); + }); + + it('should have tooltip value', () => { + expect(findDropdownTooltip()).toBe(TEST_ACTION.tooltip); + }); + }); + + describe('with multiple actions and selectedKey', () => { + beforeEach(() => { + createComponent({ actions: [TEST_ACTION, TEST_ACTION_2], selectedKey: TEST_ACTION_2.key }); + }); + + it('should show action 2 as selected', () => { + expect(parseDropdownItems()).toEqual([ + expect.objectContaining({ + type: 'item', + isChecked: false, + }), + { type: 'divider' }, + expect.objectContaining({ + type: 'item', + isChecked: true, + }), + ]); + }); + + it('should have tooltip value', () => { + expect(findDropdownTooltip()).toBe(TEST_ACTION_2.tooltip); + }); + }); +}); diff --git a/spec/frontend/vue_shared/components/web_ide_link_spec.js b/spec/frontend/vue_shared/components/web_ide_link_spec.js new file mode 100644 index 00000000000..57f511903d9 --- /dev/null +++ b/spec/frontend/vue_shared/components/web_ide_link_spec.js @@ -0,0 +1,106 @@ +import { shallowMount } from '@vue/test-utils'; +import LocalStorageSync from '~/vue_shared/components/local_storage_sync.vue'; +import WebIdeLink from '~/vue_shared/components/web_ide_link.vue'; +import ActionsButton from '~/vue_shared/components/actions_button.vue'; + +const TEST_WEB_IDE_URL = '/-/ide/project/gitlab-test/test/edit/master/-/'; +const TEST_GITPOD_URL = 'https://gitpod.test/'; + +const ACTION_WEB_IDE = { + href: TEST_WEB_IDE_URL, + key: 'webide', + secondaryText: 'Quickly and easily edit multiple files in your project.', + tooltip: '', + text: 'Web IDE', + attrs: { + 'data-qa-selector': 'web_ide_button', + }, +}; +const ACTION_WEB_IDE_FORK = { + ...ACTION_WEB_IDE, + href: '#modal-confirm-fork', + handle: expect.any(Function), +}; +const ACTION_GITPOD = { + href: TEST_GITPOD_URL, + key: 'gitpod', + secondaryText: 'Launch a ready-to-code development environment for your project.', + tooltip: 'Launch a ready-to-code development environment for your project.', + text: 'Gitpod', + attrs: { + 'data-qa-selector': 'gitpod_button', + }, +}; +const ACTION_GITPOD_ENABLE = { + ...ACTION_GITPOD, + href: '#modal-enable-gitpod', + handle: expect.any(Function), +}; + +describe('Web IDE link component', () => { + let wrapper; + + function createComponent(props) { + wrapper = shallowMount(WebIdeLink, { + propsData: { + webIdeUrl: TEST_WEB_IDE_URL, + gitpodUrl: TEST_GITPOD_URL, + ...props, + }, + }); + } + + afterEach(() => { + wrapper.destroy(); + }); + + const findActionsButton = () => wrapper.find(ActionsButton); + const findLocalStorageSync = () => wrapper.find(LocalStorageSync); + + it.each` + props | expectedActions + ${{}} | ${[ACTION_WEB_IDE]} + ${{ needsToFork: true }} | ${[ACTION_WEB_IDE_FORK]} + ${{ showWebIdeButton: false, showGitpodButton: true, gitpodEnabled: true }} | ${[ACTION_GITPOD]} + ${{ showWebIdeButton: false, showGitpodButton: true, gitpodEnabled: false }} | ${[ACTION_GITPOD_ENABLE]} + ${{ showGitpodButton: true, gitpodEnabled: false }} | ${[ACTION_WEB_IDE, ACTION_GITPOD_ENABLE]} + `('renders actions with props=$props', ({ props, expectedActions }) => { + createComponent(props); + + expect(findActionsButton().props('actions')).toEqual(expectedActions); + }); + + describe('with multiple actions', () => { + beforeEach(() => { + createComponent({ showWebIdeButton: true, showGitpodButton: true, gitpodEnabled: true }); + }); + + it('selected Web IDE by default', () => { + expect(findActionsButton().props()).toMatchObject({ + actions: [ACTION_WEB_IDE, ACTION_GITPOD], + selectedKey: ACTION_WEB_IDE.key, + }); + }); + + it('should set selection with local storage value', async () => { + expect(findActionsButton().props('selectedKey')).toBe(ACTION_WEB_IDE.key); + + findLocalStorageSync().vm.$emit('input', ACTION_GITPOD.key); + + await wrapper.vm.$nextTick(); + + expect(findActionsButton().props('selectedKey')).toBe(ACTION_GITPOD.key); + }); + + it('should update local storage when selection changes', async () => { + expect(findLocalStorageSync().props('value')).toBe(ACTION_WEB_IDE.key); + + findActionsButton().vm.$emit('select', ACTION_GITPOD.key); + + await wrapper.vm.$nextTick(); + + expect(findActionsButton().props('selectedKey')).toBe(ACTION_GITPOD.key); + expect(findLocalStorageSync().props('value')).toBe(ACTION_GITPOD.key); + }); + }); +}); diff --git a/spec/helpers/tree_helper_spec.rb b/spec/helpers/tree_helper_spec.rb index 307479744ef..97b6802dde9 100644 --- a/spec/helpers/tree_helper_spec.rb +++ b/spec/helpers/tree_helper_spec.rb @@ -156,21 +156,36 @@ RSpec.describe TreeHelper do end describe '#vue_file_list_data' do - before do - allow(helper).to receive(:current_user).and_return(nil) - end - it 'returns a list of attributes related to the project' do expect(helper.vue_file_list_data(project, sha)).to include( - can_push_code: nil, - fork_path: nil, - escaped_ref: sha, - ref: sha, project_path: project.full_path, project_short_path: project.path, + ref: sha, + escaped_ref: sha, full_name: project.name_with_namespace ) end + end + + describe '#vue_ide_link_data' do + before do + allow(helper).to receive(:current_user).and_return(nil) + allow(helper).to receive(:can_collaborate_with_project?).and_return(true) + allow(helper).to receive(:can?).and_return(true) + end + + subject { helper.vue_ide_link_data(project, sha) } + + it 'returns a list of attributes related to the project' do + expect(subject).to include( + ide_base_path: project.full_path, + needs_to_fork: false, + show_web_ide_button: true, + show_gitpod_button: false, + gitpod_url: "", + gitpod_enabled: nil + ) + end context 'user does not have write access but a personal fork exists' do include ProjectForksHelper @@ -185,9 +200,9 @@ RSpec.describe TreeHelper do allow(helper).to receive(:current_user).and_return(user) end - it 'includes fork_path too' do - expect(helper.vue_file_list_data(project, sha)).to include( - fork_path: forked_project.full_path + it 'includes ide_base_path: forked_project.full_path' do + expect(subject).to include( + ide_base_path: forked_project.full_path ) end end @@ -201,9 +216,54 @@ RSpec.describe TreeHelper do allow(helper).to receive(:current_user).and_return(user) end - it 'includes can_push_code: true' do - expect(helper.vue_file_list_data(project, sha)).to include( - can_push_code: "true" + it 'includes ide_base_path: project.full_path' do + expect(subject).to include( + ide_base_path: project.full_path + ) + end + end + + context 'gitpod feature is enabled' do + let_it_be(:user) { create(:user) } + + before do + stub_feature_flags(gitpod: true) + allow(Gitlab::CurrentSettings) + .to receive(:gitpod_enabled) + .and_return(true) + + allow(helper).to receive(:current_user).and_return(user) + end + + it 'has show_gitpod_button: true' do + expect(subject).to include( + show_gitpod_button: true + ) + end + + it 'has gitpod_enabled: true when user has enabled gitpod' do + user.gitpod_enabled = true + + expect(subject).to include( + gitpod_enabled: true + ) + end + + it 'has gitpod_enabled: false when user has not enabled gitpod' do + user.gitpod_enabled = false + + expect(subject).to include( + gitpod_enabled: false + ) + end + + it 'has show_gitpod_button: false when web ide button is not shown' do + allow(helper).to receive(:can_collaborate_with_project?).and_return(false) + allow(helper).to receive(:can?).and_return(false) + + expect(subject).to include( + show_web_ide_button: false, + show_gitpod_button: false ) end end diff --git a/spec/lib/gitlab/ci/trace/stream_spec.rb b/spec/lib/gitlab/ci/trace/stream_spec.rb index e28469c9404..d65b6fb41f6 100644 --- a/spec/lib/gitlab/ci/trace/stream_spec.rb +++ b/spec/lib/gitlab/ci/trace/stream_spec.rb @@ -151,6 +151,28 @@ RSpec.describe Gitlab::Ci::Trace::Stream, :clean_gitlab_redis_cache do it_behaves_like 'appends' end + + describe 'metrics' do + let(:metrics) { spy('metrics') } + let(:io) { StringIO.new } + let(:stream) { described_class.new(metrics) { io } } + + it 'increments trace streamed operation' do + stream.append(+'123456', 0) + + expect(metrics) + .to have_received(:increment_trace_operation) + .with(operation: :streamed) + end + + it 'increments trace bytes counter' do + stream.append(+'123456', 0) + + expect(metrics) + .to have_received(:increment_trace_bytes) + .with(6) + end + end end describe '#set' do diff --git a/spec/lib/gitlab/consul/internal_spec.rb b/spec/lib/gitlab/consul/internal_spec.rb index d57713bd8ba..5889dd8b41d 100644 --- a/spec/lib/gitlab/consul/internal_spec.rb +++ b/spec/lib/gitlab/consul/internal_spec.rb @@ -116,44 +116,16 @@ RSpec.describe Gitlab::Consul::Internal do it_behaves_like 'handles failure response' end - describe '.discover_prometheus_uri' do - subject { described_class.discover_prometheus_uri } + describe '.discover_prometheus_server_address' do + subject { described_class.discover_prometheus_server_address } before do stub_consul_discover_prometheus .to_return(status: 200, body: '[{"ServiceAddress":"prom.net","ServicePort":9090}]') - stub_request(:get, /\/-\/healthy/) - .to_return(status: 200, body: Gitlab::PrometheusClient::HEALTHY_RESPONSE) end - context 'both TLS and non-TLS connection are healthy' do - it 'returns https uri' do - is_expected.to eq('https://prom.net:9090') - end - end - - context 'TLS connection is not healthy' do - before do - stub_request(:get, /https:\/\/.*\/-\/healthy/) - .to_return(status: 200, body: 'failed') - end - - it 'returns http uri' do - is_expected.to eq('http://prom.net:9090') - end - end - - context 'neither TLS nor non-TLS connection is healthy' do - before do - stub_request(:get, /https:\/\/.*\/-\/healthy/) - .to_return(status: 200, body: 'failed') - stub_request(:get, /http:\/\/.*\/-\/healthy/) - .to_return(status: 200, body: 'failed') - end - - it 'returns nil' do - is_expected.to be_nil - end + it 'returns the server address' do + is_expected.to eq('prom.net:9090') end it_behaves_like 'returns nil given blank value of', :api_url diff --git a/spec/lib/gitlab/gitpod_spec.rb b/spec/lib/gitlab/gitpod_spec.rb new file mode 100644 index 00000000000..f4dda42aeb4 --- /dev/null +++ b/spec/lib/gitlab/gitpod_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Gitpod do + let_it_be(:user) { create(:user) } + let(:feature_scope) { true } + + before do + stub_feature_flags(gitpod: feature_scope) + end + + describe '.feature_conditional?' do + subject { described_class.feature_conditional? } + + context 'when feature is enabled globally' do + it { is_expected.to be_falsey } + end + + context 'when feature is enabled only to a resource' do + let(:feature_scope) { user } + + it { is_expected.to be_truthy } + end + end + + describe '.feature_available?' do + subject { described_class.feature_available? } + + context 'when feature is enabled globally' do + it { is_expected.to be_truthy } + end + + context 'when feature is enabled only to a resource' do + let(:feature_scope) { user } + + it { is_expected.to be_truthy } + end + end + + describe '.feature_enabled?' do + let(:current_user) { nil } + + subject { described_class.feature_enabled?(current_user) } + + context 'when feature is enabled globally' do + it { is_expected.to be_truthy } + end + + context 'when feature is enabled only to a resource' do + let(:feature_scope) { user } + + context 'for the same resource' do + let(:current_user) { user } + + it { is_expected.to be_truthy } + end + + context 'for a different resource' do + let(:current_user) { create(:user) } + + it { is_expected.to be_falsey } + end + end + end +end diff --git a/spec/lib/gitlab/prometheus/internal_spec.rb b/spec/lib/gitlab/prometheus/internal_spec.rb index f3fbf0cafe6..7771d85222a 100644 --- a/spec/lib/gitlab/prometheus/internal_spec.rb +++ b/spec/lib/gitlab/prometheus/internal_spec.rb @@ -48,7 +48,7 @@ RSpec.describe Gitlab::Prometheus::Internal do let(:listen_address) { nil } it 'does not fail' do - expect(described_class.uri).to eq(nil) + expect(described_class.uri).to be_nil end end @@ -56,7 +56,27 @@ RSpec.describe Gitlab::Prometheus::Internal do let(:listen_address) { '' } it 'does not configure prometheus' do - expect(described_class.uri).to eq(nil) + expect(described_class.uri).to be_nil + end + end + end + + describe '.server_address' do + context 'self.uri returns valid uri' do + ['http://localhost:9090', 'https://localhost:9090 '].each do |valid_uri| + it 'returns correct server address' do + expect(described_class).to receive(:uri).and_return(valid_uri) + + expect(described_class.server_address).to eq('localhost:9090') + end + end + end + + context 'self.uri returns nil' do + it 'returns nil' do + expect(described_class).to receive(:uri).and_return(nil) + + expect(described_class.server_address).to be_nil end end end @@ -101,7 +121,7 @@ RSpec.describe Gitlab::Prometheus::Internal do end it 'does not fail' do - expect(described_class.listen_address).to eq(nil) + expect(described_class.listen_address).to be_nil end end end diff --git a/spec/lib/gitlab/prometheus_client_spec.rb b/spec/lib/gitlab/prometheus_client_spec.rb index 7c0c2962aed..82ef4675553 100644 --- a/spec/lib/gitlab/prometheus_client_spec.rb +++ b/spec/lib/gitlab/prometheus_client_spec.rb @@ -36,6 +36,28 @@ RSpec.describe Gitlab::PrometheusClient do end end + describe '#ready?' do + it 'returns true when status code is 200' do + stub_request(:get, subject.ready_url).to_return(status: 200, body: 'Prometheus is Ready.\n') + + expect(subject.ready?).to eq(true) + end + + it 'returns false when status code is not 200' do + [503, 500].each do |code| + stub_request(:get, subject.ready_url).to_return(status: code, body: 'Service Unavailable') + + expect(subject.ready?).to eq(false) + end + end + + it 'raises error when ready api throws exception' do + stub_request(:get, subject.ready_url).to_raise(Net::OpenTimeout) + + expect { subject.ready? }.to raise_error(Gitlab::PrometheusClient::UnexpectedResponseError) + end + end + # This shared examples expect: # - query_url: A query URL # - execute_query: A query call diff --git a/spec/lib/gitlab/relative_positioning/item_context_spec.rb b/spec/lib/gitlab/relative_positioning/item_context_spec.rb new file mode 100644 index 00000000000..daea8d8470d --- /dev/null +++ b/spec/lib/gitlab/relative_positioning/item_context_spec.rb @@ -0,0 +1,215 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::RelativePositioning::ItemContext do + let_it_be(:default_user) { create_default(:user) } + let_it_be(:project, reload: true) { create(:project) } + + def create_issue(pos) + create(:issue, project: project, relative_position: pos) + end + + range = (101..107) # A deliberately small range, so we can test everything + indices = (0..).take(range.size) + + let(:start) { ((range.first + range.last) / 2.0).floor } + let(:subjects) { issues.map { |i| described_class.new(i.reset, range) } } + + # This allows us to refer to range in methods and examples + let_it_be(:full_range) { range } + + context 'there are gaps at the start and end' do + let_it_be(:issues) { (range.first.succ..range.last.pred).map { |pos| create_issue(pos) } } + + it 'is always possible to find a gap' do + expect(subjects) + .to all(have_attributes(find_next_gap_before: be_present, find_next_gap_after: be_present)) + end + + where(:index) { indices.reverse.drop(2) } + + with_them do + subject { subjects[index] } + + let(:positions) { subject.scoped_items.map(&:relative_position) } + + it 'is possible to shift_right, which will consume the gap at the end' do + subject.shift_right + + expect(subject.find_next_gap_after).not_to be_present + + expect(positions).to all(be_between(range.first, range.last)) + expect(positions).to eq(positions.uniq) + end + + it 'is possible to create_space_right, which will move the gap to immediately after' do + subject.create_space_right + + expect(subject.find_next_gap_after).to have_attributes(start_pos: subject.relative_position) + expect(positions).to all(be_between(range.first, range.last)) + expect(positions).to eq(positions.uniq) + end + + it 'is possible to shift_left, which will consume the gap at the start' do + subject.shift_left + + expect(subject.find_next_gap_before).not_to be_present + expect(positions).to all(be_between(range.first, range.last)) + expect(positions).to eq(positions.uniq) + end + + it 'is possible to create_space_left, which will move the gap to immediately before' do + subject.create_space_left + + expect(subject.find_next_gap_before).to have_attributes(start_pos: subject.relative_position) + expect(positions).to all(be_between(range.first, range.last)) + expect(positions).to eq(positions.uniq) + end + end + end + + context 'there is a gap of multiple spaces' do + let_it_be(:issues) { [range.first, range.last].map { |pos| create_issue(pos) } } + + it 'is impossible to move the last element to the right' do + expect { subjects.last.shift_right }.to raise_error(Gitlab::RelativePositioning::NoSpaceLeft) + end + + it 'is impossible to move the first element to the left' do + expect { subjects.first.shift_left }.to raise_error(Gitlab::RelativePositioning::NoSpaceLeft) + end + + it 'is possible to move the last element to the left' do + subject = subjects.last + + expect { subject.shift_left }.to change { subject.relative_position }.by(be < 0) + end + + it 'is possible to move the first element to the right' do + subject = subjects.first + + expect { subject.shift_right }.to change { subject.relative_position }.by(be > 0) + end + + it 'is possible to find the gap from the right' do + gap = Gitlab::RelativePositioning::Gap.new(range.last, range.first) + + expect(subjects.last).to have_attributes( + find_next_gap_before: eq(gap), + find_next_gap_after: be_nil + ) + end + + it 'is possible to find the gap from the left' do + gap = Gitlab::RelativePositioning::Gap.new(range.first, range.last) + + expect(subjects.first).to have_attributes( + find_next_gap_before: be_nil, + find_next_gap_after: eq(gap) + ) + end + end + + context 'there are several free spaces' do + let_it_be(:issues) { range.select(&:even?).map { |pos| create_issue(pos) } } + let_it_be(:gaps) do + range.select(&:odd?).map do |pos| + rhs = pos.succ.clamp(range.first, range.last) + lhs = pos.pred.clamp(range.first, range.last) + + { + before: Gitlab::RelativePositioning::Gap.new(rhs, lhs), + after: Gitlab::RelativePositioning::Gap.new(lhs, rhs) + } + end + end + + def issue_at(position) + issues.find { |i| i.relative_position == position } + end + + where(:current_pos) { range.select(&:even?) } + + with_them do + let(:subject) { subjects.find { |s| s.relative_position == current_pos } } + let(:siblings) { subjects.reject { |s| s.relative_position == current_pos } } + + def covered_by_range(pos) + full_range.cover?(pos) ? pos : nil + end + + it 'finds the closest gap' do + closest_gap_before = gaps + .map { |gap| gap[:before] } + .select { |gap| gap.start_pos <= subject.relative_position } + .max_by { |gap| gap.start_pos } + closest_gap_after = gaps + .map { |gap| gap[:after] } + .select { |gap| gap.start_pos >= subject.relative_position } + .min_by { |gap| gap.start_pos } + + expect(subject).to have_attributes( + find_next_gap_before: closest_gap_before, + find_next_gap_after: closest_gap_after + ) + end + + it 'finds the neighbours' do + expect(subject).to have_attributes( + lhs_neighbour: subject.neighbour(issue_at(subject.relative_position - 2)), + rhs_neighbour: subject.neighbour(issue_at(subject.relative_position + 2)) + ) + end + + it 'finds the next relative_positions' do + expect(subject).to have_attributes( + prev_relative_position: covered_by_range(subject.relative_position - 2), + next_relative_position: covered_by_range(subject.relative_position + 2) + ) + end + + it 'finds the min/max positions' do + expect(subject).to have_attributes( + min_relative_position: issues.first.relative_position, + max_relative_position: issues.last.relative_position + ) + end + + it 'finds the min/max siblings' do + expect(subject).to have_attributes( + min_sibling: siblings.first, + max_sibling: siblings.last + ) + end + end + end + + context 'there is at least one free space' do + where(:free_space) { range.to_a } + + with_them do + let(:issues) { range.reject { |x| x == free_space }.map { |p| create_issue(p) } } + let(:gap_rhs) { free_space.succ.clamp(range.first, range.last) } + let(:gap_lhs) { free_space.pred.clamp(range.first, range.last) } + + it 'can always find a gap before if there is space to the left' do + expected_gap = Gitlab::RelativePositioning::Gap.new(gap_rhs, gap_lhs) + + to_the_right_of_gap = subjects.select { |s| free_space < s.relative_position } + + expect(to_the_right_of_gap) + .to all(have_attributes(find_next_gap_before: eq(expected_gap), find_next_gap_after: be_nil)) + end + + it 'can always find a gap after if there is space to the right' do + expected_gap = Gitlab::RelativePositioning::Gap.new(gap_lhs, gap_rhs) + + to_the_left_of_gap = subjects.select { |s| s.relative_position < free_space } + + expect(to_the_left_of_gap) + .to all(have_attributes(find_next_gap_before: be_nil, find_next_gap_after: eq(expected_gap))) + end + end + end +end diff --git a/spec/lib/gitlab/relative_positioning/mover_spec.rb b/spec/lib/gitlab/relative_positioning/mover_spec.rb new file mode 100644 index 00000000000..c49230c2415 --- /dev/null +++ b/spec/lib/gitlab/relative_positioning/mover_spec.rb @@ -0,0 +1,487 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe RelativePositioning::Mover do + let_it_be(:user) { create(:user) } + let_it_be(:one_sibling, reload: true) { create(:project, creator: user, namespace: user.namespace) } + let_it_be(:one_free_space, reload: true) { create(:project, creator: user, namespace: user.namespace) } + let_it_be(:fully_occupied, reload: true) { create(:project, creator: user, namespace: user.namespace) } + let_it_be(:no_issues, reload: true) { create(:project, creator: user, namespace: user.namespace) } + let_it_be(:three_sibs, reload: true) { create(:project, creator: user, namespace: user.namespace) } + + def create_issue(pos, parent = project) + create(:issue, author: user, project: parent, relative_position: pos) + end + + range = (101..105) + indices = (0..).take(range.size) + + let(:start) { ((range.first + range.last) / 2.0).floor } + + subject { described_class.new(start, range) } + + let_it_be(:full_set) do + range.each_with_index.map do |pos, i| + create(:issue, iid: i.succ, project: fully_occupied, relative_position: pos) + end + end + + let_it_be(:sole_sibling) { create(:issue, iid: 1, project: one_sibling, relative_position: nil) } + let_it_be(:one_sibling_set) { [sole_sibling] } + let_it_be(:one_free_space_set) do + indices.drop(1).map { |iid| create(:issue, project: one_free_space, iid: iid.succ) } + end + let_it_be(:three_sibs_set) do + [1, 2, 3].map { |iid| create(:issue, iid: iid, project: three_sibs) } + end + + def set_positions(positions) + vals = issues.zip(positions).map do |issue, pos| + issue.relative_position = pos + "(#{issue.id}, #{pos})" + end.join(', ') + + Issue.connection.exec_query(<<~SQL, 'set-positions') + WITH cte(cte_id, new_pos) AS ( + SELECT * FROM (VALUES #{vals}) as t (id, pos) + ) + UPDATE issues SET relative_position = new_pos FROM cte WHERE id = cte_id + ; + SQL + end + + def ids_in_position_order + project.issues.reorder(:relative_position).pluck(:id) + end + + def relative_positions + project.issues.pluck(:relative_position) + end + + describe '#move_to_end' do + def max_position + project.issues.maximum(:relative_position) + end + + def move_to_end(issue) + subject.move_to_end(issue) + issue.save! + end + + shared_examples 'able to place a new item at the end' do + it 'can place any new item' do + existing_issues = ids_in_position_order + new_item = create_issue(nil) + + expect do + move_to_end(new_item) + end.to change { project.issues.pluck(:id, :relative_position) } + + expect(new_item.relative_position).to eq(max_position) + expect(relative_positions).to all(be_between(range.first, range.last)) + expect(ids_in_position_order).to eq(existing_issues + [new_item.id]) + end + end + + shared_examples 'able to move existing items to the end' do + it 'can move any existing item' do + issues = project.issues.reorder(:relative_position).to_a + issue = issues[index] + other_issues = issues.reject { |i| i == issue } + expect(relative_positions).to all(be_between(range.first, range.last)) + + if issues.last == issue + move_to_end(issue) # May not change the positions + else + expect do + move_to_end(issue) + end.to change { project.issues.pluck(:id, :relative_position) } + end + + project.reset + + expect(relative_positions).to all(be_between(range.first, range.last)) + expect(issue.relative_position).to eq(max_position) + expect(ids_in_position_order).to eq(other_issues.map(&:id) + [issue.id]) + end + end + + context 'all positions are taken' do + let(:issues) { full_set } + let(:project) { fully_occupied } + + it 'raises an error when placing a new item' do + new_item = create_issue(nil) + + expect { subject.move_to_end(new_item) }.to raise_error(RelativePositioning::NoSpaceLeft) + end + + where(:index) { indices } + + with_them do + it_behaves_like 'able to move existing items to the end' + end + end + + context 'there are no siblings' do + let(:issues) { [] } + let(:project) { no_issues } + + it_behaves_like 'able to place a new item at the end' + end + + context 'there is only one sibling' do + where(:pos) { range.to_a } + + with_them do + let(:issues) { one_sibling_set } + let(:project) { one_sibling } + let(:index) { 0 } + + before do + sole_sibling.reset.update!(relative_position: pos) + end + + it_behaves_like 'able to place a new item at the end' + + it_behaves_like 'able to move existing items to the end' + end + end + + context 'at least one position is free' do + where(:free_space, :index) do + is = indices.take(range.size - 1) + + range.to_a.product(is) + end + + with_them do + let(:issues) { one_free_space_set } + let(:project) { one_free_space } + + before do + positions = range.reject { |x| x == free_space } + set_positions(positions) + end + + it_behaves_like 'able to place a new item at the end' + + it_behaves_like 'able to move existing items to the end' + end + end + end + + describe '#move_to_start' do + def min_position + project.issues.minimum(:relative_position) + end + + def move_to_start(issue) + subject.move_to_start(issue) + issue.save! + end + + shared_examples 'able to place a new item at the start' do + it 'can place any new item' do + existing_issues = ids_in_position_order + new_item = create_issue(nil) + + expect do + move_to_start(new_item) + end.to change { project.issues.pluck(:id, :relative_position) } + + expect(relative_positions).to all(be_between(range.first, range.last)) + expect(new_item.relative_position).to eq(min_position) + expect(ids_in_position_order).to eq([new_item.id] + existing_issues) + end + end + + shared_examples 'able to move existing items to the start' do + it 'can move any existing item' do + issues = project.issues.reorder(:relative_position).to_a + issue = issues[index] + other_issues = issues.reject { |i| i == issue } + expect(relative_positions).to all(be_between(range.first, range.last)) + + if issues.first == issue + move_to_start(issue) # May not change the positions + else + expect do + move_to_start(issue) + end.to change { project.issues.pluck(:id, :relative_position) } + end + + project.reset + + expect(relative_positions).to all(be_between(range.first, range.last)) + expect(issue.relative_position).to eq(min_position) + expect(ids_in_position_order).to eq([issue.id] + other_issues.map(&:id)) + end + end + + context 'all positions are taken' do + let(:issues) { full_set } + let(:project) { fully_occupied } + + it 'raises an error when placing a new item' do + new_item = create(:issue, project: project, relative_position: nil) + + expect { subject.move_to_start(new_item) }.to raise_error(RelativePositioning::NoSpaceLeft) + end + + where(:index) { indices } + + with_them do + it_behaves_like 'able to move existing items to the start' + end + end + + context 'there are no siblings' do + let(:project) { no_issues } + let(:issues) { [] } + + it_behaves_like 'able to place a new item at the start' + end + + context 'there is only one sibling' do + where(:pos) { range.to_a } + + with_them do + let(:issues) { one_sibling_set } + let(:project) { one_sibling } + let(:index) { 0 } + + before do + sole_sibling.reset.update!(relative_position: pos) + end + + it_behaves_like 'able to place a new item at the start' + + it_behaves_like 'able to move existing items to the start' + end + end + + context 'at least one position is free' do + where(:free_space, :index) do + range.to_a.product((0..).take(range.size - 1).to_a) + end + + with_them do + let(:issues) { one_free_space_set } + let(:project) { one_free_space } + + before do + set_positions(range.reject { |x| x == free_space }) + end + + it_behaves_like 'able to place a new item at the start' + + it_behaves_like 'able to move existing items to the start' + end + end + end + + describe '#move' do + shared_examples 'able to move a new item' do + let(:other_issues) { project.issues.reorder(relative_position: :asc).to_a } + let!(:previous_order) { other_issues.map(&:id) } + + it 'can place any new item betwen two others' do + new_item = create_issue(nil) + + subject.move(new_item, lhs, rhs) + new_item.save! + lhs.reset + rhs.reset + + expect(new_item.relative_position).to be_between(range.first, range.last) + expect(new_item.relative_position).to be_between(lhs.relative_position, rhs.relative_position) + + ids = project.issues.reorder(:relative_position).pluck(:id).reject { |id| id == new_item.id } + expect(ids).to eq(previous_order) + end + + it 'can place any new item after another' do + new_item = create_issue(nil) + + subject.move(new_item, lhs, nil) + new_item.save! + lhs.reset + + expect(new_item.relative_position).to be_between(range.first, range.last) + expect(new_item.relative_position).to be > lhs.relative_position + + ids = project.issues.reorder(:relative_position).pluck(:id).reject { |id| id == new_item.id } + expect(ids).to eq(previous_order) + end + + it 'can place any new item before another' do + new_item = create_issue(nil) + + subject.move(new_item, nil, rhs) + new_item.save! + rhs.reset + + expect(new_item.relative_position).to be_between(range.first, range.last) + expect(new_item.relative_position).to be < rhs.relative_position + + ids = project.issues.reorder(:relative_position).pluck(:id).reject { |id| id == new_item.id } + expect(ids).to eq(previous_order) + end + end + + shared_examples 'able to move an existing item' do + let(:all_issues) { project.issues.reorder(:relative_position).to_a } + let(:item) { all_issues[index] } + let(:positions) { project.reset.issues.pluck(:relative_position) } + let(:other_issues) { all_issues.reject { |i| i == item } } + let!(:previous_order) { other_issues.map(&:id) } + let(:new_order) do + project.issues.where.not(id: item.id).reorder(:relative_position).pluck(:id) + end + + it 'can place any item betwen two others' do + subject.move(item, lhs, rhs) + item.save! + lhs.reset + rhs.reset + + expect(positions).to all(be_between(range.first, range.last)) + expect(positions).to match_array(positions.uniq) + expect(item.relative_position).to be_between(lhs.relative_position, rhs.relative_position) + + expect(new_order).to eq(previous_order) + end + + def sequence(expected_sequence) + range = (expected_sequence.first.relative_position..expected_sequence.last.relative_position) + + project.issues.reorder(:relative_position).where(relative_position: range) + end + + it 'can place any item after another' do + subject.move(item, lhs, nil) + item.save! + lhs.reset + + expect(positions).to all(be_between(range.first, range.last)) + expect(positions).to match_array(positions.uniq) + expect(item.relative_position).to be >= lhs.relative_position + + expected_sequence = [lhs, item].uniq + + expect(sequence(expected_sequence)).to eq(expected_sequence) + + expect(new_order).to eq(previous_order) + end + + it 'can place any item before another' do + subject.move(item, nil, rhs) + item.save! + rhs.reset + + expect(positions).to all(be_between(range.first, range.last)) + expect(positions).to match_array(positions.uniq) + expect(item.relative_position).to be <= rhs.relative_position + + expected_sequence = [item, rhs].uniq + + expect(sequence(expected_sequence)).to eq(expected_sequence) + + expect(new_order).to eq(previous_order) + end + end + + context 'all positions are taken' do + let(:issues) { full_set } + let(:project) { fully_occupied } + + where(:idx_a, :idx_b) do + indices.product(indices).select { |a, b| a < b } + end + + with_them do + let(:lhs) { issues[idx_a].reset } + let(:rhs) { issues[idx_b].reset } + + it 'raises an error when placing a new item anywhere' do + new_item = create_issue(nil) + + expect { subject.move(new_item, lhs, rhs) } + .to raise_error(Gitlab::RelativePositioning::NoSpaceLeft) + + expect { subject.move(new_item, nil, rhs) } + .to raise_error(Gitlab::RelativePositioning::NoSpaceLeft) + + expect { subject.move(new_item, lhs, nil) } + .to raise_error(Gitlab::RelativePositioning::NoSpaceLeft) + end + + where(:index) { indices } + + with_them do + it_behaves_like 'able to move an existing item' + end + end + end + + context 'there are no siblings' do + let(:project) { no_issues } + + it 'raises an ArgumentError when both first and last are nil' do + new_item = create_issue(nil) + + expect { subject.move(new_item, nil, nil) }.to raise_error(ArgumentError) + end + end + + context 'there are a couple of siblings' do + where(:pos_movable, :pos_a, :pos_b) do + xs = range.to_a + + xs.product(xs).product(xs).map(&:flatten) + .select { |vals| vals == vals.uniq && vals[1] < vals[2] } + end + + with_them do + let(:issues) { three_sibs_set } + let(:project) { three_sibs } + let(:index) { 0 } + let(:lhs) { issues[1] } + let(:rhs) { issues[2] } + + before do + set_positions([pos_movable, pos_a, pos_b]) + end + + it_behaves_like 'able to move a new item' + it_behaves_like 'able to move an existing item' + end + end + + context 'at least one position is free' do + where(:free_space, :index, :pos_a, :pos_b) do + is = indices.reverse.drop(1) + + range.to_a.product(is).product(is).product(is) + .map(&:flatten) + .select { |_, _, a, b| a < b } + end + + with_them do + let(:issues) { one_free_space_set } + let(:project) { one_free_space } + let(:lhs) { issues[pos_a] } + let(:rhs) { issues[pos_b] } + + before do + set_positions(range.reject { |x| x == free_space }) + end + + it_behaves_like 'able to move a new item' + it_behaves_like 'able to move an existing item' + end + end + end +end diff --git a/spec/lib/gitlab/relative_positioning/range_spec.rb b/spec/lib/gitlab/relative_positioning/range_spec.rb new file mode 100644 index 00000000000..c3386336493 --- /dev/null +++ b/spec/lib/gitlab/relative_positioning/range_spec.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::RelativePositioning::Range do + item_a = OpenStruct.new(relative_position: 100, object: :x, positioned?: true) + item_b = OpenStruct.new(relative_position: 200, object: :y, positioned?: true) + + before do + allow(item_a).to receive(:lhs_neighbour) { nil } + allow(item_a).to receive(:rhs_neighbour) { item_b } + + allow(item_b).to receive(:lhs_neighbour) { item_a } + allow(item_b).to receive(:rhs_neighbour) { nil } + end + + describe 'RelativePositioning.range' do + it 'raises if lhs and rhs are nil' do + expect { Gitlab::RelativePositioning.range(nil, nil) }.to raise_error(ArgumentError) + end + + it 'raises an error if there is no extent' do + expect { Gitlab::RelativePositioning.range(item_a, item_a) }.to raise_error(ArgumentError) + end + + it 'constructs a closed range when both termini are provided' do + range = Gitlab::RelativePositioning.range(item_a, item_b) + expect(range).to be_a_kind_of(Gitlab::RelativePositioning::Range) + expect(range).to be_a_kind_of(Gitlab::RelativePositioning::ClosedRange) + end + + it 'constructs a starting-from range when only the LHS is provided' do + range = Gitlab::RelativePositioning.range(item_a, nil) + expect(range).to be_a_kind_of(Gitlab::RelativePositioning::Range) + expect(range).to be_a_kind_of(Gitlab::RelativePositioning::StartingFrom) + end + + it 'constructs an ending-at range when only the RHS is provided' do + range = Gitlab::RelativePositioning.range(nil, item_b) + expect(range).to be_a_kind_of(Gitlab::RelativePositioning::Range) + expect(range).to be_a_kind_of(Gitlab::RelativePositioning::EndingAt) + end + end + + it 'infers neighbours correctly' do + starting_at_a = Gitlab::RelativePositioning.range(item_a, nil) + ending_at_b = Gitlab::RelativePositioning.range(nil, item_b) + + expect(starting_at_a).to eq(ending_at_b) + end + + describe '#open_on_left?' do + where(:lhs, :rhs, :expected_result) do + [ + [item_a, item_b, false], + [item_a, nil, false], + [nil, item_b, false], + [item_b, nil, false], + [nil, item_a, true] + ] + end + + with_them do + it 'is true if there is no LHS terminus' do + range = Gitlab::RelativePositioning.range(lhs, rhs) + + expect(range.open_on_left?).to be(expected_result) + end + end + end + + describe '#open_on_right?' do + where(:lhs, :rhs, :expected_result) do + [ + [item_a, item_b, false], + [item_a, nil, false], + [nil, item_b, false], + [item_b, nil, true], + [nil, item_a, false] + ] + end + + with_them do + it 'is true if there is no RHS terminus' do + range = Gitlab::RelativePositioning.range(lhs, rhs) + + expect(range.open_on_right?).to be(expected_result) + end + end + end + + describe '#cover?' do + item_c = OpenStruct.new(relative_position: 150, object: :z, positioned?: true) + item_d = OpenStruct.new(relative_position: 050, object: :w, positioned?: true) + item_e = OpenStruct.new(relative_position: 250, object: :r, positioned?: true) + item_f = OpenStruct.new(positioned?: false) + item_ax = OpenStruct.new(relative_position: 100, object: :not_x, positioned?: true) + item_bx = OpenStruct.new(relative_position: 200, object: :not_y, positioned?: true) + + where(:lhs, :rhs, :item, :expected_result) do + [ + [item_a, item_b, item_a, true], + [item_a, item_b, item_b, true], + [item_a, item_b, item_c, true], + [item_a, item_b, item_d, false], + [item_a, item_b, item_e, false], + [item_a, item_b, item_ax, false], + [item_a, item_b, item_bx, false], + [item_a, item_b, item_f, false], + [item_a, item_b, nil, false], + + [nil, item_b, item_a, true], + [nil, item_b, item_b, true], + [nil, item_b, item_c, true], + [nil, item_b, item_d, false], + [nil, item_b, item_e, false], + [nil, item_b, item_ax, false], + [nil, item_b, item_bx, false], + [nil, item_b, item_f, false], + [nil, item_b, nil, false], + + [item_a, nil, item_a, true], + [item_a, nil, item_b, true], + [item_a, nil, item_c, true], + [item_a, nil, item_d, false], + [item_a, nil, item_e, false], + [item_a, nil, item_ax, false], + [item_a, nil, item_bx, false], + [item_a, nil, item_f, false], + [item_a, nil, nil, false], + + [nil, item_a, item_a, true], + [nil, item_a, item_b, false], + [nil, item_a, item_c, false], + [nil, item_a, item_d, true], + [nil, item_a, item_e, false], + [nil, item_a, item_ax, false], + [nil, item_a, item_bx, false], + [nil, item_a, item_f, false], + [nil, item_a, nil, false], + + [item_b, nil, item_a, false], + [item_b, nil, item_b, true], + [item_b, nil, item_c, false], + [item_b, nil, item_d, false], + [item_b, nil, item_e, true], + [item_b, nil, item_ax, false], + [item_b, nil, item_bx, false], + [item_b, nil, item_f, false], + [item_b, nil, nil, false] + ] + end + + with_them do + it 'is true when the object is within the bounds of the range' do + range = Gitlab::RelativePositioning.range(lhs, rhs) + + expect(range.cover?(item)).to be(expected_result) + end + end + end +end diff --git a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb index 0ff2dbb234a..4a952a2040a 100644 --- a/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb +++ b/spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb @@ -49,7 +49,7 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do expect { subject }.not_to raise_exception end - it 'logs exception message once and raise execption and log stop message' do + it 'logs exception message once and raise exception and log stop message' do expect(Sidekiq.logger).to receive(:warn).once .with( class: described_class.to_s, @@ -68,7 +68,7 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do pid: pid, message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon') - expect { subject }.to raise_exception + expect { subject }.to raise_exception(Exception, 'My Exception') end it 'logs stop message once' do @@ -402,12 +402,14 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do subject { memory_killer.send(:rss_increase_by_jobs) } it 'adds up individual rss_increase_by_job' do + allow(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs_mutex, :synchronize).and_yield expect(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs).and_return(running_jobs) expect(memory_killer).to receive(:rss_increase_by_job).and_return(11, 22) expect(subject).to eq(33) end it 'return 0 if no job' do + allow(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs_mutex, :synchronize).and_yield expect(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs).and_return({}) expect(subject).to eq(0) end diff --git a/spec/lib/gitlab/usage_data/topology_spec.rb b/spec/lib/gitlab/usage_data/topology_spec.rb index 46e3908251a..d731a385b8e 100644 --- a/spec/lib/gitlab/usage_data/topology_spec.rb +++ b/spec/lib/gitlab/usage_data/topology_spec.rb @@ -17,6 +17,7 @@ RSpec.describe Gitlab::UsageData::Topology do context 'tracking node metrics' do it 'contains node level metrics for each instance' do expect_prometheus_api_to( + receive_ready_check_query, receive_app_request_volume_query, receive_query_apdex_ratio_query, receive_node_memory_query, @@ -103,6 +104,7 @@ RSpec.describe Gitlab::UsageData::Topology do context 'and some node memory metrics are missing' do it 'removes the respective entries and includes the failures' do expect_prometheus_api_to( + receive_ready_check_query, receive_app_request_volume_query(result: []), receive_query_apdex_ratio_query(result: []), receive_node_memory_query(result: []), @@ -243,6 +245,7 @@ RSpec.describe Gitlab::UsageData::Topology do it 'normalizes equivalent instance values and maps them to the same node' do expect_prometheus_api_to( + receive_ready_check_query, receive_app_request_volume_query(result: []), receive_query_apdex_ratio_query(result: []), receive_node_memory_query(result: node_memory_response), @@ -309,6 +312,7 @@ RSpec.describe Gitlab::UsageData::Topology do context 'and node metrics are missing but service metrics exist' do it 'still reports service metrics' do expect_prometheus_api_to( + receive_ready_check_query, receive_app_request_volume_query(result: []), receive_query_apdex_ratio_query(result: []), receive_node_memory_query(result: []), @@ -384,6 +388,7 @@ RSpec.describe Gitlab::UsageData::Topology do it 'filters out unknown service data and reports the unknown services as a failure' do expect_prometheus_api_to( + receive_ready_check_query, receive_app_request_volume_query(result: []), receive_query_apdex_ratio_query(result: []), receive_node_memory_query(result: []), @@ -408,25 +413,26 @@ RSpec.describe Gitlab::UsageData::Topology do context 'and an error is raised when querying Prometheus' do context 'without timeout failures' do it 'returns empty result and executes subsequent queries as usual' do - expect_prometheus_api_to receive(:query) - .at_least(:once) - .and_raise(Gitlab::PrometheusClient::ConnectionError) + expect_prometheus_api_to( + receive_ready_check_query, + receive(:query).at_least(:once).and_raise(Gitlab::PrometheusClient::UnexpectedResponseError) + ) expect(subject[:topology]).to eq({ duration_s: 0, failures: [ - { 'app_requests' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'query_apdex' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'node_memory' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'node_memory_utilization' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'node_cpus' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'node_cpu_utilization' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'node_uname_info' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_rss' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_uss' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_pss' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_process_count' => 'Gitlab::PrometheusClient::ConnectionError' }, - { 'service_workers' => 'Gitlab::PrometheusClient::ConnectionError' } + { 'app_requests' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'query_apdex' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'node_memory' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'node_memory_utilization' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'node_cpus' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'node_cpu_utilization' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'node_uname_info' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'service_rss' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'service_uss' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'service_pss' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'service_process_count' => 'Gitlab::PrometheusClient::UnexpectedResponseError' }, + { 'service_workers' => 'Gitlab::PrometheusClient::UnexpectedResponseError' } ], nodes: [] }) @@ -440,8 +446,10 @@ RSpec.describe Gitlab::UsageData::Topology do with_them do it 'returns empty result and cancelled subsequent queries' do - expect_prometheus_api_to receive(:query) - .and_raise(exception) + expect_prometheus_api_to( + receive_ready_check_query, + receive(:query).and_raise(exception) + ) expect(subject[:topology]).to eq({ duration_s: 0, @@ -467,35 +475,69 @@ RSpec.describe Gitlab::UsageData::Topology do end end - context 'when Prometheus is available from Prometheus settings' do + shared_examples 'returns empty result with no failures' do + it do + expect(subject[:topology]).to eq({ + duration_s: 0, + failures: [] + }) + end + end + + shared_examples 'try to query Prometheus with given address' do + context 'Prometheus is ready' do + it_behaves_like 'query topology data from Prometheus' + end + + context 'Prometheus is not ready' do + before do + # readiness check over HTTPS connection returns false + expect_prometheus_api_to(receive_ready_check_query(result: false)) + # readiness check over HTTP connection also returns false + expect_prometheus_api_to(receive_ready_check_query(result: false)) + end + + it_behaves_like 'returns empty result with no failures' + end + + context 'Prometheus is not reachable' do + before do + # HTTPS connection is not reachable + expect_prometheus_api_to(receive_ready_check_query(raise_error: Errno::ECONNREFUSED)) + # HTTP connection is also not reachable + expect_prometheus_api_to(receive_ready_check_query(raise_error: Errno::ECONNREFUSED)) + end + + it_behaves_like 'returns empty result with no failures' + end + end + + context 'when Prometheus server address is available from Prometheus settings' do before do expect(Gitlab::Prometheus::Internal).to receive(:prometheus_enabled?).and_return(true) expect(Gitlab::Prometheus::Internal).to receive(:uri).and_return('http://prom:9090') end - include_examples 'query topology data from Prometheus' + include_examples 'try to query Prometheus with given address' end - context 'when Prometheus is available from Consul service discovery' do + context 'when Prometheus server address is available from Consul service discovery' do before do expect(Gitlab::Prometheus::Internal).to receive(:prometheus_enabled?).and_return(false) expect(Gitlab::Consul::Internal).to receive(:api_url).and_return('http://127.0.0.1:8500') - expect(Gitlab::Consul::Internal).to receive(:discover_prometheus_uri).and_return('http://prom.net:9090') + expect(Gitlab::Consul::Internal).to receive(:discover_prometheus_server_address).and_return('prom.net:9090') end - include_examples 'query topology data from Prometheus' + include_examples 'try to query Prometheus with given address' end - context 'when Prometheus is not available' do - it 'returns empty result with no failures' do + context 'when Prometheus server address is not available' do + before do expect(Gitlab::Prometheus::Internal).to receive(:prometheus_enabled?).and_return(false) expect(Gitlab::Consul::Internal).to receive(:api_url).and_return(nil) - - expect(subject[:topology]).to eq({ - duration_s: 0, - failures: [] - }) end + + include_examples 'returns empty result with no failures' end context 'when top-level function raises error' do @@ -512,6 +554,14 @@ RSpec.describe Gitlab::UsageData::Topology do end end + def receive_ready_check_query(result: nil, raise_error: nil) + if raise_error.nil? + receive(:ready?).and_return(result.nil? ? true : result) + else + receive(:ready?).and_raise(raise_error) + end + end + def receive_app_request_volume_query(result: nil) receive(:query) .with(/gitlab_usage_ping:ops:rate/) diff --git a/spec/lib/gitlab/utils/usage_data_spec.rb b/spec/lib/gitlab/utils/usage_data_spec.rb index 80a3567186c..bafd5cc199e 100644 --- a/spec/lib/gitlab/utils/usage_data_spec.rb +++ b/spec/lib/gitlab/utils/usage_data_spec.rb @@ -106,42 +106,86 @@ RSpec.describe Gitlab::Utils::UsageData do end end - context 'when Prometheus is available from settings' do + shared_examples 'does not query data from Prometheus' do + it 'returns nil by default' do + result = described_class.with_prometheus_client { |client| client } + + expect(result).to be_nil + end + + it 'returns fallback if provided' do + result = described_class.with_prometheus_client(fallback: []) { |client| client } + + expect(result).to eq([]) + end + end + + shared_examples 'try to query Prometheus with given address' do + context 'Prometheus is ready' do + before do + stub_request(:get, /\/-\/ready/) + .to_return(status: 200, body: 'Prometheus is Ready.\n') + end + + context 'Prometheus is reachable through HTTPS' do + it_behaves_like 'query data from Prometheus' + end + + context 'Prometheus is not reachable through HTTPS' do + before do + stub_request(:get, /https:\/\/.*/).to_raise(Errno::ECONNREFUSED) + end + + context 'Prometheus is reachable through HTTP' do + it_behaves_like 'query data from Prometheus' + end + + context 'Prometheus is not reachable through HTTP' do + before do + stub_request(:get, /http:\/\/.*/).to_raise(Errno::ECONNREFUSED) + end + + it_behaves_like 'does not query data from Prometheus' + end + end + end + + context 'Prometheus is not ready' do + before do + stub_request(:get, /\/-\/ready/) + .to_return(status: 503, body: 'Service Unavailable') + end + + it_behaves_like 'does not query data from Prometheus' + end + end + + context 'when Prometheus server address is available from settings' do before do expect(Gitlab::Prometheus::Internal).to receive(:prometheus_enabled?).and_return(true) - expect(Gitlab::Prometheus::Internal).to receive(:uri).and_return('http://prom:9090') + expect(Gitlab::Prometheus::Internal).to receive(:server_address).and_return('prom:9090') end - it_behaves_like 'query data from Prometheus' + it_behaves_like 'try to query Prometheus with given address' end - context 'when Prometheus is available from Consul service discovery' do + context 'when Prometheus server address is available from Consul service discovery' do before do expect(Gitlab::Prometheus::Internal).to receive(:prometheus_enabled?).and_return(false) expect(Gitlab::Consul::Internal).to receive(:api_url).and_return('http://localhost:8500') - expect(Gitlab::Consul::Internal).to receive(:discover_prometheus_uri).and_return('http://prom:9090') + expect(Gitlab::Consul::Internal).to receive(:discover_prometheus_server_address).and_return('prom:9090') end - it_behaves_like 'query data from Prometheus' + it_behaves_like 'try to query Prometheus with given address' end - context 'when Prometheus is not available' do + context 'when Prometheus server address is not available' do before do expect(Gitlab::Prometheus::Internal).to receive(:prometheus_enabled?).and_return(false) expect(Gitlab::Consul::Internal).to receive(:api_url).and_return(nil) end - it 'returns nil by default' do - result = described_class.with_prometheus_client { |client| client } - - expect(result).to be nil - end - - it 'returns fallback if provided' do - result = described_class.with_prometheus_client(fallback: []) { |client| client } - - expect(result).to eq([]) - end + it_behaves_like 'does not query data from Prometheus' end end diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 5d809a6a929..9f76fb3330d 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -638,6 +638,29 @@ RSpec.describe ApplicationSetting do is_expected.to be_invalid end end + + context 'gitpod settings' do + it 'is invalid if gitpod is enabled and no url is provided' do + allow(subject).to receive(:gitpod_enabled).and_return(true) + allow(subject).to receive(:gitpod_url).and_return(nil) + + is_expected.to be_invalid + end + + it 'is invalid if gitpod is enabled and an empty url is provided' do + allow(subject).to receive(:gitpod_enabled).and_return(true) + allow(subject).to receive(:gitpod_url).and_return('') + + is_expected.to be_invalid + end + + it 'is invalid if gitpod is enabled and an invalid url is provided' do + allow(subject).to receive(:gitpod_enabled).and_return(true) + allow(subject).to receive(:gitpod_url).and_return('javascript:alert("test")//') + + is_expected.to be_invalid + end + end end context 'restrict creating duplicates' do diff --git a/spec/models/ci/build_trace_chunk_spec.rb b/spec/models/ci/build_trace_chunk_spec.rb index 7e2430e6f2e..fefe5e3bfca 100644 --- a/spec/models/ci/build_trace_chunk_spec.rb +++ b/spec/models/ci/build_trace_chunk_spec.rb @@ -21,6 +21,25 @@ RSpec.describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do stub_artifacts_object_storage end + describe 'chunk creation' do + let(:metrics) { spy('metrics') } + + before do + allow(::Gitlab::Ci::Trace::Metrics) + .to receive(:new) + .and_return(metrics) + end + + it 'increments trace operation chunked metric' do + build_trace_chunk.save! + + expect(metrics) + .to have_received(:increment_trace_operation) + .with(operation: :chunked) + .once + end + end + context 'FastDestroyAll' do let(:parent) { create(:project) } let(:pipeline) { create(:ci_pipeline, project: parent) } @@ -346,6 +365,24 @@ RSpec.describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do it_behaves_like 'Scheduling no sidekiq worker' end end + + describe 'append metrics' do + let(:metrics) { spy('metrics') } + + before do + allow(::Gitlab::Ci::Trace::Metrics) + .to receive(:new) + .and_return(metrics) + end + + it 'increments trace operation appended metric' do + build_trace_chunk.append('123456', 0) + + expect(metrics) + .to have_received(:increment_trace_operation) + .with(operation: :appended) + end + end end describe '#truncate' do diff --git a/spec/models/design_management/design_spec.rb b/spec/models/design_management/design_spec.rb index f78b66b572c..d4adc0d42d0 100644 --- a/spec/models/design_management/design_spec.rb +++ b/spec/models/design_management/design_spec.rb @@ -12,8 +12,10 @@ RSpec.describe DesignManagement::Design do let_it_be(:deleted_design) { create(:design, :with_versions, deleted: true) } it_behaves_like 'a class that supports relative positioning' do + let_it_be(:relative_parent) { create(:issue) } + let(:factory) { :design } - let(:default_params) { { issue: issue } } + let(:default_params) { { issue: relative_parent } } end describe 'relations' do diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb index f1902d19993..283d945157b 100644 --- a/spec/models/issue_spec.rb +++ b/spec/models/issue_spec.rb @@ -1187,29 +1187,20 @@ RSpec.describe Issue do describe 'scheduling rebalancing' do before do - allow(issue).to receive(:find_next_gap) { raise ActiveRecord::QueryCanceled } + allow_next_instance_of(RelativePositioning::Mover) do |mover| + allow(mover).to receive(:move) { raise ActiveRecord::QueryCanceled } + end end - let(:project) { build(:project_empty_repo) } + let(:project) { build_stubbed(:project_empty_repo) } let(:issue) { build_stubbed(:issue, relative_position: 100, project: project) } - describe '#find_next_gap_before' do - it 'schedules rebalancing if we time-out when finding a gap' do - lhs = build_stubbed(:issue, relative_position: 99, project: project) - to_move = build(:issue, project: project) - expect(IssueRebalancingWorker).to receive(:perform_async).with(nil, project.id) + it 'schedules rebalancing if we time-out when moving' do + lhs = build_stubbed(:issue, relative_position: 99, project: project) + to_move = build(:issue, project: project) + expect(IssueRebalancingWorker).to receive(:perform_async).with(nil, project.id) - expect { to_move.move_between(lhs, issue) }.to raise_error(ActiveRecord::QueryCanceled) - end - end - - describe '#find_next_gap_after' do - it 'schedules rebalancing if we time-out when finding a gap' do - allow(issue).to receive(:find_next_gap) { raise ActiveRecord::QueryCanceled } - expect(IssueRebalancingWorker).to receive(:perform_async).with(nil, project.id) - - expect { issue.move_sequence_after }.to raise_error(ActiveRecord::QueryCanceled) - end + expect { to_move.move_between(lhs, issue) }.to raise_error(ActiveRecord::QueryCanceled) end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index a0ec61c4117..1841288cd4b 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -47,6 +47,9 @@ RSpec.describe User do it { is_expected.to delegate_method(:sourcegraph_enabled).to(:user_preference) } it { is_expected.to delegate_method(:sourcegraph_enabled=).to(:user_preference).with_arguments(:args) } + it { is_expected.to delegate_method(:gitpod_enabled).to(:user_preference) } + it { is_expected.to delegate_method(:gitpod_enabled=).to(:user_preference).with_arguments(:args) } + it { is_expected.to delegate_method(:setup_for_company).to(:user_preference) } it { is_expected.to delegate_method(:setup_for_company=).to(:user_preference).with_arguments(:args) } diff --git a/spec/requests/api/ci/runner/jobs_artifacts_spec.rb b/spec/requests/api/ci/runner/jobs_artifacts_spec.rb index 77ff561bbdf..97110b63ff6 100644 --- a/spec/requests/api/ci/runner/jobs_artifacts_spec.rb +++ b/spec/requests/api/ci/runner/jobs_artifacts_spec.rb @@ -238,9 +238,24 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state do expect(json_response['ProcessLsif']).to be_truthy end + it 'tracks code_intelligence usage ping' do + tracking_params = { + event_names: 'i_source_code_code_intelligence', + start_date: Date.yesterday, + end_date: Date.today + } + + expect { authorize_artifacts_with_token_in_headers(artifact_type: :lsif) } + .to change { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(tracking_params) } + .by(1) + end + context 'code_navigation feature flag is disabled' do - it 'responds with a forbidden error' do + before do stub_feature_flags(code_navigation: false) + end + + it 'responds with a forbidden error' do authorize_artifacts_with_token_in_headers(artifact_type: :lsif) aggregate_failures do @@ -248,6 +263,17 @@ RSpec.describe API::Ci::Runner, :clean_gitlab_redis_shared_state do expect(json_response['ProcessLsif']).to be_falsy end end + + it 'does not track code_intelligence usage ping' do + tracking_params = { + event_names: 'i_source_code_code_intelligence', + start_date: Date.yesterday, + end_date: Date.today + } + + expect { authorize_artifacts_with_token_in_headers(artifact_type: :lsif) } + .not_to change { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(tracking_params) } + end end end diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb index 356a3d8d2d7..ef12f6dbed3 100644 --- a/spec/requests/api/settings_spec.rb +++ b/spec/requests/api/settings_spec.rb @@ -39,6 +39,7 @@ RSpec.describe API::Settings, 'Settings' do expect(json_response['snippet_size_limit']).to eq(50.megabytes) expect(json_response['spam_check_endpoint_enabled']).to be_falsey expect(json_response['spam_check_endpoint_url']).to be_nil + expect(json_response['wiki_page_max_content_bytes']).to be_a(Integer) end end @@ -116,7 +117,8 @@ RSpec.describe API::Settings, 'Settings' do spam_check_endpoint_enabled: true, spam_check_endpoint_url: 'https://example.com/spam_check', disabled_oauth_sign_in_sources: 'unknown', - import_sources: 'github,bitbucket' + import_sources: 'github,bitbucket', + wiki_page_max_content_bytes: 12345 } expect(response).to have_gitlab_http_status(:ok) @@ -158,6 +160,7 @@ RSpec.describe API::Settings, 'Settings' do expect(json_response['spam_check_endpoint_url']).to eq('https://example.com/spam_check') expect(json_response['disabled_oauth_sign_in_sources']).to eq([]) expect(json_response['import_sources']).to match_array(%w(github bitbucket)) + expect(json_response['wiki_page_max_content_bytes']).to eq(12345) end end diff --git a/spec/services/ci/update_build_state_service_spec.rb b/spec/services/ci/update_build_state_service_spec.rb index 050add930cf..f5ad732bf7e 100644 --- a/spec/services/ci/update_build_state_service_spec.rb +++ b/spec/services/ci/update_build_state_service_spec.rb @@ -8,7 +8,7 @@ RSpec.describe Ci::UpdateBuildStateService do let(:build) { create(:ci_build, :running, pipeline: pipeline) } let(:metrics) { spy('metrics') } - subject { described_class.new(build, params, metrics) } + subject { described_class.new(build, params) } before do stub_feature_flags(ci_enable_live_trace: true) @@ -31,7 +31,7 @@ RSpec.describe Ci::UpdateBuildStateService do end it 'does not increment finalized trace metric' do - subject.execute + execute_with_stubbed_metrics! expect(metrics) .not_to have_received(:increment_trace_operation) @@ -50,11 +50,16 @@ RSpec.describe Ci::UpdateBuildStateService do context 'when request payload carries a trace' do let(:params) { { state: 'success', trace: 'overwritten' } } - it 'overwrites a trace and updates trace operation metric' do + it 'overwrites a trace' do result = subject.execute expect(build.trace.raw).to eq 'overwritten' expect(result.status).to eq 200 + end + + it 'updates overwrite operation metric' do + execute_with_stubbed_metrics! + expect(metrics) .to have_received(:increment_trace_operation) .with(operation: :overwrite) @@ -96,7 +101,7 @@ RSpec.describe Ci::UpdateBuildStateService do end it 'increments trace finalized operation metric' do - subject.execute + execute_with_stubbed_metrics! expect(metrics) .to have_received(:increment_trace_operation) @@ -130,7 +135,7 @@ RSpec.describe Ci::UpdateBuildStateService do end it 'increments trace accepted operation metric' do - subject.execute + execute_with_stubbed_metrics! expect(metrics) .to have_received(:increment_trace_operation) @@ -172,7 +177,7 @@ RSpec.describe Ci::UpdateBuildStateService do end it 'increments discarded traces metric' do - subject.execute + execute_with_stubbed_metrics! expect(metrics) .to have_received(:increment_trace_operation) @@ -180,7 +185,7 @@ RSpec.describe Ci::UpdateBuildStateService do end it 'does not increment finalized trace metric' do - subject.execute + execute_with_stubbed_metrics! expect(metrics) .not_to have_received(:increment_trace_operation) @@ -203,7 +208,7 @@ RSpec.describe Ci::UpdateBuildStateService do end it 'increments conflict trace metric' do - subject.execute + execute_with_stubbed_metrics! expect(metrics) .to have_received(:increment_trace_operation) @@ -224,4 +229,10 @@ RSpec.describe Ci::UpdateBuildStateService do end end end + + def execute_with_stubbed_metrics! + described_class + .new(build, params, metrics) + .execute + end end diff --git a/spec/support/shared_examples/models/relative_positioning_shared_examples.rb b/spec/support/shared_examples/models/relative_positioning_shared_examples.rb index 0ad8d8cbde1..d1437244082 100644 --- a/spec/support/shared_examples/models/relative_positioning_shared_examples.rb +++ b/spec/support/shared_examples/models/relative_positioning_shared_examples.rb @@ -1,9 +1,25 @@ # frozen_string_literal: true +# Notes for implementing classes: +# +# The following let bindings should be defined: +# - `factory`: A symbol naming a factory to use to create items +# - `default_params`: A HashMap of factory parameters to pass to the factory. +# +# The `default_params` should include the relative parent, so that any item +# created with these parameters passed to the `factory` will be considered in +# the same set of items relative to each other. +# +# For the purposes of efficiency, it is a good idea to bind the parent in +# `let_it_be`, so that it is re-used across examples, but be careful that it +# does not have any other children - it should only be used within this set of +# shared examples. RSpec.shared_examples 'a class that supports relative positioning' do let(:item1) { create_item } let(:item2) { create_item } - let(:new_item) { create_item } + let(:new_item) { create_item(relative_position: nil) } + + let(:set_size) { RelativePositioning.mover.context(item1).scoped_items.count } def create_item(params = {}) create(factory, params.merge(default_params)) @@ -17,6 +33,7 @@ RSpec.shared_examples 'a class that supports relative positioning' do describe '.move_nulls_to_end' do let(:item3) { create_item } + let(:sibling_query) { item1.class.relative_positioning_query_base(item1) } it 'moves items with null relative_position to the end' do item1.update!(relative_position: 1000) @@ -28,10 +45,9 @@ RSpec.shared_examples 'a class that supports relative positioning' do expect(items.sort_by(&:relative_position)).to eq(items) expect(item1.relative_position).to be(1000) - expect(item1.prev_relative_position).to be_nil - expect(item1.next_relative_position).to eq(item2.relative_position) - expect(item2.next_relative_position).to eq(item3.relative_position) - expect(item3.next_relative_position).to be_nil + + expect(sibling_query.where(relative_position: nil)).not_to exist + expect(sibling_query.reorder(:relative_position, :id)).to eq([item1, item2, item3]) end it 'preserves relative position' do @@ -120,6 +136,7 @@ RSpec.shared_examples 'a class that supports relative positioning' do describe '.move_nulls_to_start' do let(:item3) { create_item } + let(:sibling_query) { item1.class.relative_positioning_query_base(item1) } it 'moves items with null relative_position to the start' do item1.update!(relative_position: nil) @@ -131,10 +148,8 @@ RSpec.shared_examples 'a class that supports relative positioning' do items.map(&:reload) expect(items.sort_by(&:relative_position)).to eq(items) - expect(item1.prev_relative_position).to eq nil - expect(item1.next_relative_position).to eq item2.relative_position - expect(item2.next_relative_position).to eq item3.relative_position - expect(item3.next_relative_position).to eq nil + expect(sibling_query.where(relative_position: nil)).not_to exist + expect(sibling_query.reorder(:relative_position, :id)).to eq(items) expect(item3.relative_position).to be(1000) end @@ -194,194 +209,6 @@ RSpec.shared_examples 'a class that supports relative positioning' do end end - describe '#max_relative_position' do - it 'returns maximum position' do - expect(item1.max_relative_position).to eq item2.relative_position - end - end - - describe '#prev_relative_position' do - it 'returns previous position if there is an item above' do - item1.update!(relative_position: 5) - item2.update!(relative_position: 15) - - expect(item2.prev_relative_position).to eq item1.relative_position - end - - it 'returns nil if there is no item above' do - expect(item1.prev_relative_position).to eq nil - end - end - - describe '#next_relative_position' do - it 'returns next position if there is an item below' do - item1.update!(relative_position: 5) - item2.update!(relative_position: 15) - - expect(item1.next_relative_position).to eq item2.relative_position - end - - it 'returns nil if there is no item below' do - expect(item2.next_relative_position).to eq nil - end - end - - describe '#find_next_gap_before' do - context 'there is no gap' do - let(:items) { create_items_with_positions(run_at_start) } - - it 'returns nil' do - items.each do |item| - expect(item.send(:find_next_gap_before)).to be_nil - end - end - end - - context 'there is a sequence ending at MAX_POSITION' do - let(:items) { create_items_with_positions(run_at_end) } - - let(:gaps) do - items.map { |item| item.send(:find_next_gap_before) } - end - - it 'can find the gap at the start for any item in the sequence' do - gap = { start: items.first.relative_position, end: RelativePositioning::MIN_POSITION } - - expect(gaps).to all(eq(gap)) - end - - it 'respects lower bounds' do - gap = { start: items.first.relative_position, end: 10 } - new_item.update!(relative_position: 10) - - expect(gaps).to all(eq(gap)) - end - end - - specify do - item1.update!(relative_position: 5) - - (0..10).each do |pos| - item2.update!(relative_position: pos) - - gap = item2.send(:find_next_gap_before) - - expect(gap[:start]).to be <= item2.relative_position - expect((gap[:end] - gap[:start]).abs).to be >= RelativePositioning::MIN_GAP - expect(gap[:start]).to be_valid_position - expect(gap[:end]).to be_valid_position - end - end - - it 'deals with there not being any items to the left' do - create_items_with_positions([1, 2, 3]) - new_item.update!(relative_position: 0) - - expect(new_item.send(:find_next_gap_before)).to eq(start: 0, end: RelativePositioning::MIN_POSITION) - end - - it 'finds the next gap to the left, skipping adjacent values' do - create_items_with_positions([1, 9, 10]) - new_item.update!(relative_position: 11) - - expect(new_item.send(:find_next_gap_before)).to eq(start: 9, end: 1) - end - - it 'finds the next gap to the left' do - create_items_with_positions([2, 10]) - - new_item.update!(relative_position: 15) - expect(new_item.send(:find_next_gap_before)).to eq(start: 15, end: 10) - - new_item.update!(relative_position: 11) - expect(new_item.send(:find_next_gap_before)).to eq(start: 10, end: 2) - - new_item.update!(relative_position: 9) - expect(new_item.send(:find_next_gap_before)).to eq(start: 9, end: 2) - - new_item.update!(relative_position: 5) - expect(new_item.send(:find_next_gap_before)).to eq(start: 5, end: 2) - end - end - - describe '#find_next_gap_after' do - context 'there is no gap' do - let(:items) { create_items_with_positions(run_at_end) } - - it 'returns nil' do - items.each do |item| - expect(item.send(:find_next_gap_after)).to be_nil - end - end - end - - context 'there is a sequence starting at MIN_POSITION' do - let(:items) { create_items_with_positions(run_at_start) } - - let(:gaps) do - items.map { |item| item.send(:find_next_gap_after) } - end - - it 'can find the gap at the end for any item in the sequence' do - gap = { start: items.last.relative_position, end: RelativePositioning::MAX_POSITION } - - expect(gaps).to all(eq(gap)) - end - - it 'respects upper bounds' do - gap = { start: items.last.relative_position, end: 10 } - new_item.update!(relative_position: 10) - - expect(gaps).to all(eq(gap)) - end - end - - specify do - item1.update!(relative_position: 5) - - (0..10).each do |pos| - item2.update!(relative_position: pos) - - gap = item2.send(:find_next_gap_after) - - expect(gap[:start]).to be >= item2.relative_position - expect((gap[:end] - gap[:start]).abs).to be >= RelativePositioning::MIN_GAP - expect(gap[:start]).to be_valid_position - expect(gap[:end]).to be_valid_position - end - end - - it 'deals with there not being any items to the right' do - create_items_with_positions([1, 2, 3]) - new_item.update!(relative_position: 5) - - expect(new_item.send(:find_next_gap_after)).to eq(start: 5, end: RelativePositioning::MAX_POSITION) - end - - it 'finds the next gap to the right, skipping adjacent values' do - create_items_with_positions([1, 2, 10]) - new_item.update!(relative_position: 0) - - expect(new_item.send(:find_next_gap_after)).to eq(start: 2, end: 10) - end - - it 'finds the next gap to the right' do - create_items_with_positions([2, 10]) - - new_item.update!(relative_position: 0) - expect(new_item.send(:find_next_gap_after)).to eq(start: 0, end: 2) - - new_item.update!(relative_position: 1) - expect(new_item.send(:find_next_gap_after)).to eq(start: 2, end: 10) - - new_item.update!(relative_position: 3) - expect(new_item.send(:find_next_gap_after)).to eq(start: 3, end: 10) - - new_item.update!(relative_position: 5) - expect(new_item.send(:find_next_gap_after)).to eq(start: 5, end: 10) - end - end - describe '#move_before' do let(:item3) { create(factory, default_params) } @@ -446,36 +273,39 @@ RSpec.shared_examples 'a class that supports relative positioning' do end context 'leap-frogging to the left' do + let(:item3) { create(factory, default_params) } + let(:start) { RelativePositioning::START_POSITION } + before do - start = RelativePositioning::START_POSITION item1.update!(relative_position: start - RelativePositioning::IDEAL_DISTANCE * 0) item2.update!(relative_position: start - RelativePositioning::IDEAL_DISTANCE * 1) item3.update!(relative_position: start - RelativePositioning::IDEAL_DISTANCE * 2) end - let(:item3) { create(factory, default_params) } - - def leap_frog(steps) - a = item1 - b = item2 + def leap_frog + a, b = [item1.reset, item2.reset].sort_by(&:relative_position) - steps.times do |i| - a.move_before(b) - a.save! - a, b = b, a - end + b.move_before(a) + b.save! end - it 'can leap-frog STEPS - 1 times before needing to rebalance' do - # This is less efficient than going right, due to the flooring of - # integer division - expect { leap_frog(RelativePositioning::STEPS - 1) } - .not_to change { item3.reload.relative_position } + it 'can leap-frog STEPS times before needing to rebalance' do + expect { RelativePositioning::STEPS.times { leap_frog } } + .to change { item3.reload.relative_position }.by(0) + .and change { item1.reload.relative_position }.by(be < 0) + .and change { item2.reload.relative_position }.by(be < 0) + + expect { leap_frog } + .to change { item3.reload.relative_position }.by(be < 0) end - it 'rebalances after leap-frogging STEPS times' do - expect { leap_frog(RelativePositioning::STEPS) } - .to change { item3.reload.relative_position } + context 'there is no space to the left after moving STEPS times' do + let(:start) { RelativePositioning::MIN_POSITION + (2 * RelativePositioning::IDEAL_DISTANCE) } + + it 'rebalances to the right' do + expect { RelativePositioning::STEPS.succ.times { leap_frog } } + .not_to change { item3.reload.relative_position } + end end end end @@ -538,25 +368,25 @@ RSpec.shared_examples 'a class that supports relative positioning' do let(:item3) { create(factory, default_params) } - def leap_frog(steps) - a = item1 - b = item2 + def leap_frog + a, b = [item1.reset, item2.reset].sort_by(&:relative_position) - steps.times do |i| - a.move_after(b) - a.save! - a, b = b, a - end + a.move_after(b) + a.save! end - it 'can leap-frog STEPS times before needing to rebalance' do - expect { leap_frog(RelativePositioning::STEPS) } - .not_to change { item3.reload.relative_position } - end + it 'rebalances after STEPS jumps' do + RelativePositioning::STEPS.pred.times do + expect { leap_frog } + .to change { item3.reload.relative_position }.by(0) + .and change { item1.reset.relative_position }.by(be >= 0) + .and change { item2.reset.relative_position }.by(be >= 0) + end - it 'rebalances after leap-frogging STEPS+1 times' do - expect { leap_frog(RelativePositioning::STEPS + 1) } - .to change { item3.reload.relative_position } + expect { leap_frog } + .to change { item3.reload.relative_position }.by(0) + .and change { item1.reset.relative_position }.by(be < 0) + .and change { item2.reset.relative_position }.by(be < 0) end end end @@ -569,7 +399,7 @@ RSpec.shared_examples 'a class that supports relative positioning' do end it 'places items at most IDEAL_DISTANCE from the start when the range is open' do - n = item1.send(:scoped_items).count + n = set_size expect([item1, item2].map(&:relative_position)).to all(be >= (RelativePositioning::START_POSITION - (n * RelativePositioning::IDEAL_DISTANCE))) end @@ -620,7 +450,7 @@ RSpec.shared_examples 'a class that supports relative positioning' do end it 'places items at most IDEAL_DISTANCE from the start when the range is open' do - n = item1.send(:scoped_items).count + n = set_size expect([item1, item2].map(&:relative_position)).to all(be <= (RelativePositioning::START_POSITION + (n * RelativePositioning::IDEAL_DISTANCE))) end @@ -802,63 +632,6 @@ RSpec.shared_examples 'a class that supports relative positioning' do end end - describe '#move_sequence_before' do - it 'moves the whole sequence of items to the middle of the nearest gap' do - items = create_items_with_positions([90, 100, 101, 102]) - - items.last.move_sequence_before - items.last.save! - - positions = items.map { |item| item.reload.relative_position } - expect(positions).to eq([90, 95, 96, 102]) - end - - it 'raises an error if there is no space' do - items = create_items_with_positions(run_at_start) - - expect { items.last.move_sequence_before }.to raise_error(RelativePositioning::NoSpaceLeft) - end - - it 'finds a gap if there are unused positions' do - items = create_items_with_positions([100, 101, 102]) - - items.last.move_sequence_before - items.last.save! - - positions = items.map { |item| item.reload.relative_position } - - expect(positions.last - positions.second).to be > RelativePositioning::MIN_GAP - end - end - - describe '#move_sequence_after' do - it 'moves the whole sequence of items to the middle of the nearest gap' do - items = create_items_with_positions([100, 101, 102, 110]) - - items.first.move_sequence_after - items.first.save! - - positions = items.map { |item| item.reload.relative_position } - expect(positions).to eq([100, 105, 106, 110]) - end - - it 'finds a gap if there are unused positions' do - items = create_items_with_positions([100, 101, 102]) - - items.first.move_sequence_after - items.first.save! - - positions = items.map { |item| item.reload.relative_position } - expect(positions.second - positions.first).to be > RelativePositioning::MIN_GAP - end - - it 'raises an error if there is no space' do - items = create_items_with_positions(run_at_end) - - expect { items.first.move_sequence_after }.to raise_error(RelativePositioning::NoSpaceLeft) - end - end - def be_valid_position be_between(RelativePositioning::MIN_POSITION, RelativePositioning::MAX_POSITION) end diff --git a/spec/tasks/gitlab/usage_data_rake_spec.rb b/spec/tasks/gitlab/usage_data_rake_spec.rb index 2a596946d86..0ee6fbef53f 100644 --- a/spec/tasks/gitlab/usage_data_rake_spec.rb +++ b/spec/tasks/gitlab/usage_data_rake_spec.rb @@ -6,7 +6,14 @@ RSpec.describe 'gitlab:usage data take tasks' do before do Rake.application.rake_require 'tasks/gitlab/usage_data' # stub prometheus external http calls https://gitlab.com/gitlab-org/gitlab/-/issues/245277 - stub_request(:get, %r{^http://::1:9090/api/v1/query\?query=.*}) + stub_request(:get, %r{^http[s]?://::1:9090/-/ready}) + .to_return( + status: 200, + body: [{}].to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + stub_request(:get, %r{^http[s]?://::1:9090/api/v1/query\?query=.*}) .to_return( status: 200, body: [{}].to_json, diff --git a/spec/views/notify/autodevops_disabled_email.text.erb_spec.rb b/spec/views/notify/autodevops_disabled_email.text.erb_spec.rb new file mode 100644 index 00000000000..c3cb0c83f35 --- /dev/null +++ b/spec/views/notify/autodevops_disabled_email.text.erb_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'notify/autodevops_disabled_email.text.erb' do + include Devise::Test::ControllerHelpers + + let(:user) { create(:user, developer_projects: [project]) } + let(:project) { create(:project, :repository) } + + let(:pipeline) do + create(:ci_pipeline, + :failed, + project: project, + user: user, + ref: project.default_branch, + sha: project.commit.sha) + end + + before do + assign(:project, project) + assign(:pipeline, pipeline) + end + + context 'when the pipeline contains a failed job' do + let!(:build) { create(:ci_build, :failed, :trace_live, pipeline: pipeline, project: pipeline.project) } + + it 'renders the email correctly' do + render + + expect(rendered).to have_content("Auto DevOps pipeline was disabled for #{project.name}") + expect(rendered).to match(/Pipeline ##{pipeline.id} .* triggered by #{pipeline.user.name}/) + expect(rendered).to have_content("Stage: #{build.stage}") + expect(rendered).to have_content("Name: #{build.name}") + expect(rendered).not_to have_content("Trace:") + end + end +end |
