diff options
123 files changed, 2208 insertions, 1097 deletions
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index aee815bc3ff..430587e92d6 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -34,6 +34,7 @@ - rspec_profiling/ - tmp/capybara/ - tmp/memory_test/ + - tmp/feature_flags/ - log/*.log reports: junit: junit_rspec.xml @@ -360,6 +361,7 @@ rspec:coverage: - run_timed_command "bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet --without default development test production puma unicorn kerberos metrics omnibus ed25519" - run_timed_command "bundle exec scripts/merge-simplecov" - run_timed_command "bundle exec scripts/gather-test-memory-data" + - run_timed_command "bundle exec scripts/used-feature-flags" coverage: '/LOC \((\d+\.\d+%)\) covered.$/' artifacts: name: coverage diff --git a/.gitlab/merge_request_templates/Change Documentation Location.md b/.gitlab/merge_request_templates/Change Documentation Location.md index 155694c947d..1197c6adc40 100644 --- a/.gitlab/merge_request_templates/Change Documentation Location.md +++ b/.gitlab/merge_request_templates/Change Documentation Location.md @@ -21,12 +21,10 @@ https://docs.gitlab.com/ee/development/documentation/index.html#move-or-rename-a a link to the new location. - [ ] Make sure internal links pointing to the document in question are not broken. - [ ] Search and replace any links referring to old docs in GitLab Rails app, - specifically under the `app/views/` and `ee/app/views` (for GitLab EE) directories. -- [ ] Make sure to add [`redirect_from`](https://docs.gitlab.com/ce/development/documentation/index.html#redirections-for-pages-with-disqus-comments) + specifically under the `app/views/` and `ee/app/views` (for GitLab EE) directories. +- [ ] Make sure to add [`redirect_from`](https://docs.gitlab.com/ee/development/documentation/index.html#redirections-for-pages-with-disqus-comments) to the new document if there are any Disqus comments on the old document thread. - [ ] Update the link in `features.yml` (if applicable) -- [ ] If working on CE and the `ee-compat-check` jobs fails, submit an MR to EE - with the changes as well (https://docs.gitlab.com/ce/development/documentation/index.html#cherry-picking-from-ce-to-ee). -- [ ] Ping one of the technical writers for review. +- [ ] Assign one of the technical writers for review. /label ~documentation diff --git a/.rubocop.yml b/.rubocop.yml index 2f99c57f489..eb5beb6633a 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -362,6 +362,15 @@ Graphql/AuthorizeTypes: - 'spec/**/*.rb' - 'ee/spec/**/*.rb' +Graphql/GIDExpectedType: + Enabled: true + Include: + - 'app/graphql/**/*' + - 'ee/app/graphql/**/*' + Exclude: + - 'spec/**/*.rb' + - 'ee/spec/**/*.rb' + Graphql/JSONType: Enabled: true Include: diff --git a/PROCESS.md b/PROCESS.md index 45f28b33a63..820f19a290b 100644 --- a/PROCESS.md +++ b/PROCESS.md @@ -73,7 +73,7 @@ sensitive as to how you word things. Use Emoji to express your feelings (heart, star, smile, etc.). Some good tips about code reviews can be found in our [Code Review Guidelines]. -[Code Review Guidelines]: https://docs.gitlab.com/ce/development/code_review.html +[Code Review Guidelines]: https://docs.gitlab.com/ee/development/code_review.html ## Feature flags @@ -217,5 +217,5 @@ rebase with master to see if that solves the issue. [team]: https://about.gitlab.com/team/ [done]: https://docs.gitlab.com/ee/development/contributing/merge_request_workflow.html#definition-of-done -[automatic_ce_ee_merge]: https://docs.gitlab.com/ce/development/automatic_ce_ee_merge.html -[ee_features]: https://docs.gitlab.com/ce/development/ee_features.html +[automatic_ce_ee_merge]: https://docs.gitlab.com/ee/development/automatic_ce_ee_merge.html +[ee_features]: https://docs.gitlab.com/ee/development/ee_features.html diff --git a/app/assets/javascripts/api.js b/app/assets/javascripts/api.js index 996ca4e8b73..b358cdf494f 100644 --- a/app/assets/javascripts/api.js +++ b/app/assets/javascripts/api.js @@ -376,7 +376,7 @@ const Api = { }, commitMultiple(id, data) { - // see https://docs.gitlab.com/ce/api/commits.html#create-a-commit-with-multiple-files-and-actions + // see https://docs.gitlab.com/ee/api/commits.html#create-a-commit-with-multiple-files-and-actions const url = Api.buildUrl(Api.commitsPath).replace(':id', encodeURIComponent(id)); return axios.post(url, JSON.stringify(data), { headers: { diff --git a/app/assets/javascripts/clusters/components/applications.vue b/app/assets/javascripts/clusters/components/applications.vue index 7add8d16912..b03cf6fc31b 100644 --- a/app/assets/javascripts/clusters/components/applications.vue +++ b/app/assets/javascripts/clusters/components/applications.vue @@ -360,7 +360,7 @@ export default { > <template #link="{ content }"> <gl-link - href="https://docs.gitlab.com/ce/user/project/integrations/prometheus.html" + href="https://docs.gitlab.com/ee/user/project/integrations/prometheus.html" target="_blank" >{{ content }}</gl-link > diff --git a/app/assets/javascripts/ide/components/commit_sidebar/message_field.vue b/app/assets/javascripts/ide/components/commit_sidebar/message_field.vue index 5b392470e41..7d08815b033 100644 --- a/app/assets/javascripts/ide/components/commit_sidebar/message_field.vue +++ b/app/assets/javascripts/ide/components/commit_sidebar/message_field.vue @@ -116,6 +116,7 @@ export default { :placeholder="placeholder" :value="text" class="note-textarea ide-commit-message-textarea" + data-qa-selector="ide_commit_message_field" dir="auto" name="commit-message" @scroll="handleScroll" diff --git a/app/assets/javascripts/pages/projects/ci/lints/ci_lint_editor.js b/app/assets/javascripts/pages/projects/ci/lints/ci_lint_editor.js index d270bee25c7..df635522e94 100644 --- a/app/assets/javascripts/pages/projects/ci/lints/ci_lint_editor.js +++ b/app/assets/javascripts/pages/projects/ci/lints/ci_lint_editor.js @@ -1,13 +1,11 @@ -import createFlash from '~/flash'; -import { BLOB_EDITOR_ERROR } from '~/blob_edit/constants'; +import EditorLite from '~/editor/editor_lite'; export default class CILintEditor { constructor() { - const monacoEnabled = window?.gon?.features?.monacoCi; this.clearYml = document.querySelector('.clear-yml'); this.clearYml.addEventListener('click', this.clear.bind(this)); - return monacoEnabled ? this.initEditorLite() : this.initAce(); + return this.initEditorLite(); } clear() { @@ -15,34 +13,20 @@ export default class CILintEditor { } initEditorLite() { - import(/* webpackChunkName: 'monaco_editor_lite' */ '~/editor/editor_lite') - .then(({ default: EditorLite }) => { - const editorEl = document.getElementById('editor'); - const fileContentEl = document.getElementById('content'); - const form = document.querySelector('.js-ci-lint-form'); + const editorEl = document.getElementById('editor'); + const fileContentEl = document.getElementById('content'); + const form = document.querySelector('.js-ci-lint-form'); - const rootEditor = new EditorLite(); + const rootEditor = new EditorLite(); - this.editor = rootEditor.createInstance({ - el: editorEl, - blobPath: '.gitlab-ci.yml', - blobContent: editorEl.innerText, - }); - - form.addEventListener('submit', () => { - fileContentEl.value = this.editor.getValue(); - }); - }) - .catch(() => createFlash({ message: BLOB_EDITOR_ERROR })); - } - - initAce() { - this.editor = window.ace.edit('ci-editor'); - this.textarea = document.getElementById('content'); + this.editor = rootEditor.createInstance({ + el: editorEl, + blobPath: '.gitlab-ci.yml', + blobContent: editorEl.innerText, + }); - this.editor.getSession().setMode('ace/mode/yaml'); - this.editor.on('input', () => { - this.textarea.value = this.editor.getSession().getValue(); + form.addEventListener('submit', () => { + fileContentEl.value = this.editor.getValue(); }); } } diff --git a/app/assets/javascripts/pipelines/components/pipeline_graph/drawing_utils.js b/app/assets/javascripts/pipelines/components/pipeline_graph/drawing_utils.js index b7b695376e1..45940d4a39c 100644 --- a/app/assets/javascripts/pipelines/components/pipeline_graph/drawing_utils.js +++ b/app/assets/javascripts/pipelines/components/pipeline_graph/drawing_utils.js @@ -64,10 +64,18 @@ export const generateLinksData = ({ links }, jobs, containerID) => { // Start point path.moveTo(sourceNodeX, sourceNodeY); + // Make cross-stages lines a straight line all the way + // until we can safely draw the bezier to look nice. + const straightLineDestinationX = targetNodeX - 100; + const controlPointX = straightLineDestinationX + (targetNodeX - straightLineDestinationX) / 2; + + if (straightLineDestinationX > 0) { + path.lineTo(straightLineDestinationX, sourceNodeY); + } + // Add bezier curve. The first 4 coordinates are the 2 control // points to create the curve, and the last one is the end point (x, y). // We want our control points to be in the middle of the line - const controlPointX = sourceNodeX + (targetNodeX - sourceNodeX) / 2; path.bezierCurveTo( controlPointX, sourceNodeY, diff --git a/app/assets/javascripts/sidebar/components/assignees/assignee_title.vue b/app/assets/javascripts/sidebar/components/assignees/assignee_title.vue index 5c67e429383..18c3fa9d121 100644 --- a/app/assets/javascripts/sidebar/components/assignees/assignee_title.vue +++ b/app/assets/javascripts/sidebar/components/assignees/assignee_title.vue @@ -1,6 +1,6 @@ <script> import { GlLoadingIcon } from '@gitlab/ui'; -import { n__ } from '~/locale'; +import { n__, __ } from '~/locale'; export default { name: 'AssigneeTitle', @@ -26,12 +26,19 @@ export default { required: false, default: false, }, + changing: { + type: Boolean, + required: true, + }, }, computed: { assigneeTitle() { const assignees = this.numberOfAssignees; return n__('Assignee', `%d Assignees`, assignees); }, + titleCopy() { + return this.changing ? __('Apply') : __('Edit'); + }, }, }; </script> @@ -43,11 +50,12 @@ export default { v-if="editable" class="js-sidebar-dropdown-toggle edit-link float-right" href="#" + data-test-id="edit-link" data-track-event="click_edit_button" data-track-label="right_sidebar" data-track-property="assignee" > - {{ __('Edit') }} + {{ titleCopy }} </a> <a v-if="showToggle" diff --git a/app/assets/javascripts/sidebar/components/assignees/sidebar_assignees.vue b/app/assets/javascripts/sidebar/components/assignees/sidebar_assignees.vue index 2f714ac3847..33a20d41bde 100644 --- a/app/assets/javascripts/sidebar/components/assignees/sidebar_assignees.vue +++ b/app/assets/javascripts/sidebar/components/assignees/sidebar_assignees.vue @@ -89,6 +89,8 @@ export default { .saveAssignees(this.field) .then(() => { this.loading = false; + this.store.resetChanging(); + refreshUserMergeRequestCounts(); }) .catch(() => { @@ -113,6 +115,7 @@ export default { :loading="loading || store.isFetching.assignees" :editable="store.editable" :show-toggle="!signedIn" + :changing="store.changing" /> <assignees v-if="!store.isFetching.assignees" diff --git a/app/assets/javascripts/sidebar/components/lock/issuable_lock_form.vue b/app/assets/javascripts/sidebar/components/lock/issuable_lock_form.vue index 53ee7f46ad9..b96a2b93712 100644 --- a/app/assets/javascripts/sidebar/components/lock/issuable_lock_form.vue +++ b/app/assets/javascripts/sidebar/components/lock/issuable_lock_form.vue @@ -1,8 +1,7 @@ <script> import { mapGetters } from 'vuex'; -import { GlIcon } from '@gitlab/ui'; +import { GlIcon, GlTooltipDirective } from '@gitlab/ui'; import { __ } from '~/locale'; -import tooltip from '~/vue_shared/directives/tooltip'; import eventHub from '~/sidebar/event_hub'; import editForm from './edit_form.vue'; @@ -26,7 +25,7 @@ export default { }, directives: { - tooltip, + GlTooltip: GlTooltipDirective, }, props: { @@ -79,13 +78,9 @@ export default { <template> <div class="block issuable-sidebar-item lock"> <div - v-tooltip - :title="tooltipLabel" + v-gl-tooltip.left.viewport="{ title: tooltipLabel }" class="sidebar-collapsed-icon" data-testid="sidebar-collapse-icon" - data-container="body" - data-placement="left" - data-boundary="viewport" @click="toggleForm" > <gl-icon :name="lockStatus.icon" class="sidebar-item-icon is-active" /> diff --git a/app/assets/javascripts/sidebar/stores/sidebar_store.js b/app/assets/javascripts/sidebar/stores/sidebar_store.js index 97cc83b7381..d53393052eb 100644 --- a/app/assets/javascripts/sidebar/stores/sidebar_store.js +++ b/app/assets/javascripts/sidebar/stores/sidebar_store.js @@ -33,6 +33,7 @@ export default class SidebarStore { this.projectEmailsDisabled = false; this.subscribeDisabledDescription = ''; this.subscribed = null; + this.changing = false; SidebarStore.singleton = this; } @@ -51,6 +52,10 @@ export default class SidebarStore { } } + resetChanging() { + this.changing = false; + } + setTimeTrackingData(data) { this.timeEstimate = data.time_estimate; this.totalTimeSpent = data.total_time_spent; @@ -80,6 +85,7 @@ export default class SidebarStore { addAssignee(assignee) { if (!this.findAssignee(assignee)) { + this.changing = true; this.assignees.push(assignee); } } @@ -100,6 +106,7 @@ export default class SidebarStore { removeAssignee(assignee) { if (assignee) { + this.changing = true; this.assignees = this.assignees.filter(({ id }) => id !== assignee.id); } } @@ -111,6 +118,7 @@ export default class SidebarStore { } removeAllAssignees() { + this.changing = true; this.assignees = []; } diff --git a/app/assets/stylesheets/pages/merge_requests.scss b/app/assets/stylesheets/pages/merge_requests.scss index 5835f665ada..6f71177e870 100644 --- a/app/assets/stylesheets/pages/merge_requests.scss +++ b/app/assets/stylesheets/pages/merge_requests.scss @@ -471,12 +471,6 @@ $mr-widget-min-height: 69px; flex: 1; } - .issuable-meta { - .author-link { - display: inline-block; - } - } - .merge-request-title { margin-bottom: 2px; diff --git a/app/assets/stylesheets/pages/pipelines.scss b/app/assets/stylesheets/pages/pipelines.scss index 0e8f29e793e..63715f7d08f 100644 --- a/app/assets/stylesheets/pages/pipelines.scss +++ b/app/assets/stylesheets/pages/pipelines.scss @@ -28,7 +28,10 @@ height: 14px; width: 14px; vertical-align: middle; - fill: $gl-text-color-secondary; + + &:not(.text-warning) { + fill: $gl-text-color-secondary; + } } .sprite { diff --git a/app/controllers/dashboard/labels_controller.rb b/app/controllers/dashboard/labels_controller.rb index e3773f65744..b661efa12c0 100644 --- a/app/controllers/dashboard/labels_controller.rb +++ b/app/controllers/dashboard/labels_controller.rb @@ -11,8 +11,8 @@ class Dashboard::LabelsController < Dashboard::ApplicationController def labels finder_params = { project_ids: projects.select(:id) } - labels = LabelsFinder.new(current_user, finder_params).execute - GlobalLabel.build_collection(labels) + LabelsFinder.new(current_user, finder_params).execute + .select('DISTINCT ON (labels.title) labels.*') end end diff --git a/app/finders/ci/jobs_finder.rb b/app/finders/ci/jobs_finder.rb index 8515b77ec0b..40c610f8209 100644 --- a/app/finders/ci/jobs_finder.rb +++ b/app/finders/ci/jobs_finder.rb @@ -25,7 +25,7 @@ module Ci attr_reader :current_user, :pipeline, :project, :params, :type def init_collection - if Feature.enabled?(:ci_jobs_finder_refactor) + if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true) pipeline_jobs || project_jobs || all_jobs else project ? project_builds : all_jobs @@ -59,7 +59,7 @@ module Ci end def filter_by_scope(builds) - if Feature.enabled?(:ci_jobs_finder_refactor) + if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true) return filter_by_statuses!(params[:scope], builds) if params[:scope].is_a?(Array) end diff --git a/app/finders/issuable_finder.rb b/app/finders/issuable_finder.rb index f13dc8c2451..b040e1b5b34 100644 --- a/app/finders/issuable_finder.rb +++ b/app/finders/issuable_finder.rb @@ -102,7 +102,7 @@ class IssuableFinder items = filter_items(items) # Let's see if we have to negate anything - items = filter_negated_items(items) + items = filter_negated_items(items) if should_filter_negated_args? # This has to be last as we use a CTE as an optimization fence # for counts by passing the force_cte param and passing the @@ -134,13 +134,15 @@ class IssuableFinder by_my_reaction_emoji(items) end - # Negates all params found in `negatable_params` - def filter_negated_items(items) - return items unless Feature.enabled?(:not_issuable_queries, params.group || params.project, default_enabled: true) + def should_filter_negated_args? + return false unless Feature.enabled?(:not_issuable_queries, params.group || params.project, default_enabled: true) # API endpoints send in `nil` values so we test if there are any non-nil - return items unless not_params.present? && not_params.values.any? + not_params.present? && not_params.values.any? + end + # Negates all params found in `negatable_params` + def filter_negated_items(items) items = by_negated_author(items) items = by_negated_assignee(items) items = by_negated_label(items) diff --git a/app/graphql/mutations/issues/common_mutation_arguments.rb b/app/graphql/mutations/issues/common_mutation_arguments.rb new file mode 100644 index 00000000000..7a87d9c2393 --- /dev/null +++ b/app/graphql/mutations/issues/common_mutation_arguments.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Mutations + module Issues + module CommonMutationArguments + extend ActiveSupport::Concern + + included do + argument :description, GraphQL::STRING_TYPE, + required: false, + description: copy_field_description(Types::IssueType, :description) + + argument :due_date, GraphQL::Types::ISO8601Date, + required: false, + description: copy_field_description(Types::IssueType, :due_date) + + argument :confidential, GraphQL::BOOLEAN_TYPE, + required: false, + description: copy_field_description(Types::IssueType, :confidential) + + argument :locked, GraphQL::BOOLEAN_TYPE, + as: :discussion_locked, + required: false, + description: copy_field_description(Types::IssueType, :discussion_locked) + end + end + end +end + +Mutations::Issues::CommonMutationArguments.prepend_if_ee('::EE::Mutations::Issues::CommonMutationArguments') diff --git a/app/graphql/mutations/issues/create.rb b/app/graphql/mutations/issues/create.rb new file mode 100644 index 00000000000..1454916bc77 --- /dev/null +++ b/app/graphql/mutations/issues/create.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +module Mutations + module Issues + class Create < BaseMutation + include ResolvesProject + graphql_name 'CreateIssue' + + authorize :create_issue + + include CommonMutationArguments + + argument :project_path, GraphQL::ID_TYPE, + required: true, + description: 'Project full path the issue is associated with' + + argument :iid, GraphQL::INT_TYPE, + required: false, + description: 'The IID (internal ID) of a project issue. Only admins and project owners can modify' + + argument :title, GraphQL::STRING_TYPE, + required: true, + description: copy_field_description(Types::IssueType, :title) + + argument :milestone_id, ::Types::GlobalIDType[::Milestone], + required: false, + description: 'The ID of the milestone to assign to the issue. On update milestone will be removed if set to null' + + argument :labels, [GraphQL::STRING_TYPE], + required: false, + description: copy_field_description(Types::IssueType, :labels) + + argument :label_ids, [::Types::GlobalIDType[::Label]], + required: false, + description: 'The IDs of labels to be added to the issue' + + argument :created_at, Types::TimeType, + required: false, + description: 'Timestamp when the issue was created. Available only for admins and project owners' + + argument :merge_request_to_resolve_discussions_of, ::Types::GlobalIDType[::MergeRequest], + required: false, + description: 'The IID of a merge request for which to resolve discussions' + + argument :discussion_to_resolve, GraphQL::STRING_TYPE, + required: false, + description: 'The ID of a discussion to resolve. Also pass `merge_request_to_resolve_discussions_of`' + + argument :assignee_ids, [::Types::GlobalIDType[::User]], + required: false, + description: 'The array of user IDs to assign to the issue' + + field :issue, + Types::IssueType, + null: true, + description: 'The issue after mutation' + + def ready?(**args) + if args.slice(*mutually_exclusive_label_args).size > 1 + arg_str = mutually_exclusive_label_args.map { |x| x.to_s.camelize(:lower) }.join(' or ') + raise Gitlab::Graphql::Errors::ArgumentError, "one and only one of #{arg_str} is required." + end + + if args[:discussion_to_resolve].present? && args[:merge_request_to_resolve_discussions_of].blank? + raise Gitlab::Graphql::Errors::ArgumentError, + 'to resolve a discussion please also provide `merge_request_to_resolve_discussions_of` parameter' + end + + super + end + + def resolve(project_path:, **attributes) + project = authorized_find!(full_path: project_path) + params = build_create_issue_params(attributes.merge(author_id: current_user.id)) + + issue = ::Issues::CreateService.new(project, current_user, params).execute + + if issue.spam? + issue.errors.add(:base, 'Spam detected.') + end + + { + issue: issue.valid? ? issue : nil, + errors: errors_on_object(issue) + } + end + + private + + def build_create_issue_params(params) + params[:milestone_id] &&= params[:milestone_id]&.model_id + params[:assignee_ids] &&= params[:assignee_ids].map { |assignee_id| assignee_id&.model_id } + params[:label_ids] &&= params[:label_ids].map { |label_id| label_id&.model_id } + + params + end + + def mutually_exclusive_label_args + [:labels, :label_ids] + end + + def find_object(full_path:) + resolve_project(full_path: full_path) + end + end + end +end + +Mutations::Issues::Create.prepend_if_ee('::EE::Mutations::Issues::Create') diff --git a/app/graphql/mutations/issues/update.rb b/app/graphql/mutations/issues/update.rb index 5000d2a90fb..9b216b31f9b 100644 --- a/app/graphql/mutations/issues/update.rb +++ b/app/graphql/mutations/issues/update.rb @@ -5,49 +5,26 @@ module Mutations class Update < Base graphql_name 'UpdateIssue' - argument :title, - GraphQL::STRING_TYPE, - required: false, - description: copy_field_description(Types::IssueType, :title) + include CommonMutationArguments - argument :description, - GraphQL::STRING_TYPE, - required: false, - description: copy_field_description(Types::IssueType, :description) - - argument :due_date, - Types::TimeType, - required: false, - description: copy_field_description(Types::IssueType, :due_date) - - argument :confidential, - GraphQL::BOOLEAN_TYPE, - required: false, - description: copy_field_description(Types::IssueType, :confidential) - - argument :locked, - GraphQL::BOOLEAN_TYPE, - as: :discussion_locked, + argument :title, GraphQL::STRING_TYPE, required: false, - description: copy_field_description(Types::IssueType, :discussion_locked) + description: copy_field_description(Types::IssueType, :title) - argument :add_label_ids, - [GraphQL::ID_TYPE], + argument :milestone_id, GraphQL::ID_TYPE, required: false, - description: 'The IDs of labels to be added to the issue.' + description: 'The ID of the milestone to assign to the issue. On update milestone will be removed if set to null' - argument :remove_label_ids, - [GraphQL::ID_TYPE], + argument :add_label_ids, [GraphQL::ID_TYPE], required: false, - description: 'The IDs of labels to be removed from the issue.' + description: 'The IDs of labels to be added to the issue' - argument :milestone_id, - GraphQL::ID_TYPE, + argument :remove_label_ids, [GraphQL::ID_TYPE], required: false, - description: 'The ID of the milestone to be assigned, milestone will be removed if set to null.' + description: 'The IDs of labels to be removed from the issue' argument :state_event, Types::IssueStateEventEnum, - description: 'Close or reopen an issue.', + description: 'Close or reopen an issue', required: false def resolve(project_path:, iid:, **args) diff --git a/app/graphql/mutations/notes/create/note.rb b/app/graphql/mutations/notes/create/note.rb index 5236e48026e..8caf50ab9c6 100644 --- a/app/graphql/mutations/notes/create/note.rb +++ b/app/graphql/mutations/notes/create/note.rb @@ -17,7 +17,7 @@ module Mutations discussion_id = nil if args[:discussion_id] - discussion = GitlabSchema.object_from_id(args[:discussion_id]) + discussion = GitlabSchema.object_from_id(args[:discussion_id], expected_type: ::Discussion) authorize_discussion!(discussion) discussion_id = discussion.id diff --git a/app/graphql/types/design_management/design_type.rb b/app/graphql/types/design_management/design_type.rb index 4e11a7aaf09..bab22015dc4 100644 --- a/app/graphql/types/design_management/design_type.rb +++ b/app/graphql/types/design_management/design_type.rb @@ -30,7 +30,7 @@ module Types # most recent `Version` for an issue Gitlab::SafeRequestStore.fetch([request_cache_base_key, 'stateful_version', object.issue_id, version_gid]) do if version_gid - GitlabSchema.object_from_id(version_gid)&.sync + GitlabSchema.object_from_id(version_gid, expected_type: ::DesignManagement::Version)&.sync else object.issue.design_versions.most_recent end diff --git a/app/graphql/types/mutation_type.rb b/app/graphql/types/mutation_type.rb index 64325333e08..de60279a71f 100644 --- a/app/graphql/types/mutation_type.rb +++ b/app/graphql/types/mutation_type.rb @@ -23,6 +23,7 @@ module Types mount_mutation Mutations::Branches::Create, calls_gitaly: true mount_mutation Mutations::Commits::Create, calls_gitaly: true mount_mutation Mutations::Discussions::ToggleResolve + mount_mutation Mutations::Issues::Create mount_mutation Mutations::Issues::SetAssignees mount_mutation Mutations::Issues::SetConfidential mount_mutation Mutations::Issues::SetLocked diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 6ffb9b7642a..0053b4dc70f 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -61,4 +61,8 @@ class ApplicationRecord < ActiveRecord::Base def self.underscore Gitlab::SafeRequestStore.fetch("model:#{self}:underscore") { self.to_s.underscore } end + + def self.where_exists(query) + where('EXISTS (?)', query.select(1)) + end end diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 5fabed8feec..6926ccd9438 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -136,6 +136,10 @@ module Ci # We are using optimistic locking combined with Redis locking to ensure # that a chunk gets migrated properly. # + # We are catching an exception related to an exclusive lock not being + # acquired because it is creating a lot of noise, and is a result of + # duplicated workers running in parallel for the same build trace chunk. + # def persist_data! in_lock(*lock_params) do # exclusive Redis lock is acquired first raise FailedToPersistDataError, 'Modifed build trace chunk detected' if has_changes_to_save? @@ -144,6 +148,8 @@ module Ci chunk.unsafe_persist_data! # we migrate the data and update data store end end + rescue FailedToObtainLockError + metrics.increment_trace_operation(operation: :stalled) rescue ActiveRecord::StaleObjectError raise FailedToPersistDataError, <<~MSG Data migration race condition detected diff --git a/app/models/global_label.rb b/app/models/global_label.rb deleted file mode 100644 index 7c020dd3b3d..00000000000 --- a/app/models/global_label.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -class GlobalLabel - include Presentable - - attr_accessor :title, :labels - alias_attribute :name, :title - - delegate :color, :text_color, :description, :scoped_label?, to: :@first_label - - def for_display - @first_label - end - - def self.build_collection(labels) - labels = labels.group_by(&:title) - - labels.map do |title, labels| - new(title, labels) - end - end - - def initialize(title, labels) - @title = title - @labels = labels - @first_label = labels.find { |lbl| lbl.description.present? } || labels.first - end - - def present(attributes) - super(attributes.merge(presenter_class: ::LabelPresenter)) - end -end diff --git a/app/models/packages/event.rb b/app/models/packages/event.rb index 42b92f3d671..f1d0af64ccd 100644 --- a/app/models/packages/event.rb +++ b/app/models/packages/event.rb @@ -3,7 +3,6 @@ class Packages::Event < ApplicationRecord belongs_to :package, optional: true - # FIXME: Remove debian: 9 from here when it's added to the types in package.rb model EVENT_SCOPES = ::Packages::Package.package_types.merge(container: 1000, tag: 1001).freeze enum event_scope: EVENT_SCOPES diff --git a/app/presenters/project_presenter.rb b/app/presenters/project_presenter.rb index 0be8a4d5472..4732f895b0d 100644 --- a/app/presenters/project_presenter.rb +++ b/app/presenters/project_presenter.rb @@ -106,14 +106,26 @@ class ProjectPresenter < Gitlab::View::Presenter::Delegated add_special_file_path(file_name: 'LICENSE') end + def add_license_ide_path + ide_edit_path(project, default_branch_or_master, 'LICENSE') + end + def add_changelog_path add_special_file_path(file_name: 'CHANGELOG') end + def add_changelog_ide_path + ide_edit_path(project, default_branch_or_master, 'CHANGELOG') + end + def add_contribution_guide_path add_special_file_path(file_name: 'CONTRIBUTING.md', commit_message: 'Add CONTRIBUTING') end + def add_contribution_guide_ide_path + ide_edit_path(project, default_branch_or_master, 'CONTRIBUTING.md') + end + def add_ci_yml_path add_special_file_path(file_name: ci_config_path_or_default) end @@ -122,6 +134,10 @@ class ProjectPresenter < Gitlab::View::Presenter::Delegated add_special_file_path(file_name: 'README.md') end + def add_readme_ide_path + ide_edit_path(project, default_branch_or_master, 'README.md') + end + def license_short_name license = repository.license license&.nickname || license&.name || 'LICENSE' @@ -218,9 +234,11 @@ class ProjectPresenter < Gitlab::View::Presenter::Delegated def new_file_anchor_data if current_user && can_current_user_push_to_default_branch? + new_file_path = empty_repo? ? ide_edit_path(project, default_branch_or_master) : project_new_blob_path(project, default_branch_or_master) + AnchorData.new(false, statistic_icon + _('New file'), - project_new_blob_path(project, default_branch_or_master), + new_file_path, 'missing') end end @@ -229,7 +247,7 @@ class ProjectPresenter < Gitlab::View::Presenter::Delegated if current_user && can_current_user_push_to_default_branch? && repository.readme.nil? AnchorData.new(false, statistic_icon + _('Add README'), - add_readme_path) + empty_repo? ? add_readme_ide_path : add_readme_path) elsif repository.readme AnchorData.new(false, statistic_icon('doc-text') + _('README'), @@ -243,7 +261,7 @@ class ProjectPresenter < Gitlab::View::Presenter::Delegated if current_user && can_current_user_push_to_default_branch? && repository.changelog.blank? AnchorData.new(false, statistic_icon + _('Add CHANGELOG'), - add_changelog_path) + empty_repo? ? add_changelog_ide_path : add_changelog_path) elsif repository.changelog.present? AnchorData.new(false, statistic_icon('doc-text') + _('CHANGELOG'), @@ -264,7 +282,7 @@ class ProjectPresenter < Gitlab::View::Presenter::Delegated if current_user && can_current_user_push_to_default_branch? AnchorData.new(false, content_tag(:span, statistic_icon + _('Add LICENSE'), class: 'add-license-link d-flex'), - add_license_path) + empty_repo? ? add_license_ide_path : add_license_path) else AnchorData.new(false, icon + content_tag(:span, _('No license. All rights reserved'), class: 'project-stat-value'), @@ -277,7 +295,7 @@ class ProjectPresenter < Gitlab::View::Presenter::Delegated if current_user && can_current_user_push_to_default_branch? && repository.contribution_guide.blank? AnchorData.new(false, statistic_icon + _('Add CONTRIBUTING'), - add_contribution_guide_path) + empty_repo? ? add_contribution_guide_ide_path : add_contribution_guide_path) elsif repository.contribution_guide.present? AnchorData.new(false, statistic_icon('doc-text') + _('CONTRIBUTING'), diff --git a/app/serializers/build_details_entity.rb b/app/serializers/build_details_entity.rb index 6da3261a4a1..109213ab729 100644 --- a/app/serializers/build_details_entity.rb +++ b/app/serializers/build_details_entity.rb @@ -133,7 +133,7 @@ class BuildDetailsEntity < JobEntity def callout_message return super unless build.failure_reason.to_sym == :missing_dependency_failure - docs_url = "https://docs.gitlab.com/ce/ci/yaml/README.html#dependencies" + docs_url = "https://docs.gitlab.com/ee/ci/yaml/README.html#dependencies" [ failure_message.html_safe, diff --git a/app/serializers/label_entity.rb b/app/serializers/label_entity.rb index be42c6dd57f..e586d7f8407 100644 --- a/app/serializers/label_entity.rb +++ b/app/serializers/label_entity.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true class LabelEntity < Grape::Entity - expose :id, if: ->(label, _) { !label.is_a?(GlobalLabel) } + expose :id expose :title expose :color expose :description expose :group_id - expose :project_id, if: ->(label, _) { !label.is_a?(GlobalLabel) } + expose :project_id expose :template expose :text_color expose :created_at diff --git a/app/services/issues/base_service.rb b/app/services/issues/base_service.rb index 0ed2b08b7b1..978ea6fe9bc 100644 --- a/app/services/issues/base_service.rb +++ b/app/services/issues/base_service.rb @@ -34,6 +34,18 @@ module Issues private + def filter_params(merge_request) + super + + moved_issue = params.delete(:moved_issue) + + # Setting created_at, updated_at and iid is allowed only for admins and owners or + # when moving an issue as we preserve the original issue attributes except id and iid. + params.delete(:iid) unless current_user.can?(:set_issue_iid, project) + params.delete(:created_at) unless moved_issue || current_user.can?(:set_issue_created_at, project) + params.delete(:updated_at) unless moved_issue || current_user.can?(:set_issue_updated_at, project) + end + def create_assignee_note(issue, old_assignees) SystemNoteService.change_issuable_assignees( issue, issue.project, current_user, old_assignees) diff --git a/app/services/issues/move_service.rb b/app/services/issues/move_service.rb index d7c14168338..90ccbd8ed21 100644 --- a/app/services/issues/move_service.rb +++ b/app/services/issues/move_service.rb @@ -52,7 +52,8 @@ module Issues iid: nil, project: target_project, author: original_entity.author, - assignee_ids: original_entity.assignee_ids + assignee_ids: original_entity.assignee_ids, + moved_issue: true } new_params = original_entity.serializable_hash.symbolize_keys.merge(new_params) diff --git a/app/views/groups/settings/_advanced.html.haml b/app/views/groups/settings/_advanced.html.haml index f415ca79bd4..c421a569a14 100644 --- a/app/views/groups/settings/_advanced.html.haml +++ b/app/views/groups/settings/_advanced.html.haml @@ -32,7 +32,7 @@ = hidden_field_tag 'new_parent_group_id' %ul - - side_effects_link_start = '<a href="https://docs.gitlab.com/ce/user/project/index.html#redirects-when-changing-repository-paths" target="_blank">' + - side_effects_link_start = '<a href="https://docs.gitlab.com/ee/user/project/index.html#redirects-when-changing-repository-paths" target="_blank">' - warning_text = s_("GroupSettings|Be careful. Changing a group's parent can have unintended %{side_effects_link_start}side effects%{side_effects_link_end}.") % { side_effects_link_start: side_effects_link_start, side_effects_link_end:'</a>' } %li= warning_text.html_safe %li= s_('GroupSettings|You can only transfer the group to a group you manage.') diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index b01665daff4..138f5569218 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -22,7 +22,7 @@ - if ref - if job.ref .icon-container.gl-display-inline-block - = job.tag? ? icon('tag') : sprite_icon('fork', css_class: 'sprite') + = job.tag? ? sprite_icon('label', css_class: 'sprite') : sprite_icon('fork', css_class: 'sprite') = link_to job.ref, project_ref_path(job.project, job.ref), class: "ref-name" - else .light= _('none') @@ -33,10 +33,12 @@ = link_to job.short_sha, project_commit_path(job.project, job.sha), class: "commit-sha mr-0" - if job.stuck? - = icon('warning', class: 'text-warning has-tooltip', title: _('Job is stuck. Check runners.')) + %span.has-tooltip{ title: _('Job is stuck. Check runners.') } + = sprite_icon('warning', css_class: 'text-warning!') - if retried - = icon('refresh', class: 'text-warning has-tooltip', title: _('Job was retried')) + %span.has-tooltip{ title: _('Job was retried') } + = sprite_icon('retry', css_class: 'text-warning') .label-container - if job.tags.any? @@ -87,7 +89,7 @@ - if job.finished_at %p.finished-at - = icon("calendar") + = sprite_icon("calendar") %span= time_ago_with_tooltip(job.finished_at) %td.coverage diff --git a/app/views/projects/ci/lints/show.html.haml b/app/views/projects/ci/lints/show.html.haml index 1b674c19cfa..64f250bd607 100644 --- a/app/views/projects/ci/lints/show.html.haml +++ b/app/views/projects/ci/lints/show.html.haml @@ -1,8 +1,5 @@ - page_title _("CI Lint") - page_description _("Validate your GitLab CI configuration file") -- unless Feature.enabled?(:monaco_ci, default_enabled: true) - - content_for :page_specific_javascripts do - = page_specific_javascript_tag('lib/ace.js') %h2.pt-3.pb-3= _("Validate your GitLab CI configuration") @@ -17,12 +14,9 @@ .file-holder .js-file-title.file-title.clearfix = _("Contents of .gitlab-ci.yml") - - if Feature.enabled?(:monaco_ci, default_enabled: true) - .file-editor.code - .js-edit-mode-pane.qa-editor#editor{ data: { 'editor-loading': true } }< - %pre.editor-loading-content= params[:content] - - else - #ci-editor.ci-editor= @content + .file-editor.code + .js-edit-mode-pane.qa-editor#editor{ data: { 'editor-loading': true } }< + %pre.editor-loading-content= params[:content] = text_area_tag(:content, @content, class: 'hidden form-control span1', rows: 7, require: true) .col-sm-12 .float-left.gl-mt-3 diff --git a/app/views/projects/merge_requests/_merge_request.html.haml b/app/views/projects/merge_requests/_merge_request.html.haml index 8f8438d3eaf..092055a5f85 100644 --- a/app/views/projects/merge_requests/_merge_request.html.haml +++ b/app/views/projects/merge_requests/_merge_request.html.haml @@ -53,7 +53,7 @@ = link_to merge_request_path(merge_request), class: "has-tooltip", title: _('Cannot be merged automatically') do = sprite_icon('warning-solid') - if merge_request.assignees.any? - %li.gl-display-flex + %li.gl-display-flex.gl-align-items-center = render 'shared/issuable/assignees', project: merge_request.project, issuable: merge_request - if Feature.enabled?(:merge_request_reviewers, @project) && merge_request.reviewers.any? %li.gl-display-flex.issuable-reviewers diff --git a/app/views/projects/tree/show.html.haml b/app/views/projects/tree/show.html.haml index 8c874ed1505..4d8c357cee1 100644 --- a/app/views/projects/tree/show.html.haml +++ b/app/views/projects/tree/show.html.haml @@ -1,5 +1,5 @@ - current_route_path = request.fullpath.match(/-\/tree\/[^\/]+\/(.+$)/).to_a[1] -- add_page_startup_graphql_call('repository/path_last_commit', { projectPath: @project.full_path, ref: current_ref, currentRoutePath: current_route_path }) +- add_page_startup_graphql_call('repository/path_last_commit', { projectPath: @project.full_path, ref: current_ref, path: current_route_path }) - breadcrumb_title _("Repository") - @content_class = "limit-container-width" unless fluid_layout diff --git a/changelogs/unreleased/229838-create-issue-graphql.yml b/changelogs/unreleased/229838-create-issue-graphql.yml new file mode 100644 index 00000000000..fb3e1976cc8 --- /dev/null +++ b/changelogs/unreleased/229838-create-issue-graphql.yml @@ -0,0 +1,5 @@ +--- +title: Add GraphQL mutation to create an issue +merge_request: 43735 +author: +type: added diff --git a/changelogs/unreleased/259060-align-badge-in-mr-list.yml b/changelogs/unreleased/259060-align-badge-in-mr-list.yml new file mode 100644 index 00000000000..e99194b4b51 --- /dev/null +++ b/changelogs/unreleased/259060-align-badge-in-mr-list.yml @@ -0,0 +1,5 @@ +--- +title: Align badge with avatar in MR List +merge_request: 44671 +author: +type: fixed diff --git a/changelogs/unreleased/27535-new-project-ide.yml b/changelogs/unreleased/27535-new-project-ide.yml new file mode 100644 index 00000000000..ce06f1c0380 --- /dev/null +++ b/changelogs/unreleased/27535-new-project-ide.yml @@ -0,0 +1,5 @@ +--- +title: Use Web IDE to create new files in empty repos +merge_request: 44287 +author: +type: added diff --git a/changelogs/unreleased/mw-replace-fa-refresh.yml b/changelogs/unreleased/mw-replace-fa-refresh.yml new file mode 100644 index 00000000000..2b26631d981 --- /dev/null +++ b/changelogs/unreleased/mw-replace-fa-refresh.yml @@ -0,0 +1,5 @@ +--- +title: Replace fa icons in CI build table +merge_request: 45123 +author: +type: changed diff --git a/changelogs/unreleased/ntepluhina-fix-incorrect-parameter.yml b/changelogs/unreleased/ntepluhina-fix-incorrect-parameter.yml new file mode 100644 index 00000000000..3764efeeb8b --- /dev/null +++ b/changelogs/unreleased/ntepluhina-fix-incorrect-parameter.yml @@ -0,0 +1,5 @@ +--- +title: Fixed incorrect parameter in GraphQL startup call +merge_request: 45115 +author: +type: fixed diff --git a/changelogs/unreleased/ss-add-apply-to-assignees.yml b/changelogs/unreleased/ss-add-apply-to-assignees.yml new file mode 100644 index 00000000000..ed370175363 --- /dev/null +++ b/changelogs/unreleased/ss-add-apply-to-assignees.yml @@ -0,0 +1,5 @@ +--- +title: Add apply button when user changes assignees +merge_request: 44812 +author: +type: added diff --git a/config/feature_flags/development/approval_rule.yml b/config/feature_flags/development/approval_rule.yml deleted file mode 100644 index 6b38fbc0db8..00000000000 --- a/config/feature_flags/development/approval_rule.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: approval_rule -introduced_by_url: -rollout_issue_url: -group: -type: development -default_enabled: true diff --git a/config/feature_flags/development/ci_jobs_finder_refactor.yml b/config/feature_flags/development/ci_jobs_finder_refactor.yml index 13b8fa03477..f43db747e0a 100644 --- a/config/feature_flags/development/ci_jobs_finder_refactor.yml +++ b/config/feature_flags/development/ci_jobs_finder_refactor.yml @@ -4,4 +4,4 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36622 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/245183 group: group::continuous integration type: development -default_enabled: false +default_enabled: true diff --git a/config/feature_flags/development/monaco_ci.yml b/config/feature_flags/development/monaco_ci.yml deleted file mode 100644 index 0f1c8b521f2..00000000000 --- a/config/feature_flags/development/monaco_ci.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: monaco_ci -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23666 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/249137 -group: group::editor -type: development -default_enabled: true diff --git a/config/feature_flags/development/release_asset_link_editing.yml b/config/feature_flags/development/release_asset_link_editing.yml deleted file mode 100644 index 6e5a0ba1b52..00000000000 --- a/config/feature_flags/development/release_asset_link_editing.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: release_asset_link_editing -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26821 -rollout_issue_url: -group: group::release management -type: development -default_enabled: true diff --git a/config/feature_flags/development/release_show_page.yml b/config/feature_flags/development/release_show_page.yml deleted file mode 100644 index fd18eca1b25..00000000000 --- a/config/feature_flags/development/release_show_page.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: release_show_page -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23792 -rollout_issue_url: -group: group::release management -type: development -default_enabled: true diff --git a/config/feature_flags/development/sql-set-operators.yml b/config/feature_flags/development/sql-set-operators.yml deleted file mode 100644 index cefe2a83782..00000000000 --- a/config/feature_flags/development/sql-set-operators.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: sql-set-operators -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39786 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39786 -group: group::access -type: development -default_enabled: false diff --git a/config/feature_flags/development/sql_set_operators.yml b/config/feature_flags/development/sql_set_operators.yml index 196753fcd74..2098a19a24a 100644 --- a/config/feature_flags/development/sql_set_operators.yml +++ b/config/feature_flags/development/sql_set_operators.yml @@ -1,7 +1,7 @@ --- name: sql_set_operators -introduced_by_url: -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39786#f99799ae4964b7650b877e081b669379d71bcca8 +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/39786 +rollout_issue_url: group: group::access type: development default_enabled: false diff --git a/config/initializers/0_marginalia.rb b/config/initializers/0_marginalia.rb index a697f67dbf2..5c6cf7752c4 100644 --- a/config/initializers/0_marginalia.rb +++ b/config/initializers/0_marginalia.rb @@ -21,4 +21,4 @@ Gitlab::Marginalia.set_application_name Gitlab::Marginalia.enable_sidekiq_instrumentation -Gitlab::Marginalia.set_feature_cache +Gitlab::Marginalia.set_enabled_from_feature_flag diff --git a/danger/pajamas/Dangerfile b/danger/pajamas/Dangerfile index 744fa902817..36bf7672cbf 100644 --- a/danger/pajamas/Dangerfile +++ b/danger/pajamas/Dangerfile @@ -26,18 +26,18 @@ PATTERNS = %w[ gl-deprecated-dropdown-divider gl-deprecated-dropdown-header gl-deprecated-dropdown-item - graphql_pagination has-tooltip has_tooltip initDeprecatedJQueryDropdown loading-button - pagination-button v-popover v-tooltip with_tooltip ].freeze BLOCKING_PATTERNS = %w[ + pagination-button + graphql_pagination ].freeze def get_added_lines(files) diff --git a/doc/administration/index.md b/doc/administration/index.md index fda3816312c..28fd8cc402b 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -2,7 +2,7 @@ description: 'Learn how to install, configure, update, and maintain your GitLab instance.' --- -# Administrator Docs **(CORE ONLY)** +# Administrator documentation **(CORE ONLY)** Learn how to administer your self-managed GitLab instance. @@ -12,18 +12,16 @@ GitLab has two product distributions available through [different subscriptions] - The open core [GitLab Enterprise Edition (EE)](https://gitlab.com/gitlab-org/gitlab). You can [install either GitLab CE or GitLab EE](https://about.gitlab.com/install/ce-or-ee/). -However, the features you'll have access to depend on the subscription you choose -(Core, Starter, Premium, or Ultimate). +However, the features you have access to depend on your chosen [subscription](https://about.gitlab.com/pricing/). -NOTE: **Note:** -GitLab Community Edition installations only have access to Core features. +GitLab Community Edition installations have access only to Core features. -GitLab.com is administered by GitLab, Inc., therefore, only GitLab team members have -access to its admin configurations. If you're a GitLab.com user, please check the -[user documentation](../user/index.md). +Non-administrator users can't access GitLab administration tools and settings. -NOTE: **Note:** -Non-administrator users don’t have access to GitLab administration tools and settings. +GitLab.com is administered by GitLab, Inc., and only GitLab team members have +access to its administration tools and settings. Users of GitLab.com should +instead refer to the [User documentation](../user/index.md) for GitLab +configuration and usage documentation. ## Installing and maintaining GitLab diff --git a/doc/administration/postgresql/pgbouncer.md b/doc/administration/postgresql/pgbouncer.md index b946c0949c4..7760b197267 100644 --- a/doc/administration/postgresql/pgbouncer.md +++ b/doc/administration/postgresql/pgbouncer.md @@ -148,6 +148,35 @@ ote_pid | tls (1 row) ``` +## Procedure for bypassing PgBouncer + +Some database changes have to be done directly, and not through PgBouncer. This includes database restores and GitLab upgrades (because of the database migrations). + +1. To find the primary node, run the following on a database node: + + ```shell + sudo gitlab-ctl repmgr cluster show + ``` + +1. Edit `/etc/gitlab/gitlab.rb` on the application node you're performing the task on, and update + `gitlab_rails['db_host']` and `gitlab_rails['db_port']` with the database + primary's host and port. + +1. Run reconfigure: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +Once you've performed the tasks or procedure, switch back to using PgBouncer: + +1. Change back `/etc/gitlab/gitlab.rb` to point to PgBouncer. +1. Run reconfigure: + + ```shell + sudo gitlab-ctl reconfigure + ``` + ## Troubleshooting In case you are experiencing any issues connecting through PgBouncer, the first diff --git a/doc/administration/postgresql/replication_and_failover.md b/doc/administration/postgresql/replication_and_failover.md index a2f9d9778f1..4e60d5d0f23 100644 --- a/doc/administration/postgresql/replication_and_failover.md +++ b/doc/administration/postgresql/replication_and_failover.md @@ -1445,7 +1445,7 @@ Considering these, you should carefully plan your PostgreSQL upgrade: sudo gitlab-ctl pg-upgrade -V 12 ``` -CAUTION: **Warning:** +NOTE: **Note:** Reverting PostgreSQL upgrade with `gitlab-ctl revert-pg-upgrade` has the same considerations as -`gitlab-ctl pg-upgrade`. It can be complicated and may involve deletion of the data directory. -If you need to do that, please contact GitLab support. +`gitlab-ctl pg-upgrade`. You should follow the same procedure by first stopping the replicas, +then reverting the leader, and finally reverting the replicas. diff --git a/doc/administration/reference_architectures/10k_users.md b/doc/administration/reference_architectures/10k_users.md index f4cc830da9e..20425d11b2b 100644 --- a/doc/administration/reference_architectures/10k_users.md +++ b/doc/administration/reference_architectures/10k_users.md @@ -114,19 +114,18 @@ The following list includes descriptions of each server and its assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -284,10 +283,9 @@ To configure Consul: 1. Go through the steps again for all the other Consul nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -664,7 +662,6 @@ The following IPs will be used as an example: 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - NOTE: **Note:** If an error `execute[generate databases.ini]` occurs, this is due to an existing [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4713). It will be resolved when you run a second `reconfigure` after the next step. @@ -798,25 +795,23 @@ to be used with GitLab. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 -NOTE: **Providing your own Redis instance:** -Managed Redis from cloud providers such as AWS ElastiCache will work. If these -services support high availability, be sure it is **not** the Redis Cluster type. -Redis version 5.0 or higher is required, as this is what ships with -Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions -do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +### Providing your own Redis instance + +Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these +services support high availability, be sure it _isn't_ of the Redis Cluster type. +Redis version 5.0 or higher is required, which is included with Omnibus GitLab +packages starting with GitLab 13.0. Older Redis versions don't support an +optional count argument to SPOP, which is required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). Note the Redis node's IP address or hostname, port, and password (if required). -These will be necessary when configuring the -[GitLab application servers](#configure-gitlab-rails) later. +These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). ### Configure the Redis and Sentinel Cache cluster This is the section where we install and set up the new Redis Cache instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Cache node @@ -870,10 +865,9 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Cache nodes @@ -936,10 +930,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-cache-nodes), and even after a @@ -957,13 +950,6 @@ are supported and can be added if needed. #### Configure the Sentinel Cache nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -971,6 +957,12 @@ servers. The following IPs will be used as an example: - `10.6.0.72`: Sentinel - Cache 2 - `10.6.0.73`: Sentinel - Cache 3 +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). + To configure the Sentinel Cache server: 1. SSH into the server that will host Consul/Sentinel. @@ -1083,10 +1075,9 @@ To configure the Sentinel Cache server: This is the section where we install and set up the new Redis Queues instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Queues node @@ -1145,10 +1136,9 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Queues nodes @@ -1211,10 +1201,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-queues-nodes), and even after a @@ -1232,13 +1221,6 @@ are supported and can be added if needed. #### Configure the Sentinel Queues nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -1246,6 +1228,12 @@ servers. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). + To configure the Sentinel Queues server: 1. SSH into the server that will host Sentinel. @@ -1519,11 +1507,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1699,15 +1686,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + The following IPs will be used as an example: - `10.6.0.111`: GitLab application 1 @@ -1716,7 +1702,7 @@ The following IPs will be used as an example: On each node perform the following: -1. Download/install Omnibus GitLab using **steps 1 and 2** from +1. Download and install Omnibus GitLab using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/install/). Do not complete other steps on the download page. @@ -1878,12 +1864,10 @@ On each node perform the following: sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration @@ -1895,12 +1879,11 @@ for more information. sudo gitlab-rake gitlab:db:configure ``` - NOTE: **Note:** - If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to - PostgreSQL it may be that your PgBouncer node's IP address is missing from - PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) - in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -1990,28 +1973,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -2037,16 +2036,13 @@ work. ## Configure Advanced Search **(STARTER ONLY)** -NOTE: **Note:** -Elasticsearch cluster design and requirements are dependent on your specific data. -For recommended best practices on how to set up your Elasticsearch cluster -alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). - -You can leverage Elasticsearch and enable Advanced Search for faster, more -advanced code search across your entire GitLab instance. +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. -[Learn how to set it up.](../../integration/elasticsearch.md) +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/25k_users.md b/doc/administration/reference_architectures/25k_users.md index 89e28eca5b6..442d0b66117 100644 --- a/doc/administration/reference_architectures/25k_users.md +++ b/doc/administration/reference_architectures/25k_users.md @@ -114,19 +114,18 @@ The following list includes descriptions of each server and its assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -284,10 +283,9 @@ To configure Consul: 1. Go through the steps again for all the other Consul nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -664,7 +662,6 @@ The following IPs will be used as an example: 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - NOTE: **Note:** If an error `execute[generate databases.ini]` occurs, this is due to an existing [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4713). It will be resolved when you run a second `reconfigure` after the next step. @@ -798,25 +795,23 @@ to be used with GitLab. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 -NOTE: **Providing your own Redis instance:** -Managed Redis from cloud providers such as AWS ElastiCache will work. If these -services support high availability, be sure it is **not** the Redis Cluster type. -Redis version 5.0 or higher is required, as this is what ships with -Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions -do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +### Providing your own Redis instance + +Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these +services support high availability, be sure it _isn't_ of the Redis Cluster type. +Redis version 5.0 or higher is required, which is included with Omnibus GitLab +packages starting with GitLab 13.0. Older Redis versions don't support an +optional count argument to SPOP, which is required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). Note the Redis node's IP address or hostname, port, and password (if required). -These will be necessary when configuring the -[GitLab application servers](#configure-gitlab-rails) later. +These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). ### Configure the Redis and Sentinel Cache cluster This is the section where we install and set up the new Redis Cache instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Cache node @@ -870,10 +865,9 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Cache nodes @@ -936,10 +930,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-cache-nodes), and even after a @@ -957,13 +950,6 @@ are supported and can be added if needed. #### Configure the Sentinel Cache nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -971,6 +957,12 @@ servers. The following IPs will be used as an example: - `10.6.0.72`: Sentinel - Cache 2 - `10.6.0.73`: Sentinel - Cache 3 +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). + To configure the Sentinel Cache server: 1. SSH into the server that will host Consul/Sentinel. @@ -1083,10 +1075,9 @@ To configure the Sentinel Cache server: This is the section where we install and set up the new Redis Queues instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Queues node @@ -1145,10 +1136,9 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Queues nodes @@ -1211,10 +1201,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-queues-nodes), and even after a @@ -1232,13 +1221,6 @@ are supported and can be added if needed. #### Configure the Sentinel Queues nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -1246,6 +1228,12 @@ servers. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). + To configure the Sentinel Queues server: 1. SSH into the server that will host Sentinel. @@ -1519,11 +1507,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1699,15 +1686,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + The following IPs will be used as an example: - `10.6.0.111`: GitLab application 1 @@ -1716,7 +1702,7 @@ The following IPs will be used as an example: On each node perform the following: -1. Download/install Omnibus GitLab using **steps 1 and 2** from +1. Download and install Omnibus GitLab using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/install/). Do not complete other steps on the download page. @@ -1878,12 +1864,10 @@ On each node perform the following: sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration @@ -1895,12 +1879,11 @@ for more information. sudo gitlab-rake gitlab:db:configure ``` - NOTE: **Note:** - If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to - PostgreSQL it may be that your PgBouncer node's IP address is missing from - PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) - in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -1990,28 +1973,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -2037,16 +2036,13 @@ work. ## Configure Advanced Search **(STARTER ONLY)** -NOTE: **Note:** -Elasticsearch cluster design and requirements are dependent on your specific data. -For recommended best practices on how to set up your Elasticsearch cluster -alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). - -You can leverage Elasticsearch and enable Advanced Search for faster, more -advanced code search across your entire GitLab instance. +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. -[Learn how to set it up.](../../integration/elasticsearch.md) +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/2k_users.md b/doc/administration/reference_architectures/2k_users.md index 4a6dbf1520f..16e283920ae 100644 --- a/doc/administration/reference_architectures/2k_users.md +++ b/doc/administration/reference_architectures/2k_users.md @@ -64,18 +64,17 @@ To set up GitLab and its components to accommodate up to 2,000 users: ## Configure the external load balancer -NOTE: **Note:** -This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/). -Although you can use a load balancer with a similar set of features, GitLab -hasn't validated other load balancers. - In an active/active GitLab configuration, you'll need a load balancer to route -traffic to the application servers. The specifics for which load balancer to -use or its exact configuration is out of scope for the GitLab documentation. -If you're managing multi-node systems (including GitLab) you'll probably -already have a load balancer of choice. Some examples including HAProxy -(open-source), F5 Big-IP LTM, and Citrix Net Scaler. This documentation -includes the ports and protocols for use with GitLab. +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + +This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) +as the load balancer. Although other load balancers with similar feature sets +could also be used, those load balancers have not been validated. The next question is how you will handle SSL in your environment. There are several different options: @@ -489,11 +488,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -537,14 +535,14 @@ To configure Gitaly with TLS: ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. + +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + On each node perform the following: 1. If you're [using NFS](#configure-nfs-optional): @@ -572,10 +570,10 @@ On each node perform the following: mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data ``` -1. Download/install Omnibus GitLab using **steps 1 and 2** from +1. Download and install Omnibus GitLab using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/install/). Do not complete other steps on the download page. -1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. +1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration. To maintain uniformity of links across nodes, the `external_url` on the application server should point to the external URL that users will use to access GitLab. This would be the URL of the [load balancer](#configure-the-external-load-balancer) @@ -671,12 +669,10 @@ On each node perform the following: [Gitaly node](#configure-gitaly) and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration @@ -688,12 +684,11 @@ for more information. sudo gitlab-rake gitlab:db:configure ``` - NOTE: **Note:** - If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to - PostgreSQL it may be that your PgBouncer node's IP address is missing from - PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) - in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -832,30 +827,44 @@ data, and is recommended over [NFS](#configure-nfs-optional). In general, object storage services are better for larger environments, as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has either tested or is aware of customers -using, includes: - -- SaaS/Cloud solutions (such as [Amazon S3](https://aws.amazon.com/s3/) or - [Google Cloud Storage](https://cloud.google.com/storage)). -- On-premises hardware and appliances, from various storage vendors. -- MinIO ([Deployment guide](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html)). - -To configure GitLab to use object storage, refer to the following guides based -on the features you intend to use: - -1. [Object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. [Object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. [Object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. [Object storage for uploads](../uploads.md#using-object-storage). -1. [Object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. [Object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. [Object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. [Object storage for packages](../packages/index.md#using-object-storage) (optional feature). -1. [Object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. [Object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. [Object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional, for improved performance). -1. [Object storage for Terraform state files](../terraform_state.md#using-object-storage). +GitLab has been tested on a number of object storage providers: + +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) +- On-premises hardware and appliances from various storage vendors. +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -879,16 +888,13 @@ functioning backups is encountered. ## Configure Advanced Search **(STARTER ONLY)** -NOTE: **Note:** -Elasticsearch cluster design and requirements are dependent on your specific data. -For recommended best practices on how to set up your Elasticsearch cluster -alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). - -You can leverage Elasticsearch and enable Advanced Search for faster, more -advanced code search across your entire GitLab instance. +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. -[Learn how to set it up.](../../integration/elasticsearch.md) +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/3k_users.md b/doc/administration/reference_architectures/3k_users.md index a6ef051aad5..c989d5cdaff 100644 --- a/doc/administration/reference_architectures/3k_users.md +++ b/doc/administration/reference_architectures/3k_users.md @@ -109,19 +109,18 @@ The following list includes descriptions of each server and its assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -279,14 +278,13 @@ The requirements for a Redis setup are the following: ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), using a firewall. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configuring the primary Redis instance -1. SSH into the **Primary** Redis server. +1. SSH in to the **Primary** Redis server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using **steps 1 and 2** from the GitLab downloads page. - Make sure you select the correct Omnibus package, with the same version @@ -336,18 +334,17 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). -You can list the current Redis Primary, Replica status via: +You can list the current Redis Primary, Replica status by using: ```shell /opt/gitlab/embedded/bin/redis-cli -h <host> -a 'redis-password-goes-here' info replication ``` -Show running GitLab services via: +Show running GitLab services by using: ```shell gitlab-ctl status @@ -365,7 +362,7 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s #### Configuring the replica Redis instances -1. SSH into the **replica** Redis server. +1. SSH in to the **replica** Redis server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using **steps 1 and 2** from the GitLab downloads page. - Make sure you select the correct Omnibus package, with the same version @@ -424,10 +421,9 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-consul-and-sentinel), and even after a @@ -445,13 +441,6 @@ are supported and can be added if needed. ## Configure Consul and Sentinel -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -459,6 +448,12 @@ servers. The following IPs will be used as an example: - `10.6.0.12`: Consul/Sentinel 2 - `10.6.0.13`: Consul/Sentinel 3 +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). + To configure the Sentinel: 1. SSH into the server that will host Consul/Sentinel. @@ -558,10 +553,9 @@ To configure the Sentinel: 1. Go through the steps again for all the other Consul/Sentinel nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -1258,11 +1252,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1430,14 +1423,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. + +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + On each node perform the following: 1. If you're [using NFS](#configure-nfs-optional): @@ -1465,10 +1458,10 @@ On each node perform the following: mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data ``` -1. Download/install Omnibus GitLab using **steps 1 and 2** from +1. Download and install Omnibus GitLab using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/install/). Do not complete other steps on the download page. -1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. +1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration. To maintain uniformity of links across nodes, the `external_url` on the application server should point to the external URL that users will use to access GitLab. This would be the URL of the [external load balancer](#configure-the-external-load-balancer) @@ -1604,12 +1597,10 @@ On each node perform the following: run: puma: (pid 4936) 8645s; run: log: (pid 29656) 79161s ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration @@ -1619,12 +1610,11 @@ for more information. gitlab-rake gitlab:db:configure ``` - NOTE: **Note:** - If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to - PostgreSQL it may be that your PgBouncer node's IP address is missing from - PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) - in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -1718,28 +1708,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -1765,16 +1771,13 @@ work. ## Configure Advanced Search **(STARTER ONLY)** -NOTE: **Note:** -Elasticsearch cluster design and requirements are dependent on your specific data. -For recommended best practices on how to set up your Elasticsearch cluster -alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. -You can leverage Elasticsearch and enable Advanced Search for faster, more -advanced code search across your entire GitLab instance. - -[Learn how to set it up.](../../integration/elasticsearch.md) +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/50k_users.md b/doc/administration/reference_architectures/50k_users.md index fff6090a36a..9e19b6bbe91 100644 --- a/doc/administration/reference_architectures/50k_users.md +++ b/doc/administration/reference_architectures/50k_users.md @@ -114,19 +114,18 @@ The following list includes descriptions of each server and its assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -284,10 +283,9 @@ To configure Consul: 1. Go through the steps again for all the other Consul nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -664,7 +662,6 @@ The following IPs will be used as an example: 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. - NOTE: **Note:** If an error `execute[generate databases.ini]` occurs, this is due to an existing [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/4713). It will be resolved when you run a second `reconfigure` after the next step. @@ -798,25 +795,23 @@ to be used with GitLab. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 -NOTE: **Providing your own Redis instance:** -Managed Redis from cloud providers such as AWS ElastiCache will work. If these -services support high availability, be sure it is **not** the Redis Cluster type. -Redis version 5.0 or higher is required, as this is what ships with -Omnibus GitLab packages starting with GitLab 13.0. Older Redis versions -do not support an optional count argument to SPOP which is now required for -[Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). +### Providing your own Redis instance + +Managed Redis from cloud providers (such as AWS ElastiCache) will work. If these +services support high availability, be sure it _isn't_ of the Redis Cluster type. +Redis version 5.0 or higher is required, which is included with Omnibus GitLab +packages starting with GitLab 13.0. Older Redis versions don't support an +optional count argument to SPOP, which is required for [Merge Trains](../../ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md). Note the Redis node's IP address or hostname, port, and password (if required). -These will be necessary when configuring the -[GitLab application servers](#configure-gitlab-rails) later. +These will be necessary later when configuring the [GitLab application servers](#configure-gitlab-rails). ### Configure the Redis and Sentinel Cache cluster This is the section where we install and set up the new Redis Cache instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Cache node @@ -870,10 +865,9 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Cache nodes @@ -936,10 +930,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-cache-nodes), and even after a @@ -957,13 +950,6 @@ are supported and can be added if needed. #### Configure the Sentinel Cache nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -971,6 +957,12 @@ servers. The following IPs will be used as an example: - `10.6.0.72`: Sentinel - Cache 2 - `10.6.0.73`: Sentinel - Cache 3 +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). + To configure the Sentinel Cache server: 1. SSH into the server that will host Consul/Sentinel. @@ -1083,10 +1075,9 @@ To configure the Sentinel Cache server: This is the section where we install and set up the new Redis Queues instances. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configure the primary Redis Queues node @@ -1145,10 +1136,9 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). #### Configure the replica Redis Queues nodes @@ -1211,10 +1201,9 @@ Read more about [roles](https://docs.gitlab.com/omnibus/roles/). 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-the-sentinel-queues-nodes), and even after a @@ -1232,13 +1221,6 @@ are supported and can be added if needed. #### Configure the Sentinel Queues nodes -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -1246,6 +1228,12 @@ servers. The following IPs will be used as an example: - `10.6.0.82`: Sentinel - Queues 2 - `10.6.0.83`: Sentinel - Queues 3 +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). + To configure the Sentinel Queues server: 1. SSH into the server that will host Sentinel. @@ -1519,11 +1507,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1699,15 +1686,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + The following IPs will be used as an example: - `10.6.0.111`: GitLab application 1 @@ -1716,7 +1702,7 @@ The following IPs will be used as an example: On each node perform the following: -1. Download/install Omnibus GitLab using **steps 1 and 2** from +1. Download and install Omnibus GitLab using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/install/). Do not complete other steps on the download page. @@ -1878,12 +1864,10 @@ On each node perform the following: sudo /opt/gitlab/embedded/bin/gitaly-hooks check /var/opt/gitlab/gitaly/config.toml ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration @@ -1895,12 +1879,11 @@ for more information. sudo gitlab-rake gitlab:db:configure ``` - NOTE: **Note:** - If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to - PostgreSQL it may be that your PgBouncer node's IP address is missing from - PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) - in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -1990,28 +1973,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -2037,16 +2036,13 @@ work. ## Configure Advanced Search **(STARTER ONLY)** -NOTE: **Note:** -Elasticsearch cluster design and requirements are dependent on your specific data. -For recommended best practices on how to set up your Elasticsearch cluster -alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). - -You can leverage Elasticsearch and enable Advanced Search for faster, more -advanced code search across your entire GitLab instance. +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. -[Learn how to set it up.](../../integration/elasticsearch.md) +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/administration/reference_architectures/5k_users.md b/doc/administration/reference_architectures/5k_users.md index 1ec5a8b214a..238e9808b55 100644 --- a/doc/administration/reference_architectures/5k_users.md +++ b/doc/administration/reference_architectures/5k_users.md @@ -109,19 +109,18 @@ The following list includes descriptions of each server and its assigned IP: ## Configure the external load balancer -NOTE: **Note:** +In an active/active GitLab configuration, you'll need a load balancer to route +traffic to the application servers. The specifics on which load balancer to use +or its exact configuration is beyond the scope of GitLab documentation. We hope +that if you're managing multi-node systems like GitLab, you already have a load +balancer of choice. Some load balancer examples include HAProxy (open-source), +F5 Big-IP LTM, and Citrix Net Scaler. This documentation outline the ports and +protocols needed for use with GitLab. + This architecture has been tested and validated with [HAProxy](https://www.haproxy.org/) as the load balancer. Although other load balancers with similar feature sets could also be used, those load balancers have not been validated. -In an active/active GitLab configuration, you will need a load balancer to route -traffic to the application servers. The specifics on which load balancer to use -or the exact configuration is beyond the scope of GitLab documentation. We hope -that if you're managing multi-node systems like GitLab you have a load balancer of -choice already. Some examples including HAProxy (open-source), F5 Big-IP LTM, -and Citrix Net Scaler. This documentation will outline what ports and protocols -you need to use with GitLab. - The next question is how you will handle SSL in your environment. There are several different options: @@ -279,10 +278,9 @@ The requirements for a Redis setup are the following: ([Internet](https://gitlab.com/gitlab-org/gitlab-foss/uploads/c4cc8cd353604bd80315f9384035ff9e/The_Internet_IT_Crowd.png)), using a firewall. -NOTE: **Note:** -Redis nodes (both primary and replica) will need the same password defined in -`redis['password']`. At any time during a failover the Sentinels can -reconfigure a node and change its status from primary to replica and vice versa. +Both the primary and replica Redis nodes need the same password defined in +`redis['password']`. At any time during a failover, the Sentinels can reconfigure +a node and change its status from primary to replica (and vice versa). #### Configuring the primary Redis instance @@ -336,10 +334,9 @@ reconfigure a node and change its status from primary to replica and vice versa. 1. [Reconfigure Omnibus GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). You can list the current Redis Primary, Replica status via: @@ -424,10 +421,9 @@ run: redis-exporter: (pid 30075) 76861s; run: log: (pid 29674) 76896s 1. Go through the steps again for all the other replica nodes, and make sure to set up the IPs correctly. -NOTE: **Note:** -You can specify multiple roles like sentinel and Redis as: -`roles ['redis_sentinel_role', 'redis_master_role']`. -Read more about [roles](https://docs.gitlab.com/omnibus/roles/). +You can specify multiple roles, like sentinel and Redis, as: +`roles ['redis_sentinel_role', 'redis_master_role']`. Read more about +[roles](https://docs.gitlab.com/omnibus/roles/). These values don't have to be changed again in `/etc/gitlab/gitlab.rb` after a failover, as the nodes will be managed by the [Sentinels](#configure-consul-and-sentinel), and even after a @@ -445,13 +441,6 @@ are supported and can be added if needed. ## Configure Consul and Sentinel -NOTE: **Note:** -If you are using an external Redis Sentinel instance, be sure -to exclude the `requirepass` parameter from the Sentinel -configuration. This parameter will cause clients to report `NOAUTH -Authentication required.`. [Redis Sentinel 3.2.x does not support -password authentication](https://github.com/antirez/redis/issues/3279). - Now that the Redis servers are all set up, let's configure the Sentinel servers. The following IPs will be used as an example: @@ -459,6 +448,12 @@ servers. The following IPs will be used as an example: - `10.6.0.12`: Consul/Sentinel 2 - `10.6.0.13`: Consul/Sentinel 3 +NOTE: **Note:** +If you're using an external Redis Sentinel instance, be sure to exclude the +`requirepass` parameter from the Sentinel configuration. This parameter causes +clients to report `NOAUTH Authentication required.`. +[Redis Sentinel 3.2.x doesn't support password authentication](https://github.com/antirez/redis/issues/3279). + To configure the Sentinel: 1. SSH into the server that will host Consul/Sentinel. @@ -558,10 +553,9 @@ To configure the Sentinel: 1. Go through the steps again for all the other Consul/Sentinel nodes, and make sure you set up the correct IPs. -NOTE: **Note:** -A Consul leader will be elected when the provisioning of the third Consul server is completed. -Viewing the Consul logs `sudo gitlab-ctl tail consul` will display -`...[INFO] consul: New leader elected: ...` +A Consul leader is _elected_ when the provisioning of the third Consul server is +complete. Viewing the Consul logs `sudo gitlab-ctl tail consul` displays +`...[INFO] consul: New leader elected: ...`. You can list the current Consul members (server, client): @@ -1257,11 +1251,10 @@ Name. If you are addressing the Gitaly server by its IP address, you must add it as a Subject Alternative Name to the certificate. [gRPC does not support using an IP address as Common Name in a certificate](https://github.com/grpc/grpc/issues/2691). -NOTE: **Note:** -It is possible to configure Gitaly servers with both an -unencrypted listening address `listen_addr` and an encrypted listening -address `tls_listen_addr` at the same time. This allows you to do a -gradual transition from unencrypted to encrypted traffic, if necessary. +It's possible to configure Gitaly servers with both an unencrypted listening +address (`listen_addr`) and an encrypted listening address (`tls_listen_addr`) +at the same time. This allows you to do a gradual transition from unencrypted to +encrypted traffic, if necessary. To configure Gitaly with TLS: @@ -1429,14 +1422,14 @@ You can also run [multiple Sidekiq processes](../operations/extra_sidekiq_proces ## Configure GitLab Rails -NOTE: **Note:** -In our architectures we run each GitLab Rails node using the Puma webserver -and have its number of workers set to 90% of available CPUs along with four threads. For -nodes that are running Rails with other components the worker value should be reduced -accordingly where we've found 50% achieves a good balance but this is dependent -on workload. - This section describes how to configure the GitLab application (Rails) component. + +In our architecture, we run each GitLab Rails node using the Puma webserver, and +have its number of workers set to 90% of available CPUs, with four threads. For +nodes running Rails with other components, the worker value should be reduced +accordingly. We've determined that a worker value of 50% achieves a good balance, +but this is dependent on workload. + On each node perform the following: 1. If you're [using NFS](#configure-nfs-optional): @@ -1464,10 +1457,10 @@ On each node perform the following: mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data ``` -1. Download/install Omnibus GitLab using **steps 1 and 2** from +1. Download and install Omnibus GitLab using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/install/). Do not complete other steps on the download page. -1. Create/edit `/etc/gitlab/gitlab.rb` and use the following configuration. +1. Create or edit `/etc/gitlab/gitlab.rb` and use the following configuration. To maintain uniformity of links across nodes, the `external_url` on the application server should point to the external URL that users will use to access GitLab. This would be the URL of the [external load balancer](#configure-the-external-load-balancer) @@ -1603,12 +1596,10 @@ On each node perform the following: run: puma: (pid 4936) 8645s; run: log: (pid 29656) 79161s ``` -NOTE: **Note:** -When you specify `https` in the `external_url`, as in the example -above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If -certificates are not present, NGINX will fail to start. See the -[NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) -for more information. +When you specify `https` in the `external_url`, as in the previous example, +GitLab expects that the SSL certificates are in `/etc/gitlab/ssl/`. If the +certificates aren't present, NGINX will fail to start. For more information, see +the [NGINX documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). ### GitLab Rails post-configuration @@ -1618,12 +1609,11 @@ for more information. gitlab-rake gitlab:db:configure ``` - NOTE: **Note:** - If you encounter a `rake aborted!` error stating that PgBouncer is failing to connect to - PostgreSQL it may be that your PgBouncer node's IP address is missing from - PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` on your database nodes. See - [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server) - in the Troubleshooting section before proceeding. + If you encounter a `rake aborted!` error message stating that PgBouncer is + failing to connect to PostgreSQL, it may be that your PgBouncer node's IP + address is missing from PostgreSQL's `trust_auth_cidr_addresses` in `gitlab.rb` + on your database nodes. Before proceeding, see + [PgBouncer error `ERROR: pgbouncer cannot connect to server`](troubleshooting.md#pgbouncer-error-error-pgbouncer-cannot-connect-to-server). 1. [Configure fast lookup of authorized SSH keys in the database](../operations/fast_ssh_key_lookup.md). @@ -1717,28 +1707,44 @@ It's recommended over [NFS](#configure-nfs-optional) and in general it's better in larger setups as object storage is typically much more performant, reliable, and scalable. -Object storage options that GitLab has tested, or is aware of customers using include: +GitLab has been tested on a number of object storage providers: -- SaaS/Cloud solutions such as [Amazon S3](https://aws.amazon.com/s3/), [Google cloud storage](https://cloud.google.com/storage). +- [Amazon S3](https://aws.amazon.com/s3/) +- [Google Cloud Storage](https://cloud.google.com/storage) +- [Digital Ocean Spaces](https://www.digitalocean.com/products/spaces/) +- [Oracle Cloud Infrastructure](https://docs.cloud.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm) +- [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) +- [Azure Blob storage](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction) - On-premises hardware and appliances from various storage vendors. -- MinIO. There is [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. - -For configuring GitLab to use Object Storage refer to the following guides -based on what features you intend to use: - -1. Configure [object storage for backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage). -1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [incremental logging](../job_logs.md#new-incremental-logging-architecture). -1. Configure [object storage for LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage). -1. Configure [object storage for uploads](../uploads.md#using-object-storage). -1. Configure [object storage for merge request diffs](../merge_request_diffs.md#using-object-storage). -1. Configure [object storage for Container Registry](../packages/container_registry.md#use-object-storage) (optional feature). -1. Configure [object storage for Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage) (optional feature). -1. Configure [object storage for packages](../packages/index.md#using-object-storage) (optional feature). -1. Configure [object storage for Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature). **(PREMIUM ONLY)** -1. Configure [object storage for Pseudonymizer](../pseudonymizer.md#configuration) (optional feature). **(ULTIMATE ONLY)** -1. Configure [object storage for autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional - for improved performance). -1. Configure [object storage for Terraform state files](../terraform_state.md#using-object-storage). +- MinIO. We have [a guide to deploying this](https://docs.gitlab.com/charts/advanced/external-object-storage/minio.html) within our Helm Chart documentation. + +There are two ways of specifying object storage configuration in GitLab: + +- [Consolidated form](../object_storage.md#consolidated-object-storage-configuration): A single credential is + shared by all supported object types. +- [Storage-specific form](../object_storage.md#storage-specific-configuration): Every object defines its + own object storage [connection and configuration](../object_storage.md#connection-settings). + +Starting with GitLab 13.2, consolidated object storage configuration is available. It simplifies your GitLab configuration since the connection details are shared across object types. Refer to [Consolidated object storage configuration](../object_storage.md#consolidated-object-storage-configuration) guide for instructions on how to set it up. + +For configuring object storage in GitLab 13.1 and earlier, or for storage types not +supported by consolidated configuration form, refer to the following guides based +on what features you intend to use: + +|Object storage type|Supported by consolidated configuration?| +|-------------------|----------------------------------------| +| [Backups](../../raketasks/backup_restore.md#uploading-backups-to-a-remote-cloud-storage)|No| +| [Job artifacts](../job_artifacts.md#using-object-storage) including archived job logs | Yes | +| [LFS objects](../lfs/index.md#storing-lfs-objects-in-remote-object-storage) | Yes | +| [Uploads](../uploads.md#using-object-storage) | Yes | +| [Container Registry](../packages/container_registry.md#use-object-storage) (optional feature) | No | +| [Merge request diffs](../merge_request_diffs.md#using-object-storage) | Yes | +| [Mattermost](https://docs.mattermost.com/administration/config-settings.html#file-storage)| No | +| [Packages](../packages/index.md#using-object-storage) (optional feature) | Yes | +| [Dependency Proxy](../packages/dependency_proxy.md#using-object-storage) (optional feature) **(PREMIUM ONLY)** | Yes | +| [Pseudonymizer](../pseudonymizer.md#configuration) (optional feature) **(ULTIMATE ONLY)** | No | +| [Autoscale runner caching](https://docs.gitlab.com/runner/configuration/autoscale.html#distributed-runners-caching) (optional for improved performance) | No | +| [Terraform state files](../terraform_state.md#using-object-storage) | Yes | Using separate buckets for each data type is the recommended approach for GitLab. @@ -1764,16 +1770,13 @@ work. ## Configure Advanced Search **(STARTER ONLY)** -NOTE: **Note:** -Elasticsearch cluster design and requirements are dependent on your specific data. -For recommended best practices on how to set up your Elasticsearch cluster -alongside your instance, read how to -[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). +You can leverage Elasticsearch and [enable Advanced Search](../../integration/elasticsearch.md) +for faster, more advanced code search across your entire GitLab instance. -You can leverage Elasticsearch and enable Advanced Search for faster, more -advanced code search across your entire GitLab instance. - -[Learn how to set it up.](../../integration/elasticsearch.md) +Elasticsearch cluster design and requirements are dependent on your specific +data. For recommended best practices about how to set up your Elasticsearch +cluster alongside your instance, read how to +[choose the optimal cluster configuration](../../integration/elasticsearch.md#guidance-on-choosing-optimal-cluster-configuration). <div align="right"> <a type="button" class="btn btn-default" href="#setup-components"> diff --git a/doc/api/graphql/reference/gitlab_schema.graphql b/doc/api/graphql/reference/gitlab_schema.graphql index aedde4928bb..fc0be078ca7 100644 --- a/doc/api/graphql/reference/gitlab_schema.graphql +++ b/doc/api/graphql/reference/gitlab_schema.graphql @@ -3423,6 +3423,111 @@ type CreateImageDiffNotePayload { } """ +Autogenerated input type of CreateIssue +""" +input CreateIssueInput { + """ + The array of user IDs to assign to the issue + """ + assigneeIds: [UserID!] + + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Indicates the issue is confidential + """ + confidential: Boolean + + """ + Timestamp when the issue was created. Available only for admins and project owners + """ + createdAt: Time + + """ + Description of the issue + """ + description: String + + """ + The ID of a discussion to resolve. Also pass `merge_request_to_resolve_discussions_of` + """ + discussionToResolve: String + + """ + Due date of the issue + """ + dueDate: ISO8601Date + + """ + The ID of an epic to associate the issue with + """ + epicId: EpicID + + """ + The IID (internal ID) of a project issue. Only admins and project owners can modify + """ + iid: Int + + """ + The IDs of labels to be added to the issue + """ + labelIds: [LabelID!] + + """ + Labels of the issue + """ + labels: [String!] + + """ + Indicates discussion is locked on the issue + """ + locked: Boolean + + """ + The IID of a merge request for which to resolve discussions + """ + mergeRequestToResolveDiscussionsOf: MergeRequestID + + """ + The ID of the milestone to assign to the issue. On update milestone will be removed if set to null + """ + milestoneId: MilestoneID + + """ + Project full path the issue is associated with + """ + projectPath: ID! + + """ + Title of the issue + """ + title: String! +} + +""" +Autogenerated return type of CreateIssue +""" +type CreateIssuePayload { + """ + A unique identifier for the client performing the mutation. + """ + clientMutationId: String + + """ + Errors encountered during execution of the mutation. + """ + errors: [String!]! + + """ + The issue after mutation + """ + issue: Issue +} + +""" Autogenerated input type of CreateIteration """ input CreateIterationInput { @@ -11189,6 +11294,11 @@ type MergeRequestEdge { } """ +Identifier of MergeRequest +""" +scalar MergeRequestID + +""" Check permissions for the current user on a merge request """ type MergeRequestPermissions { @@ -11964,6 +12074,7 @@ type Mutation { createDiffNote(input: CreateDiffNoteInput!): CreateDiffNotePayload createEpic(input: CreateEpicInput!): CreateEpicPayload createImageDiffNote(input: CreateImageDiffNoteInput!): CreateImageDiffNotePayload + createIssue(input: CreateIssueInput!): CreateIssuePayload createIteration(input: CreateIterationInput!): CreateIterationPayload createNote(input: CreateNoteInput!): CreateNotePayload createRequirement(input: CreateRequirementInput!): CreateRequirementPayload @@ -19452,7 +19563,7 @@ Autogenerated input type of UpdateIssue """ input UpdateIssueInput { """ - The IDs of labels to be added to the issue. + The IDs of labels to be added to the issue """ addLabelIds: [ID!] @@ -19474,7 +19585,7 @@ input UpdateIssueInput { """ Due date of the issue """ - dueDate: Time + dueDate: ISO8601Date """ The ID of the parent epic. NULL when removing the association @@ -19482,11 +19593,6 @@ input UpdateIssueInput { epicId: ID """ - The desired health status - """ - healthStatus: HealthStatus - - """ The IID of the issue to mutate """ iid: String! @@ -19497,7 +19603,7 @@ input UpdateIssueInput { locked: Boolean """ - The ID of the milestone to be assigned, milestone will be removed if set to null. + The ID of the milestone to assign to the issue. On update milestone will be removed if set to null """ milestoneId: ID @@ -19507,12 +19613,12 @@ input UpdateIssueInput { projectPath: ID! """ - The IDs of labels to be removed from the issue. + The IDs of labels to be removed from the issue """ removeLabelIds: [ID!] """ - Close or reopen an issue. + Close or reopen an issue """ stateEvent: IssueStateEvent diff --git a/doc/api/graphql/reference/gitlab_schema.json b/doc/api/graphql/reference/gitlab_schema.json index f44eda3709f..208c26da85e 100644 --- a/doc/api/graphql/reference/gitlab_schema.json +++ b/doc/api/graphql/reference/gitlab_schema.json @@ -9190,6 +9190,276 @@ }, { "kind": "INPUT_OBJECT", + "name": "CreateIssueInput", + "description": "Autogenerated input type of CreateIssue", + "fields": null, + "inputFields": [ + { + "name": "description", + "description": "Description of the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "dueDate", + "description": "Due date of the issue", + "type": { + "kind": "SCALAR", + "name": "ISO8601Date", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "confidential", + "description": "Indicates the issue is confidential", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "locked", + "description": "Indicates discussion is locked on the issue", + "type": { + "kind": "SCALAR", + "name": "Boolean", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "projectPath", + "description": "Project full path the issue is associated with", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "iid", + "description": "The IID (internal ID) of a project issue. Only admins and project owners can modify", + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "title", + "description": "Title of the issue", + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "milestoneId", + "description": "The ID of the milestone to assign to the issue. On update milestone will be removed if set to null", + "type": { + "kind": "SCALAR", + "name": "MilestoneID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "labels", + "description": "Labels of the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "labelIds", + "description": "The IDs of labels to be added to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "LabelID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "createdAt", + "description": "Timestamp when the issue was created. Available only for admins and project owners", + "type": { + "kind": "SCALAR", + "name": "Time", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "mergeRequestToResolveDiscussionsOf", + "description": "The IID of a merge request for which to resolve discussions", + "type": { + "kind": "SCALAR", + "name": "MergeRequestID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "discussionToResolve", + "description": "The ID of a discussion to resolve. Also pass `merge_request_to_resolve_discussions_of`", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "assigneeIds", + "description": "The array of user IDs to assign to the issue", + "type": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "UserID", + "ofType": null + } + } + }, + "defaultValue": null + }, + { + "name": "epicId", + "description": "The ID of an epic to associate the issue with", + "type": { + "kind": "SCALAR", + "name": "EpicID", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + } + ], + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "OBJECT", + "name": "CreateIssuePayload", + "description": "Autogenerated return type of CreateIssue", + "fields": [ + { + "name": "clientMutationId", + "description": "A unique identifier for the client performing the mutation.", + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "errors", + "description": "Errors encountered during execution of the mutation.", + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "LIST", + "name": null, + "ofType": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + } + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "issue", + "description": "The issue after mutation", + "args": [ + + ], + "type": { + "kind": "OBJECT", + "name": "Issue", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + + ], + "enumValues": null, + "possibleTypes": null + }, + { + "kind": "INPUT_OBJECT", "name": "CreateIterationInput", "description": "Autogenerated input type of CreateIteration", "fields": null, @@ -30674,6 +30944,16 @@ "possibleTypes": null }, { + "kind": "SCALAR", + "name": "MergeRequestID", + "description": "Identifier of MergeRequest", + "fields": null, + "inputFields": null, + "interfaces": null, + "enumValues": null, + "possibleTypes": null + }, + { "kind": "OBJECT", "name": "MergeRequestPermissions", "description": "Check permissions for the current user on a merge request", @@ -33421,6 +33701,33 @@ "deprecationReason": null }, { + "name": "createIssue", + "description": null, + "args": [ + { + "name": "input", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "INPUT_OBJECT", + "name": "CreateIssueInput", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "CreateIssuePayload", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { "name": "createIteration", "description": null, "args": [ @@ -56496,16 +56803,6 @@ "defaultValue": null }, { - "name": "title", - "description": "Title of the issue", - "type": { - "kind": "SCALAR", - "name": "String", - "ofType": null - }, - "defaultValue": null - }, - { "name": "description", "description": "Description of the issue", "type": { @@ -56520,7 +56817,7 @@ "description": "Due date of the issue", "type": { "kind": "SCALAR", - "name": "Time", + "name": "ISO8601Date", "ofType": null }, "defaultValue": null @@ -56546,8 +56843,28 @@ "defaultValue": null }, { + "name": "title", + "description": "Title of the issue", + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "milestoneId", + "description": "The ID of the milestone to assign to the issue. On update milestone will be removed if set to null", + "type": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + }, + "defaultValue": null + }, + { "name": "addLabelIds", - "description": "The IDs of labels to be added to the issue.", + "description": "The IDs of labels to be added to the issue", "type": { "kind": "LIST", "name": null, @@ -56565,7 +56882,7 @@ }, { "name": "removeLabelIds", - "description": "The IDs of labels to be removed from the issue.", + "description": "The IDs of labels to be removed from the issue", "type": { "kind": "LIST", "name": null, @@ -56582,18 +56899,8 @@ "defaultValue": null }, { - "name": "milestoneId", - "description": "The ID of the milestone to be assigned, milestone will be removed if set to null.", - "type": { - "kind": "SCALAR", - "name": "ID", - "ofType": null - }, - "defaultValue": null - }, - { "name": "stateEvent", - "description": "Close or reopen an issue.", + "description": "Close or reopen an issue", "type": { "kind": "ENUM", "name": "IssueStateEvent", @@ -56602,16 +56909,6 @@ "defaultValue": null }, { - "name": "healthStatus", - "description": "The desired health status", - "type": { - "kind": "ENUM", - "name": "HealthStatus", - "ofType": null - }, - "defaultValue": null - }, - { "name": "epicId", "description": "The ID of the parent epic. NULL when removing the association", "type": { diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 62ccf4a633b..864c1f0e2d3 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -537,6 +537,16 @@ Autogenerated return type of CreateImageDiffNote. | `errors` | String! => Array | Errors encountered during execution of the mutation. | | `note` | Note | The note after mutation | +### CreateIssuePayload + +Autogenerated return type of CreateIssue. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Errors encountered during execution of the mutation. | +| `issue` | Issue | The issue after mutation | + ### CreateIterationPayload Autogenerated return type of CreateIteration. diff --git a/doc/development/documentation/site_architecture/global_nav.md b/doc/development/documentation/site_architecture/global_nav.md index 22d97a9e2cf..9fce9b4e4b3 100644 --- a/doc/development/documentation/site_architecture/global_nav.md +++ b/doc/development/documentation/site_architecture/global_nav.md @@ -324,7 +324,6 @@ There are three main considerations on the logic built for the nav: - `https://docs.gitlab.com/ee/` - `https://docs.gitlab.com/omnibus/` - `https://docs.gitlab.com/runner/` - - `https://docs.gitlab.com/debug/` - `https://docs.gitlab.com/*` - [EE-only](#ee-only-docs): documentation only available in `/ee/`, not on `/ce/`, e.g.: - `https://docs.gitlab.com/ee/user/group/epics/` @@ -342,8 +341,8 @@ all the nav links to other pages: <% dir = @item.identifier.to_s[%r{(?<=/)[^/]+}] %> ``` -For instance, for `https://docs.gitlab.com/ce/user/index.html`, -`dir` == `ce`, and for `https://docs.gitlab.com/omnibus/README.html`, +For instance, for `https://docs.gitlab.com/ee/user/index.html`, +`dir` == `ee`, and for `https://docs.gitlab.com/omnibus/README.html`, `dir` == `omnibus`. #### Default URL diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 20ed0cbc39e..6075124ef40 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -1892,7 +1892,7 @@ for the changes to take effect. If the document you are editing resides in a place other than the GitLab CE/EE `doc/` directory, instead of the relative link, use the full path: -`https://docs.gitlab.com/ce/administration/restart_gitlab.html`. Replace +`https://docs.gitlab.com/ee/administration/restart_gitlab.html`. Replace `reconfigure` with `restart` where appropriate. ### Installation guide diff --git a/doc/development/feature_flags/controls.md b/doc/development/feature_flags/controls.md index b759fb58fb8..ef38a85bec0 100644 --- a/doc/development/feature_flags/controls.md +++ b/doc/development/feature_flags/controls.md @@ -248,13 +248,26 @@ Changes to the issue format can be submitted in the ## Cleaning up -Once the change is deemed stable, submit a new merge request to remove the -feature flag. This ensures the change is available to all users and self-managed -instances. Make sure to add the ~"feature flag" label to this merge request so -release managers are aware the changes are hidden behind a feature flag. If the -merge request has to be picked into a stable branch, make sure to also add the -appropriate `~"Pick into X.Y"` label (e.g. `~"Pick into 13.0"`). -See [the process document](process.md#including-a-feature-behind-feature-flag-in-the-final-release) for further details. +A feature flag should be removed as soon as it is no longer needed. Each additional +feature flag in the codebase increases the complexity of the application +and reduces confidence in our testing suite covering all possible combinations. +Additionally, a feature flag overwritten in some of the environments can result +in undefined and untested system behavior. + +To remove a feature flag: + +1. Open a new merge request with the ~"feature flag" label so + release managers are aware the changes are hidden behind a feature flag. +1. If the merge request has to be picked into a stable branch, add the + appropriate `~"Pick into X.Y"` label, for example `~"Pick into 13.0"`. + See [the feature flag process](process.md#including-a-feature-behind-feature-flag-in-the-final-release) + for further details. +1. Remove all references to the feature flag from the codebase. +1. Remove the YAML definition for the feature from the repository. +1. Clean up the feature flag from all environments with `/chatops run feature delete some_feature`. +1. Close the rollout issue for the feature flag after the feature flag is removed from the codebase. + +### Cleanup ChatOps When a feature gate has been removed from the code base, the feature record still exists in the database that the flag was deployed too. diff --git a/doc/development/feature_flags/development.md b/doc/development/feature_flags/development.md index 4688c74366a..067e480f6f8 100644 --- a/doc/development/feature_flags/development.md +++ b/doc/development/feature_flags/development.md @@ -153,6 +153,11 @@ default_enabled: false TIP: **Tip:** To create a feature flag that is only used in EE, add the `--ee` flag: `bin/feature-flag --ee` +## Delete a feature flag + +See [cleaning up feature flags](controls.md#cleaning-up) for more information about +deleting feature flags. + ## Develop with a feature flag There are two main ways of using Feature Flags in the GitLab codebase: diff --git a/doc/development/product_analytics/usage_ping.md b/doc/development/product_analytics/usage_ping.md index 2dae76bba80..2feeb129fe2 100644 --- a/doc/development/product_analytics/usage_ping.md +++ b/doc/development/product_analytics/usage_ping.md @@ -619,7 +619,7 @@ Ensure you comply with the [Changelog entries guide](../changelog.md). ### 8. Ask for a Product Analytics Review -On GitLab.com, we have DangerBot setup to monitor Product Analytics related files and DangerBot will recommend a Product Analytics review. Mention `@gitlab-org/growth/product-analytics/engineers` in your MR for a review. +On GitLab.com, we have DangerBot setup to monitor Product Analytics related files and DangerBot will recommend a Product Analytics review. Mention `@gitlab-org/growth/product_analytics/engineers` in your MR for a review. ### 9. Verify your metric diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 0a62feafdc5..066a38d68de 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -939,12 +939,10 @@ installed version of GitLab, the restore command aborts with an error message. Install the [correct GitLab version](https://packages.gitlab.com/gitlab/), and then try again. -There is a [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3470) -for restore not working with `pgbouncer`. To work around the issue, the Rails -node must bypass `pgbouncer` and connect directly to the primary -database node. You can do this by setting `gitlab_rails['db_host']` and -`gitlab_rails['port']` to connect to the primary database node and -[reconfiguring GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure). +NOTE: **Note:** +There is a known issue with restore not working with `pgbouncer`. The [workaround is to bypass +`pgbouncer` and connect directly to the primary database node](../administration/postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer). +[Read more about backup and restore with `pgbouncer`](#backup-and-restore-for-installations-using-pgbouncer). ### Restore for Docker image and GitLab Helm chart installations @@ -1039,6 +1037,29 @@ VM snapshots of the entire GitLab server. It's not uncommon however for a VM snapshot to require you to power down the server, which limits this solution's practical use. +## Backup and restore for installations using PgBouncer + +PgBouncer can cause the following errors when performing backups and restores: + +```ruby +ActiveRecord::StatementInvalid: PG::UndefinedTable +``` + +There is a [known issue](https://gitlab.com/gitlab-org/omnibus-gitlab/-/issues/3470) for restore not working +with `pgbouncer`. + +To workaround this issue, the GitLab server will need to bypass `pgbouncer` and +[connect directly to the primary database node](../administration/postgresql/pgbouncer.md#procedure-for-bypassing-pgbouncer) +to perform the database restore. + +There is also a [known issue](https://gitlab.com/gitlab-org/gitlab/-/issues/23211) +with PostgreSQL 9 and running a database backup through PgBouncer that can cause +an outage to GitLab. If you're still on PostgreSQL 9 and upgrading PostgreSQL isn't +an option, workarounds include having a dedicated application node just for backups, +configured to connect directly the primary database node as noted above. You're +advised to upgrade your PostgreSQL version though, GitLab 11.11 shipped with PostgreSQL +10.7, and that is the recommended version for GitLab 12+. + ## Additional notes This documentation is for GitLab Community and Enterprise Edition. We back up diff --git a/doc/raketasks/cleanup.md b/doc/raketasks/cleanup.md index c4046b36c55..0b3da39d3d5 100644 --- a/doc/raketasks/cleanup.md +++ b/doc/raketasks/cleanup.md @@ -146,7 +146,7 @@ NOTE: **Note:** These commands will not work for artifacts stored on [object storage](../administration/object_storage.md). -When you notice there are more job artifacts files on disk than there +When you notice there are more job artifacts files and/or directories on disk than there should be, you can run: ```shell @@ -157,7 +157,7 @@ This command: - Scans through the entire artifacts folder. - Checks which files still have a record in the database. -- If no database record is found, the file is deleted from disk. +- If no database record is found, the file and directory is deleted from disk. By default, this task does not delete anything but shows what it can delete. Run the command with `DRY_RUN=false` if you actually want to diff --git a/doc/user/admin_area/index.md b/doc/user/admin_area/index.md index 58430ab615b..0ddbe17580a 100644 --- a/doc/user/admin_area/index.md +++ b/doc/user/admin_area/index.md @@ -118,8 +118,15 @@ To list users matching a specific criteria, click on one of the following tabs o - **[Deactivated](activating_deactivating_users.md)** - **Without projects** -For each user, their username, email address, are listed, also the date their account was -created and the date of last activity. To edit a user, click the **Edit** button in that user's +For each user, the following are listed: + +1. Username +1. Email address +1. Project membership count +1. Date of account creation +1. Date of last activity + +To edit a user, click the **Edit** button in that user's row. To delete the user, or delete the user and their contributions, click the cog dropdown in that user's row, and select the desired option. diff --git a/doc/user/project/requirements/index.md b/doc/user/project/requirements/index.md index 9d7d3914905..9dd5dd3409b 100644 --- a/doc/user/project/requirements/index.md +++ b/doc/user/project/requirements/index.md @@ -40,13 +40,15 @@ list is sorted by creation date in descending order. ## Edit a requirement +> - [Added](https://gitlab.com/gitlab-org/gitlab/-/issues/218607) ability to mark a requirement as Satisfied in [GitLab Ultimate](https://about.gitlab.com/pricing/) 13.5. + You can edit a requirement (if you have the necessary privileges) from the requirements list page. To edit a requirement: 1. From the requirements list, click **Edit** (**{pencil}**). -1. Update the title in text input field. +1. Update the title in text input field. You can also mark (and unmark) a requirement as satisfied in the edit form by using the checkbox labeled "Satisfied". 1. Click **Save changes**. ## Archive a requirement @@ -97,7 +99,7 @@ You can also sort the requirements list by: GitLab supports [requirements test reports](../../../ci/pipelines/job_artifacts.md#artifactsreportsrequirements) now. You can add a job to your CI pipeline that, when triggered, marks all existing -requirements as Satisfied. +requirements as Satisfied (you may manually satisfy a requirement in the edit form [edit a requirement](#edit-a-requirement)). ### Add the manual job to CI diff --git a/docker/README.md b/docker/README.md index 61b41d2f109..cb6f99a6482 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,3 +1,3 @@ # GitLab Docker images -This content has been moved to [our documentation site](https://docs.gitlab.com/ce/install/docker.html). +This content has been moved to [our documentation site](https://docs.gitlab.com/ee/install/docker.html). diff --git a/lib/api/ci/pipelines.rb b/lib/api/ci/pipelines.rb index 045f81074a7..66bb676d02c 100644 --- a/lib/api/ci/pipelines.rb +++ b/lib/api/ci/pipelines.rb @@ -128,7 +128,7 @@ module API pipeline = user_project.all_pipelines.find(params[:pipeline_id]) - if Feature.enabled?(:ci_jobs_finder_refactor) + if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true) builds = ::Ci::JobsFinder .new(current_user: current_user, pipeline: pipeline, params: params) .execute @@ -157,7 +157,7 @@ module API pipeline = user_project.all_pipelines.find(params[:pipeline_id]) - if Feature.enabled?(:ci_jobs_finder_refactor) + if Feature.enabled?(:ci_jobs_finder_refactor, default_enabled: true) bridges = ::Ci::JobsFinder .new(current_user: current_user, pipeline: pipeline, params: params, type: ::Ci::Bridge) .execute diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 0e5b0fae6e2..715b09d70db 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -231,9 +231,6 @@ module API authorize! :create_issue, user_project - params.delete(:created_at) unless current_user.can?(:set_issue_created_at, user_project) - params.delete(:iid) unless current_user.can?(:set_issue_iid, user_project) - issue_params = declared_params(include_missing: false) issue_params[:system_note_timestamp] = params[:created_at] @@ -279,8 +276,6 @@ module API issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue - # Setting updated_at is allowed only for admins and owners - params.delete(:updated_at) unless current_user.can?(:set_issue_updated_at, user_project) issue.system_note_timestamp = params[:updated_at] update_params = declared_params(include_missing: false).merge(request: request, api: true) diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 368069844ea..67e58d9ee99 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -1,4 +1,4 @@ -# see https://docs.gitlab.com/ce/ci/yaml/README.html for all available options +# see https://docs.gitlab.com/ee/ci/yaml/README.html for all available options # you can delete this line if you're not using Docker image: busybox:latest diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index c3568c0d2c8..0c5850bdb8e 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -1,7 +1,7 @@ # Based on openjdk:8, already includes lein image: clojure:lein-2.7.0 # If you need to configure a database, add a `services` section here -# See https://docs.gitlab.com/ce/ci/services/postgres.html +# See https://docs.gitlab.com/ee/ci/services/postgres.html # Make sure you configure the connection as well before_script: diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 0c8859dc779..f2f92fe0704 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -1,5 +1,5 @@ # Template project: https://gitlab.com/pages/jekyll -# Docs: https://docs.gitlab.com/ce/pages/ +# Docs: https://docs.gitlab.com/ee/pages/ image: ruby:2.6 variables: diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index 51372871f39..7eb85997605 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -6,9 +6,20 @@ module Gitlab class Metrics extend Gitlab::Utils::StrongMemoize - OPERATIONS = [:appended, :streamed, :chunked, :mutated, :overwrite, - :accepted, :finalized, :discarded, :conflict, :locked, - :invalid].freeze + OPERATIONS = [ + :appended, # new trace data has been written to a chunk + :streamed, # new trace data has been sent by a runner + :chunked, # new trace chunk has been created + :mutated, # trace has been mutated when removing secrets + :overwrite, # runner requested overwritting a build trace + :accepted, # scheduled chunks for migration and responded with 202 + :finalized, # all live build trace chunks have been persisted + :discarded, # failed to persist live chunks before timeout + :conflict, # runner has sent unrecognized build state details + :locked, # build trace has been locked by a different mechanism + :stalled, # failed to migrate chunk due to a worker duplication + :invalid # malformed build trace has been detected using CRC32 + ].freeze def increment_trace_operation(operation: :unknown) unless OPERATIONS.include?(operation) diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index f948f5e3951..dc1f363c08e 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -44,7 +44,6 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend push_frontend_feature_flag(:monaco_blobs, default_enabled: true) - push_frontend_feature_flag(:monaco_ci, default_enabled: true) push_frontend_feature_flag(:webperf_experiment, default_enabled: false) push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, default_enabled: false) diff --git a/lib/gitlab/marginalia.rb b/lib/gitlab/marginalia.rb index 24e21a1d512..c0c9bb18039 100644 --- a/lib/gitlab/marginalia.rb +++ b/lib/gitlab/marginalia.rb @@ -4,8 +4,6 @@ module Gitlab module Marginalia cattr_accessor :enabled, default: false - MARGINALIA_FEATURE_FLAG = :marginalia - def self.set_application_name ::Marginalia.application_name = Gitlab.process_name end @@ -16,15 +14,11 @@ module Gitlab end end - def self.cached_feature_enabled? - enabled - end - - def self.set_feature_cache + def self.set_enabled_from_feature_flag # During db:create and db:bootstrap skip feature query as DB is not available yet. return false unless Gitlab::Database.cached_table_exists?('features') - self.enabled = Feature.enabled?(MARGINALIA_FEATURE_FLAG) + self.enabled = Feature.enabled?(:marginalia) end end end diff --git a/lib/gitlab/marginalia/active_record_instrumentation.rb b/lib/gitlab/marginalia/active_record_instrumentation.rb index 3266b9f8336..452f472bf6a 100644 --- a/lib/gitlab/marginalia/active_record_instrumentation.rb +++ b/lib/gitlab/marginalia/active_record_instrumentation.rb @@ -5,7 +5,7 @@ module Gitlab module Marginalia module ActiveRecordInstrumentation def annotate_sql(sql) - Gitlab::Marginalia.cached_feature_enabled? ? super(sql) : sql + Gitlab::Marginalia.enabled ? super(sql) : sql end end end diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index 2d8df2ca204..714ca77c3e5 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -48,7 +48,7 @@ module Gitlab *Documentation* For more information about GitLab chatops, refer to its - documentation: https://docs.gitlab.com/ce/ci/chatops/README.html. + documentation: https://docs.gitlab.com/ee/ci/chatops/README.html. MESSAGE message diff --git a/lib/gitlab/visibility_level_checker.rb b/lib/gitlab/visibility_level_checker.rb index f15f1486a4e..3ffd86c4f8c 100644 --- a/lib/gitlab/visibility_level_checker.rb +++ b/lib/gitlab/visibility_level_checker.rb @@ -3,7 +3,7 @@ # Gitlab::VisibilityLevelChecker verifies that: # - Current @project.visibility_level is not restricted # - Override visibility param is not restricted -# - @see https://docs.gitlab.com/ce/api/project_import_export.html#import-a-file +# - @see https://docs.gitlab.com/ee/api/project_import_export.html#import-a-file # # @param current_user [User] Current user object to verify visibility level against # @param project [Project] Current project that is being created/imported diff --git a/locale/gitlab.pot b/locale/gitlab.pot index d5c2584ca1b..adc1743d1d5 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -22606,6 +22606,9 @@ msgstr "" msgid "SSL Verification:" msgstr "" +msgid "Satisfied" +msgstr "" + msgid "Saturday" msgstr "" diff --git a/qa/qa/page/project/web_ide/edit.rb b/qa/qa/page/project/web_ide/edit.rb index 56c8d343cf5..fc33c753230 100644 --- a/qa/qa/page/project/web_ide/edit.rb +++ b/qa/qa/page/project/web_ide/edit.rb @@ -73,6 +73,10 @@ module QA element :project_path_content end + view 'app/assets/javascripts/ide/components/commit_sidebar/message_field.vue' do + element :ide_commit_message_field + end + def has_file?(file_name) within_element(:file_list) do page.has_content? file_name @@ -83,6 +87,10 @@ module QA has_element?(:project_path_content, project_path: project_path) end + def go_to_project + click_element(:project_path_content, Page::Project::Show) + end + def create_new_file_from_template(file_name, template) click_element(:new_file, Page::Component::WebIDE::Modal::CreateNewFile) @@ -115,7 +123,7 @@ module QA find_element(:commit_sha_content).text end - def commit_changes(open_merge_request: false) + def commit_changes(commit_message = nil, open_merge_request: false) # Clicking :begin_commit_button switches from the # edit to the commit view click_element(:begin_commit_button) @@ -133,6 +141,10 @@ module QA has_element?(:commit_button) end + if commit_message + fill_element(:ide_commit_message_field, commit_message) + end + if open_merge_request click_element(:commit_button, Page::MergeRequest::New) else diff --git a/qa/qa/resource/file.rb b/qa/qa/resource/file.rb index 76c4c71c48d..f573f3e89f0 100644 --- a/qa/qa/resource/file.rb +++ b/qa/qa/resource/file.rb @@ -27,11 +27,14 @@ module QA Page::Project::Show.perform(&:create_first_new_file!) - Page::File::Form.perform do |form| - form.add_name(@name) - form.add_content(@content) - form.add_commit_message(@commit_message) - form.commit_changes + Page::Project::WebIDE::Edit.perform do |ide| + ide.add_file(@name, @content) + ide.commit_changes(@commit_message) + ide.go_to_project + end + + Page::Project::Show.perform do |project| + project.click_file(@name) end end diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/create_edit_delete_file_via_web_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/create_edit_delete_file_via_web_spec.rb index 43f4b080c73..5aa5f0fc0a3 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/create_edit_delete_file_via_web_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/create_edit_delete_file_via_web_spec.rb @@ -18,7 +18,6 @@ module QA file.commit_message = commit_message_for_create end - expect(page).to have_content('The file has been successfully created.') expect(page).to have_content(file_name) expect(page).to have_content(file_content) expect(page).to have_content(commit_message_for_create) diff --git a/qa/qa/specs/features/browser_ui/3_create/web_ide/create_first_file_in_web_ide_spec.rb b/qa/qa/specs/features/browser_ui/3_create/web_ide/create_first_file_in_web_ide_spec.rb index ea821f8b3e6..f7a2e3081fb 100644 --- a/qa/qa/specs/features/browser_ui/3_create/web_ide/create_first_file_in_web_ide_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/web_ide/create_first_file_in_web_ide_spec.rb @@ -10,7 +10,6 @@ module QA end end - let(:web_ide_url) { current_url + '-/ide/project/' + project.path_with_namespace } let(:file_name) { 'the very first file.txt' } before do @@ -18,10 +17,8 @@ module QA end it "creates the first file in an empty project via Web IDE", testcase: 'https://gitlab.com/gitlab-org/quality/testcases/-/issues/847' do - # In the first iteration, the test opens Web IDE by modifying the URL to address past regressions. - # Once the Web IDE button is introduced for empty projects, the test will be modified to go through UI. - # See https://gitlab.com/gitlab-org/gitlab/-/issues/27915 and https://gitlab.com/gitlab-org/gitlab/-/issues/27535. - page.visit(web_ide_url) + project.visit! + Page::Project::Show.perform(&:create_first_new_file!) Page::Project::WebIDE::Edit.perform do |ide| ide.create_first_file(file_name) diff --git a/rubocop/cop/graphql/gid_expected_type.rb b/rubocop/cop/graphql/gid_expected_type.rb new file mode 100644 index 00000000000..354c5516752 --- /dev/null +++ b/rubocop/cop/graphql/gid_expected_type.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module RuboCop + module Cop + module Graphql + class GIDExpectedType < RuboCop::Cop::Cop + MSG = 'Add an expected_type parameter to #object_from_id calls if possible.' + + def_node_search :id_from_object?, <<~PATTERN + (send ... :object_from_id (...)) + PATTERN + + def on_send(node) + return unless id_from_object?(node) + + add_offense(node) + end + end + end + end +end diff --git a/rubocop/cop/migration/create_table_with_foreign_keys.rb b/rubocop/cop/migration/create_table_with_foreign_keys.rb index 01cab032049..382a2d6f65b 100644 --- a/rubocop/cop/migration/create_table_with_foreign_keys.rb +++ b/rubocop/cop/migration/create_table_with_foreign_keys.rb @@ -9,7 +9,7 @@ module RuboCop include MigrationHelpers MSG = 'Creating a table with more than one foreign key at once violates our migration style guide. ' \ - 'For more details check the https://docs.gitlab.com/ce/development/migration_style_guide.html#examples' + 'For more details check the https://docs.gitlab.com/ee/development/migration_style_guide.html#examples' def_node_matcher :create_table_with_block?, <<~PATTERN (block diff --git a/rubocop/rubocop-migrations.yml b/rubocop/rubocop-migrations.yml index 454bed71833..f8ff6e005f9 100644 --- a/rubocop/rubocop-migrations.yml +++ b/rubocop/rubocop-migrations.yml @@ -1,4 +1,4 @@ -# Make sure to update the docs if this file moves. Docs URL: https://docs.gitlab.com/ce/development/migration_style_guide.html#when-to-use-the-helper-method +# Make sure to update the docs if this file moves. Docs URL: https://docs.gitlab.com/ee/development/migration_style_guide.html#when-to-use-the-helper-method Migration/UpdateLargeTable: Enabled: true HighTrafficTables: &high_traffic_tables # size in GB (>= 10 GB on GitLab.com as of 02/2020) and/or number of records diff --git a/scripts/used-feature-flags b/scripts/used-feature-flags new file mode 100755 index 00000000000..07b8d2063ef --- /dev/null +++ b/scripts/used-feature-flags @@ -0,0 +1,94 @@ +#!/usr/bin/env ruby + +class String + def red + "\e[31m#{self}\e[0m" + end + + def yellow + "\e[33m#{self}\e[0m" + end + + def green + "\e[32m#{self}\e[0m" + end + + def bold + "\e[1m#{self}\e[0m" + end +end + +flags_paths = [ + 'config/feature_flags/**/*.yml' +] + +# For EE additionally process `ee/` feature flags +if File.exist?('ee/app/models/license.rb') && !%w[true 1].include?(ENV['FOSS_ONLY'].to_s) + flags_paths << 'ee/config/feature_flags/**/*.yml' +end + +all_flags = {} +additional_flags = Set.new + +# Iterate all defined feature flags +# to discover which were used +flags_paths.each do |flags_path| + puts flags_path + Dir.glob(flags_path).each do |path| + feature_flag_name = File.basename(path, '.yml') + + all_flags[feature_flag_name] = File.exist?(File.join('tmp', 'feature_flags', feature_flag_name + '.used')) + end +end + +# Iterate all used feature flags +# to discover which flags are undefined +Dir.glob('tmp/feature_flags/*.used').each do |path| + feature_flag_name = File.basename(path, '.used') + + additional_flags.add(feature_flag_name) unless all_flags[feature_flag_name] +end + +used_flags = all_flags.select { |name, used| used } +unused_flags = all_flags.reject { |name, used| used } + +puts "=========================================".green.bold +puts "Feature Flags usage summary:".green.bold +puts + +puts "- #{all_flags.count + additional_flags.count} was found" +puts "- #{unused_flags.count} appear(s) to be UNUSED".yellow +puts "- #{additional_flags.count} appear(s) to be unknown".yellow +puts "- #{used_flags.count} appear(s) to be used".green +puts + +if additional_flags.count > 0 + puts "==================================================".green.bold + puts "There are feature flags that appears to be unknown".yellow + puts + puts "They appear to be used by CI, but we do lack their YAML definition".yellow + puts "This is likely expected, so feel free to ignore that list:".yellow + puts + additional_flags.sort.each do |name| + puts "- #{name}".yellow + end + puts +end + +if unused_flags.count > 0 + puts "========================================".green.bold + puts "These feature flags appears to be UNUSED".red.bold + puts + puts "If they are really no longer needed REMOVE their .yml definition".red + puts "If they are needed you need to ENSURE that their usage is covered with specs to continue.".red + puts + unused_flags.keys.sort.each do |name| + puts "- #{name}".yellow + end + puts + puts "Feature flag usage check failed.".red.bold + exit(1) +end + +puts "Everything is fine here!".green +puts diff --git a/spec/controllers/concerns/redis_tracking_spec.rb b/spec/controllers/concerns/redis_tracking_spec.rb index cb47e742057..90d34e5692d 100644 --- a/spec/controllers/concerns/redis_tracking_spec.rb +++ b/spec/controllers/concerns/redis_tracking_spec.rb @@ -6,6 +6,10 @@ RSpec.describe RedisTracking do let(:feature) { 'approval_rule' } let(:user) { create(:user) } + before do + skip_feature_flags_yaml_validation + end + controller(ApplicationController) do include RedisTracking diff --git a/spec/controllers/dashboard/labels_controller_spec.rb b/spec/controllers/dashboard/labels_controller_spec.rb index 415cb821545..e7091664d1a 100644 --- a/spec/controllers/dashboard/labels_controller_spec.rb +++ b/spec/controllers/dashboard/labels_controller_spec.rb @@ -3,27 +3,32 @@ require 'spec_helper' RSpec.describe Dashboard::LabelsController do - let(:project) { create(:project) } - let(:user) { create(:user) } - let!(:label) { create(:label, project: project) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:project) } + let_it_be(:project_2) { create(:project) } + + let_it_be(:label) { create(:label, project: project, title: 'some_label') } + let_it_be(:label_with_same_title) { create(:label, project: project_2, title: 'some_label') } + let_it_be(:unrelated_label) { create(:label, project: create(:project, :public)) } + + before_all do + project.add_reporter(user) + project_2.add_reporter(user) + end before do sign_in(user) - project.add_reporter(user) end describe "#index" do - let!(:unrelated_label) { create(:label, project: create(:project, :public)) } - subject { get :index, format: :json } - it 'returns global labels for projects the user has a relationship with' do + it 'returns labels with unique titles for projects the user has a relationship with' do subject expect(json_response).to be_kind_of(Array) expect(json_response.size).to eq(1) - expect(json_response[0]["id"]).to be_nil - expect(json_response[0]["title"]).to eq(label.title) + expect(json_response[0]['title']).to eq(label.title) end it_behaves_like 'disabled when using an external authorization service' diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 993ebea2541..c3da2c9d116 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -11,7 +11,7 @@ RSpec.describe 'Database schema' do let(:columns_name_with_jsonb) { retrieve_columns_name_with_jsonb } # List of columns historically missing a FK, don't add more columns - # See: https://docs.gitlab.com/ce/development/foreign_keys.html#naming-foreign-keys + # See: https://docs.gitlab.com/ee/development/foreign_keys.html#naming-foreign-keys IGNORED_FK_COLUMNS = { abuse_reports: %w[reporter_id user_id], application_settings: %w[performance_bar_allowed_group_id slack_app_id snowplow_app_id eks_account_id eks_access_key_id], diff --git a/spec/features/issues/issue_sidebar_spec.rb b/spec/features/issues/issue_sidebar_spec.rb index 38d11ee2560..da44fe398fb 100644 --- a/spec/features/issues/issue_sidebar_spec.rb +++ b/spec/features/issues/issue_sidebar_spec.rb @@ -140,6 +140,18 @@ RSpec.describe 'Issue Sidebar' do end end end + + it 'shows label text as "Apply" when assignees are changed' do + project.add_developer(user) + visit_issue(project, issue2) + + find('.block.assignee .edit-link').click + wait_for_requests + + click_on 'Unassigned' + + expect(page).to have_link('Apply') + end end context 'as a allowed user' do diff --git a/spec/features/merge_request/user_suggests_changes_on_diff_spec.rb b/spec/features/merge_request/user_suggests_changes_on_diff_spec.rb index abdbd3a6f8a..9268190c7e0 100644 --- a/spec/features/merge_request/user_suggests_changes_on_diff_spec.rb +++ b/spec/features/merge_request/user_suggests_changes_on_diff_spec.rb @@ -250,7 +250,7 @@ RSpec.describe 'User comments on a diff', :js do end context 'multiple suggestions in a single note' do - it 'suggestions are presented' do + it 'suggestions are presented', quarantine: 'https://gitlab.com/gitlab-org/gitlab/-/issues/258989' do click_diff_line(find("[id='#{sample_compare.changes[1][:line_code]}']")) page.within('.js-discussion-note-form') do diff --git a/spec/features/projects/blobs/user_creates_new_blob_in_new_project_spec.rb b/spec/features/projects/blobs/user_creates_new_blob_in_new_project_spec.rb index a271a4f43a8..fda2992af8d 100644 --- a/spec/features/projects/blobs/user_creates_new_blob_in_new_project_spec.rb +++ b/spec/features/projects/blobs/user_creates_new_blob_in_new_project_spec.rb @@ -2,7 +2,9 @@ require 'spec_helper' -RSpec.describe 'User creates blob in new project', :js do +RSpec.describe 'User creates new blob', :js do + include WebIdeSpecHelpers + let(:user) { create(:user) } let(:project) { create(:project, :empty_repo) } @@ -12,16 +14,19 @@ RSpec.describe 'User creates blob in new project', :js do visit project_path(project) end - it 'allows the user to add a new file' do + it 'allows the user to add a new file in Web IDE' do click_link 'New file' - execute_script("monaco.editor.getModels()[0].setValue('Hello world')") + wait_for_requests + + ide_create_new_file('dummy-file', content: "Hello world\n") - fill_in(:file_name, with: 'dummy-file') + ide_commit - click_button('Commit changes') + click_button('Commit') - expect(page).to have_content('The file has been successfully created') + expect(page).to have_content('All changes are committed') + expect(project.repository.blob_at('master', 'dummy-file').data).to eql("Hello world\n") end end diff --git a/spec/features/projects/ci/lint_spec.rb b/spec/features/projects/ci/lint_spec.rb index ce435151b84..eb2efb4357d 100644 --- a/spec/features/projects/ci/lint_spec.rb +++ b/spec/features/projects/ci/lint_spec.rb @@ -8,117 +8,88 @@ RSpec.describe 'CI Lint', :js do let(:project) { create(:project, :repository) } let(:user) { create(:user) } - shared_examples 'correct ci linting process' do - describe 'YAML parsing' do - shared_examples 'validates the YAML' do - before do - stub_feature_flags(ci_lint_vue: false) - click_on 'Validate' - end + let(:content_selector) { '.content .view-lines' } - context 'YAML is correct' do - let(:yaml_content) do - File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')) - end + before do + stub_feature_flags(ci_lint_vue: false) + project.add_developer(user) + sign_in(user) - it 'parses Yaml and displays the jobs' do - expect(page).to have_content('Status: syntax is correct') + visit project_ci_lint_path(project) + editor_set_value(yaml_content) - within "table" do - aggregate_failures do - expect(page).to have_content('Job - rspec') - expect(page).to have_content('Job - spinach') - expect(page).to have_content('Deploy Job - staging') - expect(page).to have_content('Deploy Job - production') - end - end - end - end + wait_for('YAML content') do + find(content_selector).text.present? + end + end - context 'YAML is incorrect' do - let(:yaml_content) { 'value: cannot have :' } + describe 'YAML parsing' do + shared_examples 'validates the YAML' do + before do + stub_feature_flags(ci_lint_vue: false) + click_on 'Validate' + end - it 'displays information about an error' do - expect(page).to have_content('Status: syntax is incorrect') - expect(page).to have_selector(content_selector, text: yaml_content) - end + context 'YAML is correct' do + let(:yaml_content) do + File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')) end - end - it_behaves_like 'validates the YAML' + it 'parses Yaml and displays the jobs' do + expect(page).to have_content('Status: syntax is correct') - context 'when Dry Run is checked' do - before do - check 'Simulate a pipeline created for the default branch' + within "table" do + aggregate_failures do + expect(page).to have_content('Job - rspec') + expect(page).to have_content('Job - spinach') + expect(page).to have_content('Deploy Job - staging') + expect(page).to have_content('Deploy Job - production') + end + end end - - it_behaves_like 'validates the YAML' end - describe 'YAML revalidate' do - let(:yaml_content) { 'my yaml content' } + context 'YAML is incorrect' do + let(:yaml_content) { 'value: cannot have :' } - it 'loads previous YAML content after validation' do - expect(page).to have_field('content', with: 'my yaml content', visible: false, type: 'textarea') + it 'displays information about an error' do + expect(page).to have_content('Status: syntax is incorrect') + expect(page).to have_selector(content_selector, text: yaml_content) end end end - describe 'YAML clearing' do + it_behaves_like 'validates the YAML' + + context 'when Dry Run is checked' do before do - click_on 'Clear' + check 'Simulate a pipeline created for the default branch' end - context 'YAML is present' do - let(:yaml_content) do - File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')) - end - - it 'YAML content is cleared' do - expect(page).to have_field('content', with: '', visible: false, type: 'textarea') - end - end + it_behaves_like 'validates the YAML' end - end - context 'with ACE editor' do - it_behaves_like 'correct ci linting process' do - let(:content_selector) { '.ace_content' } + describe 'YAML revalidate' do + let(:yaml_content) { 'my yaml content' } - before do - stub_feature_flags(monaco_ci: false) - stub_feature_flags(ci_lint_vue: false) - project.add_developer(user) - sign_in(user) - - visit project_ci_lint_path(project) - find('#ci-editor') - execute_script("ace.edit('ci-editor').setValue(#{yaml_content.to_json});") - - # Ace editor updates a hidden textarea and it happens asynchronously - wait_for('YAML content') do - find(content_selector).text.present? - end + it 'loads previous YAML content after validation' do + expect(page).to have_field('content', with: 'my yaml content', visible: false, type: 'textarea') end end end - context 'with Editor Lite' do - it_behaves_like 'correct ci linting process' do - let(:content_selector) { '.content .view-lines' } - - before do - stub_feature_flags(monaco_ci: true) - stub_feature_flags(ci_lint_vue: false) - project.add_developer(user) - sign_in(user) + describe 'YAML clearing' do + before do + click_on 'Clear' + end - visit project_ci_lint_path(project) - editor_set_value(yaml_content) + context 'YAML is present' do + let(:yaml_content) do + File.read(Rails.root.join('spec/support/gitlab_stubs/gitlab_ci.yml')) + end - wait_for('YAML content') do - find(content_selector).text.present? - end + it 'YAML content is cleared' do + expect(page).to have_field('content', with: '', visible: false, type: 'textarea') end end end diff --git a/spec/features/projects/files/project_owner_sees_link_to_create_license_file_in_empty_project_spec.rb b/spec/features/projects/files/project_owner_sees_link_to_create_license_file_in_empty_project_spec.rb index eed1e7aaf1b..d28e31c08dc 100644 --- a/spec/features/projects/files/project_owner_sees_link_to_create_license_file_in_empty_project_spec.rb +++ b/spec/features/projects/files/project_owner_sees_link_to_create_license_file_in_empty_project_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe 'Projects > Files > Project owner sees a link to create a license file in empty project', :js do + include WebIdeSpecHelpers + let(:project) { create(:project_empty_repo) } let(:project_maintainer) { project.owner } @@ -10,36 +12,35 @@ RSpec.describe 'Projects > Files > Project owner sees a link to create a license sign_in(project_maintainer) end - it 'project maintainer creates a license file from a template' do + it 'allows project maintainer creates a license file from a template in Web IDE' do visit project_path(project) click_on 'Add LICENSE' - expect(page).to have_content('New file') - expect(current_path).to eq( - project_new_blob_path(project, 'master')) - expect(find('#file_name').value).to eq('LICENSE') - expect(page).to have_selector('.license-selector') + expect(current_path).to eq("/-/ide/project/#{project.full_path}/edit/master/-/LICENSE") + + expect(page).to have_selector('.qa-file-templates-bar') select_template('MIT License') - file_content = first('.file-editor') - expect(file_content).to have_content('MIT License') - expect(file_content).to have_content("Copyright (c) #{Time.now.year} #{project.namespace.human_name}") + expect(ide_editor_value).to have_content('MIT License') + expect(ide_editor_value).to have_content("Copyright (c) #{Time.now.year} #{project.namespace.human_name}") + + ide_commit + + click_button('Commit') + + expect(current_path).to eq("/-/ide/project/#{project.full_path}/tree/master/-/") - fill_in :commit_message, with: 'Add a LICENSE file', visible: true - click_button 'Commit changes' + expect(page).to have_content('All changes are committed') - expect(current_path).to eq( - project_blob_path(project, 'master/LICENSE')) - expect(page).to have_content('MIT License') - expect(page).to have_content("Copyright (c) #{Time.now.year} #{project.namespace.human_name}") + license_file = project.repository.blob_at('master', 'LICENSE').data + expect(license_file).to have_content('MIT License') + expect(license_file).to have_content("Copyright (c) #{Time.now.year} #{project.namespace.human_name}") end def select_template(template) - page.within('.js-license-selector-wrap') do - click_button 'Apply a template' - click_link template - wait_for_requests - end + click_button 'Choose a template...' + click_button template + wait_for_requests end end diff --git a/spec/features/projects/show/user_sees_setup_shortcut_buttons_spec.rb b/spec/features/projects/show/user_sees_setup_shortcut_buttons_spec.rb index afa9de5ce86..189aa45ff75 100644 --- a/spec/features/projects/show/user_sees_setup_shortcut_buttons_spec.rb +++ b/spec/features/projects/show/user_sees_setup_shortcut_buttons_spec.rb @@ -46,21 +46,21 @@ RSpec.describe 'Projects > Show > User sees setup shortcut buttons' do visit project_path(project) end - it '"New file" button linked to new file page' do + it '"New file" button linked to IDE new file page' do page.within('.project-buttons') do - expect(page).to have_link('New file', href: project_new_blob_path(project, project.default_branch || 'master')) + expect(page).to have_link('New file', href: presenter.ide_edit_path(project, project.default_branch || 'master')) end end - it '"Add README" button linked to new file populated for a README' do + it '"Add README" button linked to IDE new file populated for a README' do page.within('.project-buttons') do - expect(page).to have_link('Add README', href: presenter.add_readme_path) + expect(page).to have_link('Add README', href: presenter.add_readme_ide_path) end end - it '"Add license" button linked to new file populated for a license' do + it '"Add license" button linked to IDE new file populated for a license' do page.within('.project-buttons') do - expect(page).to have_link('Add LICENSE', href: presenter.add_license_path) + expect(page).to have_link('Add LICENSE', href: presenter.add_license_ide_path) end end @@ -74,9 +74,9 @@ RSpec.describe 'Projects > Show > User sees setup shortcut buttons' do visit project_path(project) end - it '"New file" button linked to new file page' do + it '"New file" button linked to IDE new file page' do page.within('.project-buttons') do - expect(page).to have_link('New file', href: project_new_blob_path(project, 'example_branch')) + expect(page).to have_link('New file', href: presenter.ide_edit_path(project, 'example_branch')) end end end @@ -144,7 +144,7 @@ RSpec.describe 'Projects > Show > User sees setup shortcut buttons' do expect(project.repository.readme).not_to be_nil page.within('.project-buttons') do - expect(page).not_to have_link('Add README', href: presenter.add_readme_path) + expect(page).not_to have_link('Add README', href: presenter.add_readme_ide_path) expect(page).to have_link('README', href: presenter.readme_path) end end @@ -164,7 +164,7 @@ RSpec.describe 'Projects > Show > User sees setup shortcut buttons' do end context 'when the project does not have a README' do - it 'shows the "Add README" button' do + it 'shows the single file editor "Add README" button' do allow(project.repository).to receive(:readme).and_return(nil) visit project_path(project) diff --git a/spec/features/tags/developer_views_tags_spec.rb b/spec/features/tags/developer_views_tags_spec.rb index 4888611472c..6bae53afe6f 100644 --- a/spec/features/tags/developer_views_tags_spec.rb +++ b/spec/features/tags/developer_views_tags_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe 'Developer views tags' do + include RepoHelpers + let(:user) { create(:user) } let(:group) { create(:group) } @@ -15,10 +17,13 @@ RSpec.describe 'Developer views tags' do let(:project) { create(:project_empty_repo, namespace: group) } before do - visit project_path(project) - click_on 'Add README' - fill_in :commit_message, with: 'Add a README file', visible: true - click_button 'Commit changes' + project.repository.create_file( + user, + 'README.md', + 'Example readme', + message: 'Add README', + branch_name: 'master') + visit project_tags_path(project) end diff --git a/spec/frontend/clusters/components/__snapshots__/applications_spec.js.snap b/spec/frontend/clusters/components/__snapshots__/applications_spec.js.snap index 3328ec724fd..b6e89281fef 100644 --- a/spec/frontend/clusters/components/__snapshots__/applications_spec.js.snap +++ b/spec/frontend/clusters/components/__snapshots__/applications_spec.js.snap @@ -94,7 +94,7 @@ exports[`Applications Prometheus application shows the correct description 1`] = Prometheus is an open-source monitoring system with <a class="gl-link" - href="https://docs.gitlab.com/ce/user/project/integrations/prometheus.html" + href="https://docs.gitlab.com/ee/user/project/integrations/prometheus.html" rel="noopener noreferrer" target="_blank" > diff --git a/spec/frontend/sidebar/assignee_title_spec.js b/spec/frontend/sidebar/assignee_title_spec.js index 92fabaa664e..b5d1e5216f8 100644 --- a/spec/frontend/sidebar/assignee_title_spec.js +++ b/spec/frontend/sidebar/assignee_title_spec.js @@ -11,6 +11,7 @@ describe('AssigneeTitle component', () => { propsData: { numberOfAssignees: 0, editable: false, + changing: false, ...props, }, }); @@ -62,6 +63,22 @@ describe('AssigneeTitle component', () => { }); }); + describe('when changing is false', () => { + it('renders "Edit"', () => { + wrapper = createComponent({ editable: true }); + + expect(wrapper.find('[data-test-id="edit-link"]').text()).toEqual('Edit'); + }); + }); + + describe('when changing is true', () => { + it('renders "Edit"', () => { + wrapper = createComponent({ editable: true, changing: true }); + + expect(wrapper.find('[data-test-id="edit-link"]').text()).toEqual('Apply'); + }); + }); + it('does not render spinner by default', () => { wrapper = createComponent({ numberOfAssignees: 0, diff --git a/spec/frontend/sidebar/lock/issuable_lock_form_spec.js b/spec/frontend/sidebar/lock/issuable_lock_form_spec.js index ab1423a9bbb..e8091dcb51d 100644 --- a/spec/frontend/sidebar/lock/issuable_lock_form_spec.js +++ b/spec/frontend/sidebar/lock/issuable_lock_form_spec.js @@ -1,5 +1,6 @@ import { shallowMount } from '@vue/test-utils'; import { mockTracking, triggerEvent } from 'helpers/tracking_helper'; +import { createMockDirective, getBinding } from 'helpers/vue_mock_directive'; import IssuableLockForm from '~/sidebar/components/lock/issuable_lock_form.vue'; import EditForm from '~/sidebar/components/lock/edit_form.vue'; import createStore from '~/notes/stores'; @@ -19,6 +20,8 @@ describe('IssuableLockForm', () => { const findLockStatus = () => wrapper.find('[data-testid="lock-status"]'); const findEditLink = () => wrapper.find('[data-testid="edit-link"]'); const findEditForm = () => wrapper.find(EditForm); + const findSidebarLockStatusTooltip = () => + getBinding(findSidebarCollapseIcon().element, 'gl-tooltip'); const initStore = isLocked => { if (issuableType === ISSUABLE_TYPE_ISSUE) { @@ -37,6 +40,9 @@ describe('IssuableLockForm', () => { isEditable: true, ...props, }, + directives: { + GlTooltip: createMockDirective(), + }, }); }; @@ -125,6 +131,13 @@ describe('IssuableLockForm', () => { expect(findEditForm().exists()).toBe(true); }); }); + + it('renders a tooltip with the lock status text', () => { + const tooltip = findSidebarLockStatusTooltip(); + + expect(tooltip).toBeDefined(); + expect(tooltip.value.title).toBe(isLocked ? 'Locked' : 'Unlocked'); + }); }); }); }); diff --git a/spec/frontend/sidebar/sidebar_assignees_spec.js b/spec/frontend/sidebar/sidebar_assignees_spec.js index 88e2d2c9514..dc4560d2ae8 100644 --- a/spec/frontend/sidebar/sidebar_assignees_spec.js +++ b/spec/frontend/sidebar/sidebar_assignees_spec.js @@ -20,6 +20,7 @@ describe('sidebar assignees', () => { mediator, field: '', projectPath: 'projectPath', + changing: false, ...props, }, provide: { diff --git a/spec/frontend/sidebar/sidebar_store_spec.js b/spec/frontend/sidebar/sidebar_store_spec.js index 6d063a7cfcf..7c18222f300 100644 --- a/spec/frontend/sidebar/sidebar_store_spec.js +++ b/spec/frontend/sidebar/sidebar_store_spec.js @@ -57,16 +57,40 @@ describe('Sidebar store', () => { expect(testContext.store.isFetching.assignees).toBe(true); }); - it('adds a new assignee', () => { - testContext.store.addAssignee(ASSIGNEE); + it('resets changing when resetChanging is called', () => { + testContext.store.changing = true; + + testContext.store.resetChanging(); - expect(testContext.store.assignees.length).toEqual(1); + expect(testContext.store.changing).toBe(false); }); - it('removes an assignee', () => { - testContext.store.removeAssignee(ASSIGNEE); + describe('when it adds a new assignee', () => { + beforeEach(() => { + testContext.store.addAssignee(ASSIGNEE); + }); - expect(testContext.store.assignees.length).toEqual(0); + it('adds a new assignee', () => { + expect(testContext.store.assignees).toHaveLength(1); + }); + + it('sets changing to true', () => { + expect(testContext.store.changing).toBe(true); + }); + }); + + describe('when it removes an assignee', () => { + beforeEach(() => { + testContext.store.removeAssignee(ASSIGNEE); + }); + + it('removes an assignee', () => { + expect(testContext.store.assignees).toHaveLength(0); + }); + + it('sets changing to true', () => { + expect(testContext.store.changing).toBe(true); + }); }); it('finds an existent assignee', () => { @@ -86,6 +110,7 @@ describe('Sidebar store', () => { testContext.store.removeAllAssignees(); expect(testContext.store.assignees.length).toEqual(0); + expect(testContext.store.changing).toBe(true); }); it('sets participants data', () => { diff --git a/spec/graphql/mutations/issues/create_spec.rb b/spec/graphql/mutations/issues/create_spec.rb new file mode 100644 index 00000000000..57658f6b358 --- /dev/null +++ b/spec/graphql/mutations/issues/create_spec.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Mutations::Issues::Create do + let_it_be(:project) { create(:project) } + let_it_be(:user) { create(:user) } + let_it_be(:assignee1) { create(:user) } + let_it_be(:assignee2) { create(:user) } + let_it_be(:project_label1) { create(:label, project: project) } + let_it_be(:project_label2) { create(:label, project: project) } + let_it_be(:milestone) { create(:milestone, project: project) } + let_it_be(:new_label1) { FFaker::Lorem.word } + let_it_be(:new_label2) { new_label1 + 'Extra' } + + let(:expected_attributes) do + { + title: 'new title', + description: 'new description', + confidential: true, + due_date: Date.tomorrow, + discussion_locked: true + } + end + + let(:mutation_params) do + { + project_path: project.full_path, + milestone_id: milestone.to_global_id, + labels: [project_label1.title, project_label2.title, new_label1, new_label2], + assignee_ids: [assignee1.to_global_id, assignee2.to_global_id] + }.merge(expected_attributes) + end + + let(:special_params) do + { + iid: non_existing_record_id, + created_at: 2.days.ago + } + end + + let(:mutation) { described_class.new(object: nil, context: { current_user: user }, field: nil) } + let(:mutated_issue) { subject[:issue] } + + specify { expect(described_class).to require_graphql_authorizations(:create_issue) } + + describe '#resolve' do + before do + stub_licensed_features(multiple_issue_assignees: false, issue_weights: false) + project.add_guest(assignee1) + project.add_guest(assignee2) + end + + subject { mutation.resolve(mutation_params) } + + context 'when the user does not have permission to create an issue' do + it 'raises an error' do + expect { subject }.to raise_error(Gitlab::Graphql::Errors::ResourceNotAvailable) + end + end + + context 'when the user can create an issue' do + context 'when creating an issue a developer' do + before do + project.add_developer(user) + end + + it 'creates issue with correct values' do + expect(mutated_issue).to have_attributes(expected_attributes) + expect(mutated_issue.milestone_id).to eq(milestone.id) + expect(mutated_issue.labels.pluck(:title)).to eq([project_label1.title, project_label2.title, new_label1, new_label2]) + expect(mutated_issue.assignees.pluck(:id)).to eq([assignee1.id]) + end + + context 'when passing in label_ids' do + before do + mutation_params.delete(:labels) + mutation_params.merge!(label_ids: [project_label1.to_global_id, project_label2.to_global_id]) + end + + it 'creates issue with correct values' do + expect(mutated_issue.labels.pluck(:title)).to eq([project_label1.title, project_label2.title]) + end + end + + context 'when trying to create issue with restricted params' do + before do + mutation_params.merge!(special_params) + end + + it 'ignores the special params' do + expect(mutated_issue).not_to be_like_time(special_params[:created_at]) + expect(mutated_issue.iid).not_to eq(special_params[:iid]) + end + end + end + + context 'when creating an issue as owner' do + let_it_be(:user) { project.owner } + + before do + mutation_params.merge!(special_params) + end + + it 'sets the special params' do + expect(mutated_issue.created_at).to be_like_time(special_params[:created_at]) + expect(mutated_issue.iid).to eq(special_params[:iid]) + end + end + end + end + + describe "#ready?" do + context 'when passing in both labels and label_ids' do + before do + mutation_params.merge!(label_ids: [project_label1.to_global_id, project_label2.to_global_id]) + end + + it 'raises exception when mutually exclusive params are given' do + expect { mutation.ready?(mutation_params) } + .to raise_error(Gitlab::Graphql::Errors::ArgumentError, /one and only one of/) + end + end + + context 'when passing only `discussion_to_resolve` param' do + before do + mutation_params.merge!(discussion_to_resolve: 'abc') + end + + it 'raises exception when mutually exclusive params are given' do + expect { mutation.ready?(mutation_params) } + .to raise_error(Gitlab::Graphql::Errors::ArgumentError, /to resolve a discussion please also provide `merge_request_to_resolve_discussions_of` parameter/) + end + end + + context 'when passing only `merge_request_to_resolve_discussions_of` param' do + before do + mutation_params.merge!(merge_request_to_resolve_discussions_of: 'abc') + end + + it 'raises exception when mutually exclusive params are given' do + expect { mutation.ready?(mutation_params) }.not_to raise_error + end + end + end +end diff --git a/spec/initializers/sidekiq_spec.rb b/spec/initializers/sidekiq_spec.rb new file mode 100644 index 00000000000..e34f59c3427 --- /dev/null +++ b/spec/initializers/sidekiq_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'sidekiq' do + describe 'enable_reliable_fetch?' do + subject { enable_reliable_fetch? } + + context 'when gitlab_sidekiq_reliable_fetcher is enabled' do + before do + stub_feature_flags(gitlab_sidekiq_reliable_fetcher: true) + end + + it { is_expected.to be_truthy } + end + + context 'when gitlab_sidekiq_reliable_fetcher is disabled' do + before do + stub_feature_flags(gitlab_sidekiq_reliable_fetcher: false) + end + + it { is_expected.to be_falsey } + end + end + + describe 'enable_semi_reliable_fetch_mode?' do + subject { enable_semi_reliable_fetch_mode? } + + context 'when gitlab_sidekiq_enable_semi_reliable_fetcher is enabled' do + before do + stub_feature_flags(gitlab_sidekiq_enable_semi_reliable_fetcher: true) + end + + it { is_expected.to be_truthy } + end + + context 'when gitlab_sidekiq_enable_semi_reliable_fetcher is disabled' do + before do + stub_feature_flags(gitlab_sidekiq_enable_semi_reliable_fetcher: false) + end + + it { is_expected.to be_falsey } + end + end +end diff --git a/spec/lib/marginalia_spec.rb b/spec/lib/marginalia_spec.rb index a920f598c24..fa0cd214c7e 100644 --- a/spec/lib/marginalia_spec.rb +++ b/spec/lib/marginalia_spec.rb @@ -24,18 +24,6 @@ RSpec.describe 'Marginalia spec' do end end - def stub_feature(value) - allow(Gitlab::Marginalia).to receive(:cached_feature_enabled?).and_return(value) - end - - def make_request(correlation_id) - request_env = Rack::MockRequest.env_for('/') - - ::Labkit::Correlation::CorrelationId.use_id(correlation_id) do - MarginaliaTestController.action(:first_user).call(request_env) - end - end - describe 'For rails web requests' do let(:correlation_id) { SecureRandom.uuid } let(:recorded) { ActiveRecord::QueryRecorder.new { make_request(correlation_id) } } @@ -149,4 +137,17 @@ RSpec.describe 'Marginalia spec' do end end end + + def stub_feature(value) + stub_feature_flags(marginalia: value) + Gitlab::Marginalia.set_enabled_from_feature_flag + end + + def make_request(correlation_id) + request_env = Rack::MockRequest.env_for('/') + + ::Labkit::Correlation::CorrelationId.use_id(correlation_id) do + MarginaliaTestController.action(:first_user).call(request_env) + end + end end diff --git a/spec/models/application_record_spec.rb b/spec/models/application_record_spec.rb index 5ea1907543a..d080b298e2f 100644 --- a/spec/models/application_record_spec.rb +++ b/spec/models/application_record_spec.rb @@ -90,4 +90,12 @@ RSpec.describe ApplicationRecord do expect(User.at_most(2).count).to eq(2) end end + + describe '.where_exists' do + it 'produces a WHERE EXISTS query' do + user = create(:user) + + expect(User.where_exists(User.limit(1))).to eq([user]) + end + end end diff --git a/spec/models/ci/build_trace_chunk_spec.rb b/spec/models/ci/build_trace_chunk_spec.rb index d8e6d5f3276..cdbdd2b1d20 100644 --- a/spec/models/ci/build_trace_chunk_spec.rb +++ b/spec/models/ci/build_trace_chunk_spec.rb @@ -508,20 +508,6 @@ RSpec.describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do subject { build_trace_chunk.persist_data! } - shared_examples_for 'Atomic operation' do - context 'when the other process is persisting' do - let(:lease_key) { "trace_write:#{build_trace_chunk.build.id}:chunks:#{build_trace_chunk.chunk_index}" } - - before do - stub_exclusive_lease_taken(lease_key) - end - - it 'raise an error' do - expect { subject }.to raise_error('Failed to obtain a lock') - end - end - end - context 'when data_store is redis' do let(:data_store) { :redis } @@ -552,8 +538,6 @@ RSpec.describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do expect(build_trace_chunk.reload.checksum).to eq '3398914352' end - - it_behaves_like 'Atomic operation' end context 'when data size has not reached CHUNK_SIZE' do @@ -606,6 +590,35 @@ RSpec.describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do /Modifed build trace chunk detected/) end end + + context 'when the chunk is being locked by a different worker' do + let(:metrics) { spy('metrics') } + + it 'does not raise an exception' do + lock_chunk do + expect { build_trace_chunk.persist_data! }.not_to raise_error + end + end + + it 'increments stalled chunk trace metric' do + allow(build_trace_chunk) + .to receive(:metrics) + .and_return(metrics) + + lock_chunk { build_trace_chunk.persist_data! } + + expect(metrics) + .to have_received(:increment_trace_operation) + .with(operation: :stalled) + .once + end + + def lock_chunk(&block) + "trace_write:#{build.id}:chunks:#{chunk_index}".then do |key| + build_trace_chunk.in_lock(key, &block) + end + end + end end end @@ -640,8 +653,6 @@ RSpec.describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do expect(Ci::BuildTraceChunks::Database.new.data(build_trace_chunk)).to be_nil expect(Ci::BuildTraceChunks::Fog.new.data(build_trace_chunk)).to eq(data) end - - it_behaves_like 'Atomic operation' end context 'when data size has not reached CHUNK_SIZE' do @@ -701,8 +712,6 @@ RSpec.describe Ci::BuildTraceChunk, :clean_gitlab_redis_shared_state do expect(Ci::BuildTraceChunks::Database.new.data(build_trace_chunk)).to be_nil expect(Ci::BuildTraceChunks::Fog.new.data(build_trace_chunk)).to eq(data) end - - it_behaves_like 'Atomic operation' end context 'when data size has not reached CHUNK_SIZE' do diff --git a/spec/requests/api/graphql/mutations/issues/create_spec.rb b/spec/requests/api/graphql/mutations/issues/create_spec.rb new file mode 100644 index 00000000000..39b408faa90 --- /dev/null +++ b/spec/requests/api/graphql/mutations/issues/create_spec.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'Create an issue' do + include GraphqlHelpers + + let_it_be(:current_user) { create(:user) } + let_it_be(:project) { create(:project) } + let_it_be(:assignee1) { create(:user) } + let_it_be(:assignee2) { create(:user) } + let_it_be(:project_label1) { create(:label, project: project) } + let_it_be(:project_label2) { create(:label, project: project) } + let_it_be(:milestone) { create(:milestone, project: project) } + let_it_be(:new_label1) { FFaker::Lorem.word } + let_it_be(:new_label2) { FFaker::Lorem.word } + + let(:input) do + { + 'title' => 'new title', + 'description' => 'new description', + 'confidential' => true, + 'dueDate' => Date.tomorrow.strftime('%Y-%m-%d') + } + end + + let(:mutation) { graphql_mutation(:createIssue, input.merge('projectPath' => project.full_path, 'locked' => true)) } + + let(:mutation_response) { graphql_mutation_response(:create_issue) } + + context 'the user is not allowed to create an issue' do + it_behaves_like 'a mutation that returns a top-level access error' + end + + context 'when user has permissions to create an issue' do + before do + project.add_developer(current_user) + end + + it 'updates the issue' do + post_graphql_mutation(mutation, current_user: current_user) + + expect(response).to have_gitlab_http_status(:success) + expect(mutation_response['issue']).to include(input) + expect(mutation_response['issue']).to include('discussionLocked' => true) + end + end +end diff --git a/spec/requests/api/graphql/mutations/issues/update_spec.rb b/spec/requests/api/graphql/mutations/issues/update_spec.rb index af52f9d57a3..71f25dbbe49 100644 --- a/spec/requests/api/graphql/mutations/issues/update_spec.rb +++ b/spec/requests/api/graphql/mutations/issues/update_spec.rb @@ -10,13 +10,15 @@ RSpec.describe 'Update of an existing issue' do let_it_be(:issue) { create(:issue, project: project) } let(:input) do { - project_path: project.full_path, - iid: issue.iid.to_s, - locked: true + 'iid' => issue.iid.to_s, + 'title' => 'new title', + 'description' => 'new description', + 'confidential' => true, + 'dueDate' => Date.tomorrow.strftime('%Y-%m-%d') } end - let(:mutation) { graphql_mutation(:update_issue, input) } + let(:mutation) { graphql_mutation(:update_issue, input.merge(project_path: project.full_path, locked: true)) } let(:mutation_response) { graphql_mutation_response(:update_issue) } context 'the user is not allowed to update issue' do @@ -32,9 +34,8 @@ RSpec.describe 'Update of an existing issue' do post_graphql_mutation(mutation, current_user: current_user) expect(response).to have_gitlab_http_status(:success) - expect(mutation_response['issue']).to include( - 'discussionLocked' => true - ) + expect(mutation_response['issue']).to include(input) + expect(mutation_response['issue']).to include('discussionLocked' => true) end end end diff --git a/spec/rubocop/cop/graphql/gid_expected_type_spec.rb b/spec/rubocop/cop/graphql/gid_expected_type_spec.rb new file mode 100644 index 00000000000..a81af2aea5d --- /dev/null +++ b/spec/rubocop/cop/graphql/gid_expected_type_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require 'rubocop' + +require_relative '../../../../rubocop/cop/graphql/gid_expected_type' + +RSpec.describe RuboCop::Cop::Graphql::GIDExpectedType, type: :rubocop do + include CopHelper + + subject(:cop) { described_class.new } + + it 'adds an offense when there is no expected_type parameter' do + inspect_source(<<~TYPE) + GitlabSchema.object_from_id(received_id) + TYPE + + expect(cop.offenses.size).to eq 1 + end + + it 'does not add an offense for calls that have an expected_type parameter' do + expect_no_offenses(<<~TYPE.strip) + GitlabSchema.object_from_id("some_id", expected_type: SomeClass) + TYPE + end +end diff --git a/spec/rubocop/cop/migration/create_table_with_foreign_keys_spec.rb b/spec/rubocop/cop/migration/create_table_with_foreign_keys_spec.rb index fa4acc62226..93f43b0feb0 100644 --- a/spec/rubocop/cop/migration/create_table_with_foreign_keys_spec.rb +++ b/spec/rubocop/cop/migration/create_table_with_foreign_keys_spec.rb @@ -83,7 +83,7 @@ RSpec.describe RuboCop::Cop::Migration::CreateTableWithForeignKeys, type: :ruboc context 'with more than one foreign keys' do let(:offense) do 'Creating a table with more than one foreign key at once violates our migration style guide. ' \ - 'For more details check the https://docs.gitlab.com/ce/development/migration_style_guide.html#examples' + 'For more details check the https://docs.gitlab.com/ee/development/migration_style_guide.html#examples' end shared_examples 'target to high traffic table' do |dsl_method, table_name| diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2962225859d..6e41ae3fbfd 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -229,7 +229,7 @@ RSpec.configure do |config| end # Enable Marginalia feature for all specs in the test suite. - allow(Gitlab::Marginalia).to receive(:cached_feature_enabled?).and_return(true) + Gitlab::Marginalia.enabled = true # Stub these calls due to being expensive operations # It can be reenabled for specific tests via: diff --git a/spec/support/helpers/stubbed_feature.rb b/spec/support/helpers/stubbed_feature.rb index 4113a28182b..67ceb7d9b35 100644 --- a/spec/support/helpers/stubbed_feature.rb +++ b/spec/support/helpers/stubbed_feature.rb @@ -4,6 +4,14 @@ module StubbedFeature extend ActiveSupport::Concern + prepended do + cattr_reader(:persist_used) do + # persist feature flags in CI + # nil: indicates that we do not want to persist used feature flags + Gitlab::Utils.to_boolean(ENV['CI']) ? {} : nil + end + end + class_methods do # Turn stubbed feature flags on or off. def stub=(stub) @@ -33,6 +41,8 @@ module StubbedFeature feature_flag = super return feature_flag unless stub? + persist_used!(args.first) + # If feature flag is not persisted we mark the feature flag as enabled # We do `m.call` as we want to validate the execution of method arguments # and a feature flag state if it is not persisted @@ -42,5 +52,17 @@ module StubbedFeature feature_flag end + + # This method creates a temporary file in `tmp/feature_flags` + # if feature flag was touched during execution + def persist_used!(name) + return unless persist_used + return if persist_used[name] + + persist_used[name] = true + FileUtils.touch( + Rails.root.join('tmp', 'feature_flags', name.to_s + ".used") + ) + end end end diff --git a/spec/support/shared_examples/features/multiple_assignees_mr_shared_examples.rb b/spec/support/shared_examples/features/multiple_assignees_mr_shared_examples.rb index 19a5750cf6d..9d023d9514a 100644 --- a/spec/support/shared_examples/features/multiple_assignees_mr_shared_examples.rb +++ b/spec/support/shared_examples/features/multiple_assignees_mr_shared_examples.rb @@ -38,7 +38,7 @@ RSpec.shared_examples 'multiple assignees merge request' do |action, save_button page.within '.issuable-sidebar' do page.within '.assignee' do # Closing dropdown to persist - click_link 'Edit' + click_link 'Apply' expect(page).to have_content user2.name end diff --git a/tmp/feature_flags/.gitkeep b/tmp/feature_flags/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/tmp/feature_flags/.gitkeep |