diff options
193 files changed, 5242 insertions, 2778 deletions
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml index 89b5ae38072..ca55bbd32a7 100644 --- a/.gitlab/ci/reports.gitlab-ci.yml +++ b/.gitlab/ci/reports.gitlab-ci.yml @@ -10,17 +10,17 @@ code_quality: tags: [] before_script: [] cache: {} - dependencies: [] sast: extends: .dedicated-no-docs - before_script: [] tags: [] + before_script: [] + cache: {} variables: - SAST_CONFIDENCE_LEVEL: 2 - DOCKER_DRIVER: overlay2 + SAST_BRAKEMAN_LEVEL: 2 dependency_scanning: - before_script: [] - tags: [] extends: .dedicated-no-docs + tags: [] + before_script: [] + cache: {} diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index a63cb35e6f0..3f4830156cb 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -1.52.0 +1.53.0 @@ -84,6 +84,7 @@ gem 'rack-cors', '~> 1.0.0', require: 'rack/cors' gem 'graphql', '~> 1.8.0' gem 'graphiql-rails', '~> 1.4.10' gem 'apollo_upload_server', '~> 2.0.0.beta3' +gem 'graphql-docs', '~> 1.6.0', group: [:development, :test] # Disable strong_params so that Mash does not respond to :permitted? gem 'hashie-forbidden_attributes' diff --git a/Gemfile.lock b/Gemfile.lock index 1f4705dd173..1a3b3b8e125 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -216,6 +216,8 @@ GEM excon (0.62.0) execjs (2.6.0) expression_parser (0.9.0) + extended-markdown-filter (0.6.0) + html-pipeline (~> 2.0) factory_bot (4.8.2) activesupport (>= 3.0.0) factory_bot_rails (4.8.2) @@ -290,6 +292,7 @@ GEM fuubar (2.2.0) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) + gemoji (3.0.1) gemojione (3.3.0) json get_process_mem (0.2.3) @@ -370,6 +373,14 @@ GEM railties sprockets-rails graphql (1.8.1) + graphql-docs (1.6.0) + commonmarker (~> 0.16) + escape_utils (~> 1.2) + extended-markdown-filter (~> 0.4) + gemoji (~> 3.0) + graphql (~> 1.6) + html-pipeline (~> 2.8) + sass (~> 3.4) grpc (1.19.0) google-protobuf (~> 3.1) googleapis-common-protos-types (~> 1.0.0) @@ -1118,6 +1129,7 @@ DEPENDENCIES grape_logging (~> 1.7) graphiql-rails (~> 1.4.10) graphql (~> 1.8.0) + graphql-docs (~> 1.6.0) grpc (~> 1.19.0) haml_lint (~> 0.31.0) hamlit (~> 2.8.8) diff --git a/app/assets/javascripts/filterable_list.js b/app/assets/javascripts/filterable_list.js index 64b09c8b62c..77080691dcb 100644 --- a/app/assets/javascripts/filterable_list.js +++ b/app/assets/javascripts/filterable_list.js @@ -17,11 +17,13 @@ export default class FilterableList { } getFilterEndpoint() { - return `${this.filterForm.getAttribute('action')}?${$(this.filterForm).serialize()}`; + return this.getPagePath(); } getPagePath() { - return this.getFilterEndpoint(); + const action = this.filterForm.getAttribute('action'); + const params = $(this.filterForm).serialize(); + return `${action}${action.indexOf('?') > 0 ? '&' : '?'}${params}`; } initSearch() { diff --git a/app/assets/javascripts/notes/components/note_actions.vue b/app/assets/javascripts/notes/components/note_actions.vue index 844d0c3e376..6cc873359da 100644 --- a/app/assets/javascripts/notes/components/note_actions.vue +++ b/app/assets/javascripts/notes/components/note_actions.vue @@ -165,7 +165,7 @@ export default { v-gl-tooltip type="button" title="Edit comment" - class="note-action-button js-note-edit btn btn-transparent" + class="note-action-button js-note-edit btn btn-transparent qa-note-edit-button" @click="onEdit" > <icon name="pencil" css-classes="link-highlight" /> diff --git a/app/assets/javascripts/projects/projects_filterable_list.js b/app/assets/javascripts/projects/projects_filterable_list.js new file mode 100644 index 00000000000..433c894e668 --- /dev/null +++ b/app/assets/javascripts/projects/projects_filterable_list.js @@ -0,0 +1,7 @@ +import FilterableList from '~/filterable_list'; + +export default class ProjectsFilterableList extends FilterableList { + getFilterEndpoint() { + return this.getPagePath().replace('/projects?', '/projects.json?'); + } +} diff --git a/app/assets/javascripts/projects_list.js b/app/assets/javascripts/projects_list.js index c67d59d2be5..913b62ba26d 100644 --- a/app/assets/javascripts/projects_list.js +++ b/app/assets/javascripts/projects_list.js @@ -1,4 +1,4 @@ -import FilterableList from './filterable_list'; +import ProjectsFilterableList from './projects/projects_filterable_list'; /** * Makes search request for projects when user types a value in the search input. @@ -11,7 +11,7 @@ export default class ProjectsList { const holder = document.querySelector('.js-projects-list-holder'); if (form && filter && holder) { - const list = new FilterableList(form, filter, holder); + const list = new ProjectsFilterableList(form, filter, holder); list.initSearch(); } } diff --git a/app/assets/javascripts/registry/components/collapsible_container.vue b/app/assets/javascripts/registry/components/collapsible_container.vue index 1e266dd4ced..e157036871b 100644 --- a/app/assets/javascripts/registry/components/collapsible_container.vue +++ b/app/assets/javascripts/registry/components/collapsible_container.vue @@ -31,6 +31,7 @@ export default { data() { return { isOpen: false, + modalId: `confirm-repo-deletion-modal-${this.repo.id}`, }; }, computed: { @@ -80,7 +81,7 @@ export default { <gl-button v-if="repo.canDelete" v-gl-tooltip - v-gl-modal="'confirm-repo-deletion-modal'" + v-gl-modal="modalId" :title="s__('ContainerRegistry|Remove repository')" :aria-label="s__('ContainerRegistry|Remove repository')" class="js-remove-repo" @@ -100,12 +101,7 @@ export default { {{ s__('ContainerRegistry|No tags in Container Registry for this container image.') }} </div> </div> - - <gl-modal - modal-id="confirm-repo-deletion-modal" - ok-variant="danger" - @ok="handleDeleteRepository" - > + <gl-modal :modal-id="modalId" ok-variant="danger" @ok="handleDeleteRepository"> <template v-slot:modal-title>{{ s__('ContainerRegistry|Remove repository') }}</template> <p v-html=" diff --git a/app/assets/javascripts/registry/components/table_registry.vue b/app/assets/javascripts/registry/components/table_registry.vue index 0ec5e2c7a87..a498a553908 100644 --- a/app/assets/javascripts/registry/components/table_registry.vue +++ b/app/assets/javascripts/registry/components/table_registry.vue @@ -32,6 +32,7 @@ export default { data() { return { itemToBeDeleted: null, + modalId: `confirm-image-deletion-modal-${this.repo.id}`, }; }, computed: { @@ -114,7 +115,7 @@ export default { <gl-button v-if="item.canDelete" v-gl-tooltip - v-gl-modal="'confirm-image-deletion-modal'" + v-gl-modal="modalId" :title="s__('ContainerRegistry|Remove image')" :aria-label="s__('ContainerRegistry|Remove image')" variant="danger" @@ -134,11 +135,7 @@ export default { :page-info="repo.pagination" /> - <gl-modal - modal-id="confirm-image-deletion-modal" - ok-variant="danger" - @ok="handleDeleteRegistry" - > + <gl-modal :modal-id="modalId" ok-variant="danger" @ok="handleDeleteRegistry"> <template v-slot:modal-title>{{ s__('ContainerRegistry|Remove image') }}</template> <template v-slot:modal-ok>{{ s__('ContainerRegistry|Remove image and tags') }}</template> <p diff --git a/app/assets/javascripts/visual_review_toolbar/styles/toolbar.css b/app/assets/javascripts/visual_review_toolbar/styles/toolbar.css index 00a55c0027a..6a7b2f52549 100644 --- a/app/assets/javascripts/visual_review_toolbar/styles/toolbar.css +++ b/app/assets/javascripts/visual_review_toolbar/styles/toolbar.css @@ -48,6 +48,7 @@ font-size: .8rem; font-weight: 400; color: #2e2e2e; + z-index: 9999; /* toolbar should always be on top */ } .gitlab-wrapper-open { diff --git a/app/assets/stylesheets/framework/animations.scss b/app/assets/stylesheets/framework/animations.scss index 6f5a2e561af..6bc5632365f 100644 --- a/app/assets/stylesheets/framework/animations.scss +++ b/app/assets/stylesheets/framework/animations.scss @@ -104,7 +104,7 @@ } .btn { - @include transition(background-color, border-color, color, box-shadow); + @include transition(border-color); } .dropdown-menu-toggle, diff --git a/app/assets/stylesheets/framework/buttons.scss b/app/assets/stylesheets/framework/buttons.scss index 767832e242c..e0b6da31261 100644 --- a/app/assets/stylesheets/framework/buttons.scss +++ b/app/assets/stylesheets/framework/buttons.scss @@ -24,12 +24,11 @@ border-radius: $border-radius-default; font-size: $gl-font-size; font-weight: $gl-font-weight-normal; - padding: $gl-vert-padding $gl-btn-padding; + padding: $gl-bordered-btn-vert-padding $gl-bordered-btn-horz-padding; &:focus, &:active { background-color: $btn-active-gray; - box-shadow: $gl-btn-active-background; } } @@ -50,77 +49,89 @@ color: $text; } - &:hover, - &:focus { - background-color: $hover-background; - border-color: $hover-border; - color: $hover-text; + &:not(:disabled):not(.disabled) { + &:hover { + box-shadow: inset 0 0 0 1px $hover-border, 0 2px 2px 0 $gl-btn-hover-shadow-light; + } - > .icon { - color: $hover-text; + &:focus { + box-shadow: inset 0 0 0 1px $hover-border, 0 0 4px 1px $blue-300; } - } - &:focus { - box-shadow: 0 0 4px 1px $blue-300; - } + &:hover, + &:focus { + background-color: $hover-background; + border-color: $hover-border; + color: $hover-text; - &:active { - background-color: $active-background; - border-color: $active-border; - box-shadow: inset 0 2px 4px 0 rgba($black, 0.2); - color: $active-text; + > .icon { + color: $hover-text; + } + } - > .icon { + &:active, + &:active:focus { + background-color: $active-background; + border-color: $active-border; + box-shadow: inset 0 0 0 1px $hover-border, inset 0 2px 4px 0 rgba($black, 0.2); color: $active-text; - } - &:focus { - box-shadow: inset 0 2px 4px 0 rgba($black, 0.2); + > .icon { + color: $active-text; + } } } } -@mixin btn-color($light, $border-light, $normal, $border-normal, $dark, $border-dark, $color) { +@mixin btn-color($light, $border-light, $normal, $border-normal, $dark, $border-dark, $color, $hover-shadow-color: $gl-btn-hover-shadow-dark) { background-color: $light; border-color: $border-light; color: $color; - &:hover, - &:focus { - background-color: $normal; - border-color: $border-normal; - color: $color; - } + &:not(:disabled):not(.disabled) { + &:hover { + box-shadow: inset 0 0 0 1px $border-normal, 0 2px 2px 0 $hover-shadow-color; + } - &:active, - &.active { - box-shadow: $gl-btn-active-background; + &:focus { + box-shadow: inset 0 0 0 1px $border-normal, 0 0 4px 1px $blue-300; + } - background-color: $dark; - border-color: $border-dark; - color: $color; + &:hover, + &:focus { + background-color: $normal; + border-color: $border-normal; + color: $color; + } + + &:active, + &.active { + box-shadow: inset 0 2px 4px 0 $gl-btn-hover-shadow-dark; + background-color: $dark; + border-color: $border-dark; + color: $color; + } } } @mixin btn-green { - @include btn-color($green-500, $green-600, $green-600, $green-700, $green-700, $green-800, $white-light); + @include btn-color($green-500, $green-600, $green-500, $green-700, $green-600, $green-800, $white-light); } @mixin btn-blue { - @include btn-color($blue-500, $blue-600, $blue-600, $blue-700, $blue-700, $blue-800, $white-light); + @include btn-color($blue-500, $blue-600, $blue-500, $blue-700, $blue-600, $blue-800, $white-light); } @mixin btn-orange { - @include btn-color($orange-500, $orange-600, $orange-600, $orange-700, $orange-700, $orange-800, $white-light); + @include btn-color($orange-500, $orange-600, $orange-500, $orange-700, $orange-600, $orange-800, $white-light); } @mixin btn-red { - @include btn-color($red-500, $red-600, $red-600, $red-700, $red-700, $red-800, $white-light); + @include btn-color($red-500, $red-600, $red-500, $red-700, $red-600, $red-800, $white-light); } @mixin btn-white { - @include btn-color($white-light, $border-color, $white-normal, $border-white-normal, $white-dark, $border-gray-dark, $gl-text-color); + @include btn-color($white-light, $gray-400, $gray-200, $gray-400, $gl-gray-200, $gray-500, $gl-text-color, $gl-btn-hover-shadow-light); } @mixin btn-with-margin { @@ -149,23 +160,22 @@ color: $gl-text-color; white-space: nowrap; + line-height: $gl-btn-line-height; &:focus:active { outline: 0; } - &.btn-sm { - padding: 4px 10px; - font-size: $gl-btn-small-font-size; - line-height: $gl-btn-small-line-height; - } - &.btn-xs { - padding: 2px $gl-btn-padding; font-size: $gl-btn-xs-font-size; line-height: $gl-btn-xs-line-height; } + &.btn-sm, + &.btn-xs { + padding: 3px $gl-bordered-btn-vert-padding; + } + &.btn-success, &.btn-register { @include btn-green; @@ -239,7 +249,7 @@ &.dropdown-toggle { .fa-caret-down { - margin-left: 3px; + margin: 0; } } @@ -272,10 +282,7 @@ } svg { - height: 15px; - width: 15px; - position: relative; - top: 2px; + @include btn-svg; } svg, @@ -330,6 +337,12 @@ &.btn-grouped { @include btn-with-margin; } + + .btn { + border-radius: $border-radius-default; + font-size: $gl-font-size; + line-height: $gl-btn-line-height; + } } .btn-clipboard { @@ -487,18 +500,25 @@ &:active, &:focus { color: $gl-text-color-secondary; + border: 1px solid $border-gray-normal-dashed; background-color: $white-normal; } } -.btn-svg svg { - @include btn-svg; +.btn-svg { + padding: $gl-bordered-btn-vert-padding; + + svg { + @include btn-svg; + display: block; + } } // All disabled buttons, regardless of color, type, etc %disabled { background-color: $gray-light !important; border-color: $gray-200 !important; + box-shadow: none; color: $gl-text-color-disabled !important; opacity: 1 !important; cursor: default !important; diff --git a/app/assets/stylesheets/framework/dropdowns.scss b/app/assets/stylesheets/framework/dropdowns.scss index 29f63e9578d..05afcecca05 100644 --- a/app/assets/stylesheets/framework/dropdowns.scss +++ b/app/assets/stylesheets/framework/dropdowns.scss @@ -8,12 +8,6 @@ } } -@mixin chevron-active { - .fa-chevron-down { - color: $gray-darkest; - } -} - @mixin set-visible { transform: translateY(0); display: block; @@ -49,7 +43,6 @@ .dropdown-toggle, .dropdown-menu-toggle { - @include chevron-active; border-color: $gray-darkest; } @@ -65,12 +58,12 @@ .dropdown-toggle, .confidential-merge-request-fork-group .dropdown-toggle { - padding: 6px 8px 6px 10px; + padding: $gl-bordered-btn-vert-padding $gl-bordered-btn-horz-padding; background-color: $white-light; color: $gl-text-color; font-size: 14px; + line-height: $gl-btn-line-height; text-align: left; - border: 1px solid $border-color; border-radius: $border-radius-base; white-space: nowrap; @@ -103,10 +96,6 @@ padding-right: 25px; } - .fa { - color: $gray-darkest; - } - .fa-chevron-down { font-size: $dropdown-chevron-size; position: relative; @@ -115,12 +104,10 @@ } &:hover { - @include chevron-active; border-color: $gray-darkest; } &:focus:active { - @include chevron-active; border-color: $dropdown-toggle-active-border-color; outline: 0; } diff --git a/app/assets/stylesheets/framework/lists.scss b/app/assets/stylesheets/framework/lists.scss index 954551fef97..460d9ea9526 100644 --- a/app/assets/stylesheets/framework/lists.scss +++ b/app/assets/stylesheets/framework/lists.scss @@ -265,7 +265,6 @@ ul.controls { } .issuable-pipeline-broken a, - .issuable-pipeline-status a, .author-link { display: flex; } diff --git a/app/assets/stylesheets/framework/modal.scss b/app/assets/stylesheets/framework/modal.scss index b721b90fbb3..fd9a75bc5b6 100644 --- a/app/assets/stylesheets/framework/modal.scss +++ b/app/assets/stylesheets/framework/modal.scss @@ -19,16 +19,23 @@ } } - // leave enough space for the close icon .modal-title { + line-height: $gl-line-height-24; + + // leave enough space for the close icon &.mw-100, &.w-100 { - // after upgrading to Bootstrap 4.2 we can use $modal-header-padding-x here - // https://github.com/twbs/bootstrap/pull/26976 - margin-right: -28px; - padding-right: 28px; + margin-right: -$modal-header-padding-x; + padding-right: $modal-header-padding-x; } } + + .close { + font-weight: $gl-font-weight-normal; + line-height: $gl-line-height; + color: $gray-900; + opacity: 1; + } } .modal-body { @@ -63,6 +70,10 @@ margin-left: $grid-size; } + .btn-group .btn + .btn { + margin-left: -1px; + } + @include media-breakpoint-down(xs) { flex-direction: column; @@ -72,6 +83,11 @@ margin-left: 0; margin-top: $grid-size; } + + .btn-group .btn + .btn { + margin-left: -1px; + margin-top: 0; + } } } @@ -93,12 +109,12 @@ body.modal-open { .modal-content { border-radius: $modal-border-radius; - *:first-child { + > :first-child { border-top-left-radius: $modal-border-radius; border-top-right-radius: $modal-border-radius; } - *:last-child { + > :last-child { border-bottom-left-radius: $modal-border-radius; border-bottom-right-radius: $modal-border-radius; } diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index c108f45622f..047a9799c3f 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -405,6 +405,8 @@ $tanuki-yellow: #fca326; */ $green-500-focus: rgba($green-500, 0.4); $gl-btn-active-background: rgba(0, 0, 0, 0.16); +$gl-btn-hover-shadow-dark: rgba($black, 0.2); +$gl-btn-hover-shadow-light: rgba($black, 0.1); $gl-btn-active-gradient: inset 0 2px 3px $gl-btn-active-background; /* @@ -481,6 +483,8 @@ $gl-btn-padding: 10px; $gl-btn-line-height: 16px; $gl-btn-vert-padding: 8px; $gl-btn-horz-padding: 12px; +$gl-bordered-btn-vert-padding: $gl-btn-vert-padding - 1px; +$gl-bordered-btn-horz-padding: $gl-btn-horz-padding - 1px; $gl-btn-small-font-size: 13px; $gl-btn-small-line-height: 18px; $gl-btn-xs-font-size: 13px; diff --git a/app/assets/stylesheets/pages/commits.scss b/app/assets/stylesheets/pages/commits.scss index e12ea6fcb99..ffc6e433988 100644 --- a/app/assets/stylesheets/pages/commits.scss +++ b/app/assets/stylesheets/pages/commits.scss @@ -214,10 +214,10 @@ .label, .btn { - padding: $gl-vert-padding $gl-btn-padding; + padding: $gl-bordered-btn-vert-padding $gl-bordered-btn-horz-padding; border: 1px $border-color solid; font-size: $gl-font-size; - line-height: $line-height-base; + line-height: $gl-btn-line-height; border-radius: 0; display: flex; align-items: center; diff --git a/app/assets/stylesheets/pages/issuable.scss b/app/assets/stylesheets/pages/issuable.scss index 6a0127eb51c..66ea70e79da 100644 --- a/app/assets/stylesheets/pages/issuable.scss +++ b/app/assets/stylesheets/pages/issuable.scss @@ -929,10 +929,6 @@ margin: 0; } } - - .dropdown-toggle > .icon { - margin: 0 3px; - } } .right-sidebar-collapsed { diff --git a/app/assets/stylesheets/pages/issues.scss b/app/assets/stylesheets/pages/issues.scss index 8359a60ec9f..e51ca44476c 100644 --- a/app/assets/stylesheets/pages/issues.scss +++ b/app/assets/stylesheets/pages/issues.scss @@ -267,7 +267,6 @@ ul.related-merge-requests > li { .fa-caret-down { pointer-events: none; color: inherit; - margin-left: 0; } } } diff --git a/app/assets/stylesheets/pages/note_form.scss b/app/assets/stylesheets/pages/note_form.scss index c6bac33e888..1d57a4a4784 100644 --- a/app/assets/stylesheets/pages/note_form.scss +++ b/app/assets/stylesheets/pages/note_form.scss @@ -417,7 +417,6 @@ table { i { color: $white-light; - padding-right: 2px; margin-top: 2px; } diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss index c80beceae52..73ba09dbba5 100644 --- a/app/assets/stylesheets/pages/projects.scss +++ b/app/assets/stylesheets/pages/projects.scss @@ -429,7 +429,7 @@ padding: 0; background: transparent; border: 0; - line-height: 34px; + line-height: 2; margin: 0; > li + li::before { @@ -792,7 +792,6 @@ .btn { margin-top: $gl-padding; - padding: $gl-btn-vert-padding $gl-btn-padding; line-height: $gl-btn-line-height; .icon { diff --git a/app/assets/stylesheets/pages/tree.scss b/app/assets/stylesheets/pages/tree.scss index 5664f46484e..5c732ab0d1f 100644 --- a/app/assets/stylesheets/pages/tree.scss +++ b/app/assets/stylesheets/pages/tree.scss @@ -90,7 +90,7 @@ .add-to-tree { vertical-align: top; - padding: 8px; + padding: $gl-bordered-btn-vert-padding; svg { top: 0; diff --git a/app/controllers/boards/issues_controller.rb b/app/controllers/boards/issues_controller.rb index 0dd7500623d..353a9806fd1 100644 --- a/app/controllers/boards/issues_controller.rb +++ b/app/controllers/boards/issues_controller.rb @@ -2,16 +2,24 @@ module Boards class IssuesController < Boards::ApplicationController + # This is the maximum amount of issues which can be moved by one request to + # bulk_move for now. This is temporary and might be removed in future by + # introducing an alternative (async?) approach. + # (related: https://gitlab.com/groups/gitlab-org/-/epics/382) + MAX_MOVE_ISSUES_COUNT = 50 + include BoardsResponses include ControllerWithCrossProjectAccessCheck requires_cross_project_access if: -> { board&.group_board? } - before_action :whitelist_query_limiting, only: [:index, :update] + before_action :whitelist_query_limiting, only: [:index, :update, :bulk_move] before_action :authorize_read_issue, only: [:index] before_action :authorize_create_issue, only: [:create] before_action :authorize_update_issue, only: [:update] skip_before_action :authenticate_user!, only: [:index] + before_action :validate_id_list, only: [:bulk_move] + before_action :can_move_issues?, only: [:bulk_move] # rubocop: disable CodeReuse/ActiveRecord def index @@ -46,6 +54,17 @@ module Boards end end + def bulk_move + service = Boards::Issues::MoveService.new(board_parent, current_user, move_params(true)) + + issues = Issue.find(params[:ids]) + if service.execute_multiple(issues) + head :ok + else + head :unprocessable_entity + end + end + def update service = Boards::Issues::MoveService.new(board_parent, current_user, move_params) @@ -58,6 +77,10 @@ module Boards private + def can_move_issues? + head(:forbidden) unless can?(current_user, :admin_issue, board) + end + def render_issues(issues, metadata) data = { issues: serialize_as_json(issues) } data.merge!(metadata) @@ -90,8 +113,9 @@ module Boards end end - def move_params - params.permit(:board_id, :id, :from_list_id, :to_list_id, :move_before_id, :move_after_id) + def move_params(multiple = false) + id_param = multiple ? :ids : :id + params.permit(id_param, :board_id, :from_list_id, :to_list_id, :move_before_id, :move_after_id) end def issue_params @@ -112,5 +136,10 @@ module Boards # Also see https://gitlab.com/gitlab-org/gitlab-ce/issues/42439 Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42428') end + + def validate_id_list + head(:bad_request) unless params[:ids].is_a?(Array) + head(:unprocessable_entity) if params[:ids].size > MAX_MOVE_ISSUES_COUNT + end end end diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb index 8d518c14b90..2985da35d83 100644 --- a/app/controllers/chaos_controller.rb +++ b/app/controllers/chaos_controller.rb @@ -1,35 +1,36 @@ # frozen_string_literal: true class ChaosController < ActionController::Base - before_action :validate_request + before_action :validate_chaos_secret, unless: :development? + before_action :request_start_time def leakmem - memory_mb = (params[:memory_mb]&.to_i || 100) - duration_s = (params[:duration_s]&.to_i || 30).seconds - - start = Time.now retainer = [] # Add `n` 1mb chunks of memory to the retainer array memory_mb.times { retainer << "x" * 1.megabyte } - duration_taken = (Time.now - start).seconds - Kernel.sleep duration_s - duration_taken if duration_s > duration_taken + Kernel.sleep(duration_left) render plain: "OK" end - def cpuspin - duration_s = (params[:duration_s]&.to_i || 30).seconds - end_time = Time.now + duration_s.seconds - - rand while Time.now < end_time + def cpu_spin + rand while Time.now < expected_end_time render plain: "OK" end + def db_spin + while Time.now < expected_end_time + ActiveRecord::Base.connection.execute("SELECT 1") + + end_interval_time = Time.now + [duration_s, interval_s].min + rand while Time.now < end_interval_time + end + end + def sleep - duration_s = (params[:duration_s]&.to_i || 30).seconds - Kernel.sleep duration_s + Kernel.sleep(duration_left) render plain: "OK" end @@ -40,17 +41,57 @@ class ChaosController < ActionController::Base private - def validate_request - secret = ENV['GITLAB_CHAOS_SECRET'] - # GITLAB_CHAOS_SECRET is required unless you're running in Development mode - if !secret && !Rails.env.development? - render plain: "chaos misconfigured: please configure GITLAB_CHAOS_SECRET when using GITLAB_ENABLE_CHAOS_ENDPOINTS outside of a development environment", status: :internal_server_error - end + def request_start_time + @start_time ||= Time.now + end + + def expected_end_time + request_start_time + duration_s + end - return unless secret + def duration_left + # returns 0 if over time + [expected_end_time - Time.now, 0].max + end + + def validate_chaos_secret + unless chaos_secret_configured + render plain: "chaos misconfigured: please configure GITLAB_CHAOS_SECRET", + status: :internal_server_error + return + end - unless request.headers["HTTP_X_CHAOS_SECRET"] == secret - render plain: "To experience chaos, please set X-Chaos-Secret header", status: :unauthorized + unless Devise.secure_compare(chaos_secret_configured, chaos_secret_request) + render plain: "To experience chaos, please set a valid `X-Chaos-Secret` header or `token` param", + status: :unauthorized + return end end + + def chaos_secret_configured + ENV['GITLAB_CHAOS_SECRET'] + end + + def chaos_secret_request + request.headers["HTTP_X_CHAOS_SECRET"] || params[:token] + end + + def interval_s + interval_s = params[:interval_s] || 1 + interval_s.to_f.seconds + end + + def duration_s + duration_s = params[:duration_s] || 30 + duration_s.to_i.seconds + end + + def memory_mb + memory_mb = params[:memory_mb] || 100 + memory_mb.to_i + end + + def development? + Rails.env.development? + end end diff --git a/app/helpers/dropdowns_helper.rb b/app/helpers/dropdowns_helper.rb index 64c5fae7d96..fd94f07cc2c 100644 --- a/app/helpers/dropdowns_helper.rb +++ b/app/helpers/dropdowns_helper.rb @@ -46,7 +46,7 @@ module DropdownsHelper def dropdown_toggle(toggle_text, data_attr, options = {}) default_label = data_attr[:default_label] - content_tag(:button, disabled: options[:disabled], class: "dropdown-menu-toggle #{options[:toggle_class] if options.key?(:toggle_class)}", id: (options[:id] if options.key?(:id)), type: "button", data: data_attr) do + content_tag(:button, disabled: options[:disabled], class: "dropdown-menu-toggle btn #{options[:toggle_class] if options.key?(:toggle_class)}", id: (options[:id] if options.key?(:id)), type: "button", data: data_attr) do output = content_tag(:span, toggle_text, class: "dropdown-toggle-text #{'is-default' if toggle_text == default_label}") output << icon('chevron-down') output.html_safe diff --git a/app/helpers/tree_helper.rb b/app/helpers/tree_helper.rb index 4690b6ffbe1..a3575462de0 100644 --- a/app/helpers/tree_helper.rb +++ b/app/helpers/tree_helper.rb @@ -147,4 +147,11 @@ module TreeHelper def relative_url_root Gitlab.config.gitlab.relative_url_root.presence || '/' end + + # project and path are used on the EE version + def tree_content_data(logs_path, project, path) + { + "logs-path" => logs_path + } + end end diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 42203a5f214..9713e79f525 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -87,6 +87,16 @@ module CacheMarkdownField __send__(cached_markdown_fields.html_field(markdown_field)) # rubocop:disable GitlabSecurity/PublicSend end + # Updates the markdown cache if necessary, then returns the field + # Unlike `cached_html_for` it returns `nil` if the field does not exist + def updated_cached_html_for(markdown_field) + return unless cached_markdown_fields.markdown_fields.include?(markdown_field) + + refresh_markdown_cache if attribute_invalidated?(cached_markdown_fields.html_field(markdown_field)) + + cached_html_for(markdown_field) + end + def latest_cached_markdown_version @latest_cached_markdown_version ||= (Gitlab::MarkdownCache::CACHE_COMMONMARK_VERSION << 16) | local_version end @@ -139,8 +149,9 @@ module CacheMarkdownField # The HTML becomes invalid if any dependent fields change. For now, assume # author and project invalidate the cache in all circumstances. define_method(invalidation_method) do - invalidations = changed_markdown_fields & [markdown_field.to_s, *INVALIDATED_BY] - invalidations.delete(markdown_field.to_s) if changed_markdown_fields.include?("#{markdown_field}_html") + changed_fields = changed_attributes.keys + invalidations = changed_fields & [markdown_field.to_s, *INVALIDATED_BY] + invalidations.delete(markdown_field.to_s) if changed_fields.include?("#{markdown_field}_html") !invalidations.empty? || !cached_html_up_to_date?(markdown_field) end end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index e1d5ce7f7d4..91dda803031 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -59,6 +59,7 @@ module Ci variables.append(key: 'CI', value: 'true') variables.append(key: 'GITLAB_CI', value: 'true') variables.append(key: 'GITLAB_FEATURES', value: project.licensed_features.join(',')) + variables.append(key: 'CI_SERVER_HOST', value: Gitlab.config.gitlab.host) variables.append(key: 'CI_SERVER_NAME', value: 'GitLab') variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION) variables.append(key: 'CI_SERVER_VERSION_MAJOR', value: Gitlab.version_info.major.to_s) diff --git a/app/models/concerns/from_union.rb b/app/models/concerns/from_union.rb index 9b8595b1211..e28dee34815 100644 --- a/app/models/concerns/from_union.rb +++ b/app/models/concerns/from_union.rb @@ -40,11 +40,7 @@ module FromUnion .new(members, remove_duplicates: remove_duplicates) .to_sql - # This pattern is necessary as a bug in Rails 4 can cause the use of - # `from("string here").includes(:foo)` to break ActiveRecord. This is - # fixed in https://github.com/rails/rails/pull/25374, which is released as - # part of Rails 5. - from([Arel.sql("(#{union}) #{alias_as}")]) + from(Arel.sql("(#{union}) #{alias_as}")) end # rubocop: enable Gitlab/Union end diff --git a/app/models/concerns/mentionable.rb b/app/models/concerns/mentionable.rb index 0d88b34fb48..2f3f9b399d9 100644 --- a/app/models/concerns/mentionable.rb +++ b/app/models/concerns/mentionable.rb @@ -63,6 +63,9 @@ module Mentionable skip_project_check: skip_project_check? ).merge(mentionable_params) + cached_html = self.try(:updated_cached_html_for, attr.to_sym) + options[:rendered] = cached_html if cached_html + extractor.analyze(text, options) end diff --git a/app/services/boards/issues/move_service.rb b/app/services/boards/issues/move_service.rb index e27d34dbcab..755d723b9a0 100644 --- a/app/services/boards/issues/move_service.rb +++ b/app/services/boards/issues/move_service.rb @@ -4,14 +4,37 @@ module Boards module Issues class MoveService < Boards::BaseService def execute(issue) - return false unless can?(current_user, :update_issue, issue) - return false if issue_params(issue).empty? + issue_modification_params = issue_params(issue) + return false if issue_modification_params.empty? - update(issue) + move_single_issue(issue, issue_modification_params) + end + + def execute_multiple(issues) + return false if issues.empty? + + last_inserted_issue_id = nil + issues.map do |issue| + issue_modification_params = issue_params(issue) + next if issue_modification_params.empty? + + if last_inserted_issue_id + issue_modification_params[:move_between_ids] = move_between_ids({ move_after_id: nil, move_before_id: last_inserted_issue_id }) + end + + last_inserted_issue_id = issue.id + move_single_issue(issue, issue_modification_params) + end.all? end private + def move_single_issue(issue, issue_modification_params) + return false unless can?(current_user, :update_issue, issue) + + update(issue, issue_modification_params) + end + def board @board ||= parent.boards.find(params[:board_id]) end @@ -33,8 +56,8 @@ module Boards end # rubocop: enable CodeReuse/ActiveRecord - def update(issue) - ::Issues::UpdateService.new(issue.project, current_user, issue_params(issue)).execute(issue) + def update(issue, issue_modification_params) + ::Issues::UpdateService.new(issue.project, current_user, issue_modification_params).execute(issue) end def issue_params(issue) @@ -48,6 +71,7 @@ module Boards ) end + move_between_ids = move_between_ids(params) if move_between_ids attrs[:move_between_ids] = move_between_ids attrs[:board_group_id] = board.group&.id @@ -78,8 +102,8 @@ module Boards end # rubocop: enable CodeReuse/ActiveRecord - def move_between_ids - ids = [params[:move_after_id], params[:move_before_id]] + def move_between_ids(move_params) + ids = [move_params[:move_after_id], move_params[:move_before_id]] .map(&:to_i) .map { |m| m.positive? ? m : nil } diff --git a/app/services/clusters/gcp/kubernetes.rb b/app/services/clusters/gcp/kubernetes.rb index 90ed529670c..85711764785 100644 --- a/app/services/clusters/gcp/kubernetes.rb +++ b/app/services/clusters/gcp/kubernetes.rb @@ -9,6 +9,8 @@ module Clusters GITLAB_CLUSTER_ROLE_BINDING_NAME = 'gitlab-admin' GITLAB_CLUSTER_ROLE_NAME = 'cluster-admin' PROJECT_CLUSTER_ROLE_NAME = 'edit' + GITLAB_KNATIVE_SERVING_ROLE_NAME = 'gitlab-knative-serving-role' + GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME = 'gitlab-knative-serving-rolebinding' end end end diff --git a/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb b/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb index 49e766cbf13..7c5450dbcd6 100644 --- a/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb +++ b/app/services/clusters/gcp/kubernetes/create_or_update_service_account_service.rb @@ -41,7 +41,15 @@ module Clusters kubeclient.create_or_update_service_account(service_account_resource) kubeclient.create_or_update_secret(service_account_token_resource) - create_role_or_cluster_role_binding if rbac + + return unless rbac + + create_role_or_cluster_role_binding + + return unless namespace_creator + + create_or_update_knative_serving_role + create_or_update_knative_serving_role_binding end private @@ -63,6 +71,14 @@ module Clusters end end + def create_or_update_knative_serving_role + kubeclient.update_role(knative_serving_role_resource) + end + + def create_or_update_knative_serving_role_binding + kubeclient.update_role_binding(knative_serving_role_binding_resource) + end + def service_account_resource Gitlab::Kubernetes::ServiceAccount.new( service_account_name, @@ -92,6 +108,29 @@ module Clusters Gitlab::Kubernetes::RoleBinding.new( name: role_binding_name, role_name: Clusters::Gcp::Kubernetes::PROJECT_CLUSTER_ROLE_NAME, + role_kind: :ClusterRole, + namespace: service_account_namespace, + service_account_name: service_account_name + ).generate + end + + def knative_serving_role_resource + Gitlab::Kubernetes::Role.new( + name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, + namespace: service_account_namespace, + rules: [{ + apiGroups: %w(serving.knative.dev), + resources: %w(configurations configurationgenerations routes revisions revisionuids autoscalers services), + verbs: %w(get list create update delete patch watch) + }] + ).generate + end + + def knative_serving_role_binding_resource + Gitlab::Kubernetes::RoleBinding.new( + name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME, + role_name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, + role_kind: :Role, namespace: service_account_namespace, service_account_name: service_account_name ).generate diff --git a/app/views/ci/status/_icon.html.haml b/app/views/ci/status/_icon.html.haml index f38bdb2e5ed..1249b98221f 100644 --- a/app/views/ci/status/_icon.html.haml +++ b/app/views/ci/status/_icon.html.haml @@ -1,9 +1,10 @@ -- status = local_assigns.fetch(:status) -- size = local_assigns.fetch(:size, 16) -- type = local_assigns.fetch(:type, 'pipeline') -- tooltip_placement = local_assigns.fetch(:tooltip_placement, "left") -- path = local_assigns.fetch(:path, status.has_details? ? status.details_path : nil) -- css_classes = "ci-status-link ci-status-icon ci-status-icon-#{status.group} has-tooltip" +- status = local_assigns.fetch(:status) +- size = local_assigns.fetch(:size, 16) +- type = local_assigns.fetch(:type, 'pipeline') +- tooltip_placement = local_assigns.fetch(:tooltip_placement, "left") +- path = local_assigns.fetch(:path, status.has_details? ? status.details_path : nil) +- option_css_classes = local_assigns.fetch(:option_css_classes, '') +- css_classes = "ci-status-link ci-status-icon ci-status-icon-#{status.group} has-tooltip #{option_css_classes}" - title = s_("PipelineStatusTooltip|Pipeline: %{ci_status}") % {ci_status: status.label} - if type == 'commit' - title = s_("PipelineStatusTooltip|Commit: %{ci_status}") % {ci_status: status.label} diff --git a/app/views/projects/environments/show.html.haml b/app/views/projects/environments/show.html.haml index c13a47b0b09..6100fd3ad37 100644 --- a/app/views/projects/environments/show.html.haml +++ b/app/views/projects/environments/show.html.haml @@ -3,6 +3,9 @@ - breadcrumb_title @environment.name - page_title _("Environments") +- content_for :page_specific_javascripts do + = stylesheet_link_tag 'page_bundles/xterm' + %div{ class: container_class } - if can?(current_user, :stop_environment, @environment) #stop-environment-modal.modal.fade{ tabindex: -1 } diff --git a/app/views/projects/issues/import_csv/_button.html.haml b/app/views/projects/issues/import_csv/_button.html.haml index acc2c50294f..8442a53ed61 100644 --- a/app/views/projects/issues/import_csv/_button.html.haml +++ b/app/views/projects/issues/import_csv/_button.html.haml @@ -1,6 +1,6 @@ - type = local_assigns.fetch(:type, :icon) -%button.csv-import-button.btn{ title: _('Import CSV'), class: ('has-tooltip' if type == :icon), +%button.csv-import-button.btn.btn-svg{ title: _('Import CSV'), class: ('has-tooltip' if type == :icon), data: { toggle: 'modal', target: '.issues-import-modal' } } - if type == :icon = sprite_icon('upload') diff --git a/app/views/projects/merge_requests/_merge_request.html.haml b/app/views/projects/merge_requests/_merge_request.html.haml index eb516684e52..dee3931ff04 100644 --- a/app/views/projects/merge_requests/_merge_request.html.haml +++ b/app/views/projects/merge_requests/_merge_request.html.haml @@ -38,7 +38,7 @@ = link_to_label(label, type: :merge_request, css_class: 'label-link') .issuable-meta - %ul.controls + %ul.controls.d-flex.align-items-end - if merge_request.merged? %li.issuable-status.d-none.d-sm-inline-block MERGED @@ -47,14 +47,14 @@ = icon('ban') CLOSED - if can?(current_user, :read_pipeline, merge_request.head_pipeline) - %li.issuable-pipeline-status.d-none.d-sm-inline-block - = render 'ci/status/icon', status: merge_request.head_pipeline.detailed_status(current_user) + %li.issuable-pipeline-status.d-none.d-sm-flex + = render 'ci/status/icon', status: merge_request.head_pipeline.detailed_status(current_user), option_css_classes: 'd-flex' - if merge_request.open? && merge_request.broken? - %li.issuable-pipeline-broken.d-none.d-sm-inline-block + %li.issuable-pipeline-broken.d-none.d-sm-flex = link_to merge_request_path(merge_request), class: "has-tooltip", title: _('Cannot be merged automatically') do = icon('exclamation-triangle') - if merge_request.assignees.any? - %li + %li.d-flex = render 'shared/issuable/assignees', project: merge_request.project, issuable: merge_request = render_if_exists 'projects/merge_requests/approvals_count', merge_request: merge_request diff --git a/app/views/projects/settings/operations/_external_dashboard.html.haml b/app/views/projects/settings/operations/_external_dashboard.html.haml index a124283921d..08d50a336fd 100644 --- a/app/views/projects/settings/operations/_external_dashboard.html.haml +++ b/app/views/projects/settings/operations/_external_dashboard.html.haml @@ -1,3 +1,3 @@ .js-operation-settings{ data: { operations_settings_endpoint: project_settings_operations_path(@project), external_dashboard: { url: metrics_external_dashboard_url, - help_page_path: help_page_path('user/project/operations/link_to_external_dashboard') } } } + help_page_path: help_page_path('user/project/operations/linking_to_an_external_dashboard') } } } diff --git a/app/views/projects/tree/_tree_content.html.haml b/app/views/projects/tree/_tree_content.html.haml index 889a13339fd..cb459b031fc 100644 --- a/app/views/projects/tree/_tree_content.html.haml +++ b/app/views/projects/tree/_tree_content.html.haml @@ -1,4 +1,4 @@ -.tree-content-holder.js-tree-content{ 'data-logs-path': @logs_path } +.tree-content-holder.js-tree-content{ data: tree_content_data(@logs_path, @project, @path) } .table-holder.bordered-box %table.table#tree-slider{ class: "table_#{@hex_path} tree-table qa-file-tree" } %thead diff --git a/app/views/shared/issuable/_feed_buttons.html.haml b/app/views/shared/issuable/_feed_buttons.html.haml index 83f60fa6fe2..c9506a3295c 100644 --- a/app/views/shared/issuable/_feed_buttons.html.haml +++ b/app/views/shared/issuable/_feed_buttons.html.haml @@ -1,4 +1,4 @@ -= link_to safe_params.merge(rss_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: _('Subscribe to RSS feed') do - = icon('rss') -= link_to safe_params.merge(calendar_url_options), class: 'btn has-tooltip', data: { container: 'body' }, title: _('Subscribe to calendar') do - = custom_icon('icon_calendar') += link_to safe_params.merge(rss_url_options), class: 'btn btn-svg has-tooltip js-rss-button', data: { container: 'body' }, title: _('Subscribe to RSS feed') do + = sprite_icon('rss') += link_to safe_params.merge(calendar_url_options), class: 'btn btn-svg has-tooltip', data: { container: 'body' }, title: _('Subscribe to calendar') do + = sprite_icon('calendar') diff --git a/app/views/shared/projects/_project.html.haml b/app/views/shared/projects/_project.html.haml index 90fb067e75d..71bd9320593 100644 --- a/app/views/shared/projects/_project.html.haml +++ b/app/views/shared/projects/_project.html.haml @@ -58,7 +58,7 @@ .description.d-none.d-sm-block.append-right-default = markdown_field(project, :description) - .controls.d-flex.flex-sm-column.align-items-center.align-items-sm-end.flex-wrap.flex-shrink-0{ class: css_controls_class } + .controls.d-flex.flex-sm-column.align-items-center.align-items-sm-end.flex-wrap.flex-shrink-0.text-secondary{ class: css_controls_class } .icon-container.d-flex.align-items-center - if project.archived %span.d-flex.icon-wrapper.badge.badge-warning archived @@ -1,63 +1,21 @@ #!/bin/sh +set -e + cd $(dirname $0)/.. app_root=$(pwd) -# Switch to experimental PUMA configuration -if [ -n "${EXPERIMENTAL_PUMA}" ]; then - exec bin/web_puma "$@" -fi - -unicorn_pidfile="$app_root/tmp/pids/unicorn.pid" -unicorn_config="$app_root/config/unicorn.rb" -unicorn_cmd="bundle exec unicorn_rails -c $unicorn_config -E $RAILS_ENV" - -get_unicorn_pid() -{ - local pid=$(cat $unicorn_pidfile) - if [ -z "$pid" ] ; then - echo "Could not find a PID in $unicorn_pidfile" - exit 1 - fi - unicorn_pid=$pid -} - -start() -{ - exec $unicorn_cmd -D -} - -start_foreground() -{ - exec $unicorn_cmd -} - -stop() -{ - get_unicorn_pid - kill -QUIT $unicorn_pid -} +case "$USE_WEB_SERVER" in + puma|"") # and the "" defines default + exec bin/web_puma "$@" + ;; -reload() -{ - get_unicorn_pid - kill -USR2 $unicorn_pid -} + unicorn) + exec bin/web_unicorn "$@" + ;; -case "$1" in - start) - start - ;; - start_foreground) - start_foreground - ;; - stop) - stop - ;; - reload) - reload - ;; - *) - echo "Usage: RAILS_ENV=your_env $0 {start|stop|reload}" - ;; + *) + echo "Unkown web server used by USE_WEB_SERVER: $USE_WEB_SERVER." + exit 1 + ;; esac diff --git a/bin/web_unicorn b/bin/web_unicorn new file mode 100755 index 00000000000..ecd0bbd10b0 --- /dev/null +++ b/bin/web_unicorn @@ -0,0 +1,58 @@ +#!/bin/sh + +cd $(dirname $0)/.. +app_root=$(pwd) + +unicorn_pidfile="$app_root/tmp/pids/unicorn.pid" +unicorn_config="$app_root/config/unicorn.rb" +unicorn_cmd="bundle exec unicorn_rails -c $unicorn_config -E $RAILS_ENV" + +get_unicorn_pid() +{ + local pid=$(cat $unicorn_pidfile) + if [ -z "$pid" ] ; then + echo "Could not find a PID in $unicorn_pidfile" + exit 1 + fi + unicorn_pid=$pid +} + +start() +{ + exec $unicorn_cmd -D +} + +start_foreground() +{ + exec $unicorn_cmd +} + +stop() +{ + get_unicorn_pid + kill -QUIT $unicorn_pid +} + +reload() +{ + get_unicorn_pid + kill -USR2 $unicorn_pid +} + +case "$1" in + start) + start + ;; + start_foreground) + start_foreground + ;; + stop) + stop + ;; + reload) + reload + ;; + *) + echo "Usage: RAILS_ENV=your_env $0 {start|stop|reload}" + ;; +esac diff --git a/changelogs/unreleased/35757-move-issues-in-boards-pderichs.yml b/changelogs/unreleased/35757-move-issues-in-boards-pderichs.yml new file mode 100644 index 00000000000..ae50350f70d --- /dev/null +++ b/changelogs/unreleased/35757-move-issues-in-boards-pderichs.yml @@ -0,0 +1,5 @@ +--- +title: Add endpoint to move multiple issues in boards +merge_request: 30216 +author: +type: added diff --git a/changelogs/unreleased/61145-fix-button-dimensions.yml b/changelogs/unreleased/61145-fix-button-dimensions.yml new file mode 100644 index 00000000000..8f209ceaa8e --- /dev/null +++ b/changelogs/unreleased/61145-fix-button-dimensions.yml @@ -0,0 +1,5 @@ +--- +title: Updating button dimensions according to design spec +merge_request: 28545 +author: +type: fixed diff --git a/changelogs/unreleased/62088-search-back.yml b/changelogs/unreleased/62088-search-back.yml new file mode 100644 index 00000000000..4758e927880 --- /dev/null +++ b/changelogs/unreleased/62088-search-back.yml @@ -0,0 +1,5 @@ +--- +title: Fixed back navigation for projects filter +merge_request: 30373 +author: +type: fixed diff --git a/changelogs/unreleased/63667-hashed-storage-migration-count-correctly.yml b/changelogs/unreleased/63667-hashed-storage-migration-count-correctly.yml new file mode 100644 index 00000000000..ead79be2505 --- /dev/null +++ b/changelogs/unreleased/63667-hashed-storage-migration-count-correctly.yml @@ -0,0 +1,5 @@ +--- +title: Display the correct amount of projects being migrated/rolled-back to Hashed Storage when specifying ranges +merge_request: 29996 +author: +type: fixed diff --git a/changelogs/unreleased/63691-fix-doc-link.yml b/changelogs/unreleased/63691-fix-doc-link.yml new file mode 100644 index 00000000000..e63756e8227 --- /dev/null +++ b/changelogs/unreleased/63691-fix-doc-link.yml @@ -0,0 +1,5 @@ +--- +title: Correct link to docs for External Dashboard +merge_request: 30019 +author: +type: fixed diff --git a/changelogs/unreleased/64091-fix-broken-terminal.yml b/changelogs/unreleased/64091-fix-broken-terminal.yml new file mode 100644 index 00000000000..156f6de5008 --- /dev/null +++ b/changelogs/unreleased/64091-fix-broken-terminal.yml @@ -0,0 +1,5 @@ +--- +title: Fix environments broken terminal +merge_request: 30401 +author: +type: fixed diff --git a/changelogs/unreleased/64161-gitlab-fqdn.yml b/changelogs/unreleased/64161-gitlab-fqdn.yml new file mode 100644 index 00000000000..2946be37caa --- /dev/null +++ b/changelogs/unreleased/64161-gitlab-fqdn.yml @@ -0,0 +1,5 @@ +--- +title: Add CI variable to provide GitLab HOST +merge_request: 30417 +author: +type: added diff --git a/changelogs/unreleased/64314-ci-icon.yml b/changelogs/unreleased/64314-ci-icon.yml new file mode 100644 index 00000000000..8a550b6fa5b --- /dev/null +++ b/changelogs/unreleased/64314-ci-icon.yml @@ -0,0 +1,5 @@ +--- +title: Aligns CI icon in Merge Request dashboard +merge_request: 30558 +author: +type: fixed diff --git a/changelogs/unreleased/gitaly-version-v1-53-0.yml b/changelogs/unreleased/gitaly-version-v1-53-0.yml new file mode 100644 index 00000000000..7d3e5ce3cb3 --- /dev/null +++ b/changelogs/unreleased/gitaly-version-v1-53-0.yml @@ -0,0 +1,5 @@ +--- +title: Upgrade to Gitaly v1.53.0 +merge_request: 30614 +author: +type: changed diff --git a/changelogs/unreleased/hfy-apply-knative-cluster-role-on-service-account-creation.yml b/changelogs/unreleased/hfy-apply-knative-cluster-role-on-service-account-creation.yml new file mode 100644 index 00000000000..958334cc28e --- /dev/null +++ b/changelogs/unreleased/hfy-apply-knative-cluster-role-on-service-account-creation.yml @@ -0,0 +1,5 @@ +--- +title: Create Knative role and binding with service account +merge_request: 30235 +author: +type: changed diff --git a/changelogs/unreleased/mw-project-list-color-fix.yml b/changelogs/unreleased/mw-project-list-color-fix.yml new file mode 100644 index 00000000000..6f8b2742ec6 --- /dev/null +++ b/changelogs/unreleased/mw-project-list-color-fix.yml @@ -0,0 +1,5 @@ +--- +title: Add text-secondary to controls in project list +merge_request: 30567 +author: +type: fixed diff --git a/changelogs/unreleased/registry-fix-multi-delete-modal.yml b/changelogs/unreleased/registry-fix-multi-delete-modal.yml new file mode 100644 index 00000000000..94a2df7a7e7 --- /dev/null +++ b/changelogs/unreleased/registry-fix-multi-delete-modal.yml @@ -0,0 +1,5 @@ +--- +title: Prevent multiple confirmation modals from opening when deleting a repository +merge_request: 30532 +author: +type: fixed diff --git a/changelogs/unreleased/z-index-tools.yml b/changelogs/unreleased/z-index-tools.yml new file mode 100644 index 00000000000..1102612670b --- /dev/null +++ b/changelogs/unreleased/z-index-tools.yml @@ -0,0 +1,5 @@ +--- +title: 'Review Tools: Add large z-index to toolbar' +merge_request: 30583 +author: +type: fixed diff --git a/config/initializers/rack_timeout.rb b/config/initializers/rack_timeout.rb index 58f46b55725..246cf3482a4 100644 --- a/config/initializers/rack_timeout.rb +++ b/config/initializers/rack_timeout.rb @@ -14,8 +14,8 @@ if defined?(::Puma) && !Rails.env.test? Gitlab::Application.configure do |config| config.middleware.insert_before(Rack::Runtime, Rack::Timeout, - service_timeout: 60, - wait_timeout: 90) + service_timeout: ENV.fetch('GITLAB_RAILS_RACK_TIMEOUT', 60).to_i, + wait_timeout: ENV.fetch('GITLAB_RAILS_WAIT_TIMEOUT', 90).to_i) end observer = Gitlab::Cluster::RackTimeoutObserver.new diff --git a/config/routes.rb b/config/routes.rb index a42fc037227..50bb092ec2e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -82,7 +82,11 @@ Rails.application.routes.draw do resources :issues, only: [:index, :create, :update] end - resources :issues, module: :boards, only: [:index, :update] + resources :issues, module: :boards, only: [:index, :update] do + collection do + put :bulk_move, format: :json + end + end Gitlab.ee do resources :users, module: :boards, only: [:index] @@ -106,11 +110,14 @@ Rails.application.routes.draw do draw :jira_connect end - if ENV['GITLAB_ENABLE_CHAOS_ENDPOINTS'] - get '/chaos/leakmem' => 'chaos#leakmem' - get '/chaos/cpuspin' => 'chaos#cpuspin' - get '/chaos/sleep' => 'chaos#sleep' - get '/chaos/kill' => 'chaos#kill' + if ENV['GITLAB_CHAOS_SECRET'] || Rails.env.development? + resource :chaos, only: [] do + get :leakmem + get :cpu_spin + get :db_spin + get :sleep + get :kill + end end end diff --git a/config/webpack.config.js b/config/webpack.config.js index a81590e8b8e..cd793743eb7 100644 --- a/config/webpack.config.js +++ b/config/webpack.config.js @@ -344,6 +344,8 @@ module.exports = { devtool: NO_SOURCEMAPS ? false : devtool, - // sqljs requires fs - node: { fs: 'empty' }, + node: { + fs: 'empty', // sqljs requires fs + setImmediate: false, + }, }; diff --git a/doc/administration/auth/authentiq.md b/doc/administration/auth/authentiq.md index 726622d8599..835c97c0288 100644 --- a/doc/administration/auth/authentiq.md +++ b/doc/administration/auth/authentiq.md @@ -8,47 +8,48 @@ Authentiq will generate a Client ID and the accompanying Client Secret for you t 1. On your GitLab server, open the configuration file: - For omnibus installation - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + For omnibus installation - For installations from source: + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - ```sh - sudo -u git -H editor /home/git/gitlab/config/gitlab.yml - ``` + For installations from source: + + ```sh + sudo -u git -H editor /home/git/gitlab/config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings to enable single sign-on and add Authentiq as an OAuth provider. 1. Add the provider configuration for Authentiq: - For Omnibus packages: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "authentiq", - "app_id" => "YOUR_CLIENT_ID", - "app_secret" => "YOUR_CLIENT_SECRET", - "args" => { - "scope": 'aq:name email~rs address aq:push' - } - } - ] - ``` - - For installations from source: - - ```yaml - - { name: 'authentiq', - app_id: 'YOUR_CLIENT_ID', - app_secret: 'YOUR_CLIENT_SECRET', - args: { - scope: 'aq:name email~rs address aq:push' - } - } - ``` + For Omnibus packages: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "authentiq", + "app_id" => "YOUR_CLIENT_ID", + "app_secret" => "YOUR_CLIENT_SECRET", + "args" => { + "scope": 'aq:name email~rs address aq:push' + } + } + ] + ``` + + For installations from source: + + ```yaml + - { name: 'authentiq', + app_id: 'YOUR_CLIENT_ID', + app_secret: 'YOUR_CLIENT_SECRET', + args: { + scope: 'aq:name email~rs address aq:push' + } + } + ``` 1. The `scope` is set to request the user's name, email (required and signed), and permission to send push notifications to sign in on subsequent visits. See [OmniAuth Authentiq strategy](https://github.com/AuthentiqID/omniauth-authentiq/wiki/Scopes,-callback-url-configuration-and-responses) for more information on scopes and modifiers. diff --git a/doc/administration/auth/crowd.md b/doc/administration/auth/crowd.md index 6db74958d5a..86c7bad2ebf 100644 --- a/doc/administration/auth/crowd.md +++ b/doc/administration/auth/crowd.md @@ -6,55 +6,56 @@ 1. Go through the 'Add application' steps, entering the appropriate details. The screenshot below shows an example configuration. -  +  ## Configure GitLab 1. On your GitLab server, open the configuration file. - **Omnibus:** + **Omnibus:** - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - **Source:** + **Source:** - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. 1. Add the provider configuration: - **Omnibus:** - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "crowd", - "args" => { - "crowd_server_url" => "CROWD_SERVER_URL", - "application_name" => "YOUR_APP_NAME", - "application_password" => "YOUR_APP_PASSWORD" - } - } - ] - ``` - - **Source:** - - ``` - - { name: 'crowd', - args: { - crowd_server_url: 'CROWD_SERVER_URL', - application_name: 'YOUR_APP_NAME', - application_password: 'YOUR_APP_PASSWORD' } } - ``` + **Omnibus:** + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "crowd", + "args" => { + "crowd_server_url" => "CROWD_SERVER_URL", + "application_name" => "YOUR_APP_NAME", + "application_password" => "YOUR_APP_PASSWORD" + } + } + ] + ``` + + **Source:** + + ``` + - { name: 'crowd', + args: { + crowd_server_url: 'CROWD_SERVER_URL', + application_name: 'YOUR_APP_NAME', + application_password: 'YOUR_APP_PASSWORD' } } + ``` + 1. Change `CROWD_SERVER_URL` to the URL of your Crowd server. 1. Change `YOUR_APP_NAME` to the application name from Crowd applications page. 1. Change `YOUR_APP_PASSWORD` to the application password you've set. @@ -77,4 +78,4 @@ could not authorize you from Crowd because invalid credentials Please make sure the Crowd users who need to login to GitLab are authorized to [the application](#configure-a-new-crowd-application) in the step of **Authorisation**. This could be verified by try "Authentication test" for Crowd as of 2.11. -
\ No newline at end of file + diff --git a/doc/administration/auth/google_secure_ldap.md b/doc/administration/auth/google_secure_ldap.md index 1db5bb4bc3f..0e6d7ff1df1 100644 --- a/doc/administration/auth/google_secure_ldap.md +++ b/doc/administration/auth/google_secure_ldap.md @@ -66,7 +66,7 @@ values obtained during the LDAP client configuration earlier: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby + ```ruby gitlab_rails['ldap_enabled'] = true gitlab_rails['ldap_servers'] = YAML.load <<-EOS # remember to close this block with 'EOS' below main: # 'main' is the GitLab 'provider ID' of this LDAP server @@ -127,7 +127,7 @@ values obtained during the LDAP client configuration earlier: AcZSFJQjdg5BTyvdEDhaYUKGdRw= -----END PRIVATE KEY----- EOS - ``` + ``` 1. Save the file and [reconfigure] GitLab for the changes to take effect. @@ -137,7 +137,7 @@ values obtained during the LDAP client configuration earlier: 1. Edit `config/gitlab.yml`: - ```yaml + ```yaml ldap: enabled: true servers: diff --git a/doc/administration/auth/jwt.md b/doc/administration/auth/jwt.md index 497298503ad..7db22bdd5df 100644 --- a/doc/administration/auth/jwt.md +++ b/doc/administration/auth/jwt.md @@ -3,65 +3,65 @@ To enable the JWT OmniAuth provider, you must register your application with JWT. JWT will provide you with a secret key for you to use. -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For Omnibus GitLab: + For Omnibus GitLab: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration. +1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. +1. Add the provider configuration. - For Omnibus GitLab: + For Omnibus GitLab: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { name: 'jwt', - args: { - secret: 'YOUR_APP_SECRET', - algorithm: 'HS256', # Supported algorithms: 'RS256', 'RS384', 'RS512', 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512' - uid_claim: 'email', - required_claims: ['name', 'email'], - info_maps: { name: 'name', email: 'email' }, - auth_url: 'https://example.com/', - valid_within: 3600 # 1 hour - } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { name: 'jwt', + args: { + secret: 'YOUR_APP_SECRET', + algorithm: 'HS256', # Supported algorithms: 'RS256', 'RS384', 'RS512', 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512' + uid_claim: 'email', + required_claims: ['name', 'email'], + info_maps: { name: 'name', email: 'email' }, + auth_url: 'https://example.com/', + valid_within: 3600 # 1 hour + } + } + ] + ``` - For installation from source: + For installation from source: - ``` - - { name: 'jwt', - args: { - secret: 'YOUR_APP_SECRET', - algorithm: 'HS256', # Supported algorithms: 'RS256', 'RS384', 'RS512', 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512' - uid_claim: 'email', - required_claims: ['name', 'email'], - info_map: { name: 'name', email: 'email' }, - auth_url: 'https://example.com/', - valid_within: 3600 # 1 hour - } - } - ``` + ``` + - { name: 'jwt', + args: { + secret: 'YOUR_APP_SECRET', + algorithm: 'HS256', # Supported algorithms: 'RS256', 'RS384', 'RS512', 'ES256', 'ES384', 'ES512', 'HS256', 'HS384', 'HS512' + uid_claim: 'email', + required_claims: ['name', 'email'], + info_map: { name: 'name', email: 'email' }, + auth_url: 'https://example.com/', + valid_within: 3600 # 1 hour + } + } + ``` - NOTE: **Note:** For more information on each configuration option refer to - the [OmniAuth JWT usage documentation](https://github.com/mbleigh/omniauth-jwt#usage). + NOTE: **Note:** For more information on each configuration option refer to + the [OmniAuth JWT usage documentation](https://github.com/mbleigh/omniauth-jwt#usage). -1. Change `YOUR_APP_SECRET` to the client secret and set `auth_url` to your redirect URL. -1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. Change `YOUR_APP_SECRET` to the client secret and set `auth_url` to your redirect URL. +1. Save the configuration file. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a JWT icon below the regular sign in form. Click the icon to begin the authentication process. JWT will ask the user to diff --git a/doc/administration/auth/ldap-ee.md b/doc/administration/auth/ldap-ee.md index 1a8af0827ee..2afac23c20c 100644 --- a/doc/administration/auth/ldap-ee.md +++ b/doc/administration/auth/ldap-ee.md @@ -85,19 +85,19 @@ following. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - ## snip... - ## - ## Base where we can search for groups - ## - ## Ex. ou=groups,dc=gitlab,dc=example - ## - ## - group_base: ou=groups,dc=example,dc=com - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + ## snip... + ## + ## Base where we can search for groups + ## + ## Ex. ou=groups,dc=gitlab,dc=example + ## + ## + group_base: ou=groups,dc=example,dc=com + EOS + ``` 1. [Reconfigure GitLab][reconfigure] for the changes to take effect. @@ -105,14 +105,14 @@ following. 1. Edit `/home/git/gitlab/config/gitlab.yml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - group_base: ou=groups,dc=example,dc=com - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + group_base: ou=groups,dc=example,dc=com + ``` 1. [Restart GitLab][restart] for the changes to take effect. @@ -140,30 +140,30 @@ group, as opposed to the full DN. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - ## snip... - ## - ## Base where we can search for groups - ## - ## Ex. ou=groups,dc=gitlab,dc=example - ## - ## - group_base: ou=groups,dc=example,dc=com - - ## - ## The CN of a group containing GitLab administrators - ## - ## Ex. administrators - ## - ## Note: Not `cn=administrators` or the full DN - ## - ## - admin_group: my_admin_group - - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + ## snip... + ## + ## Base where we can search for groups + ## + ## Ex. ou=groups,dc=gitlab,dc=example + ## + ## + group_base: ou=groups,dc=example,dc=com + + ## + ## The CN of a group containing GitLab administrators + ## + ## Ex. administrators + ## + ## Note: Not `cn=administrators` or the full DN + ## + ## + admin_group: my_admin_group + + EOS + ``` 1. [Reconfigure GitLab][reconfigure] for the changes to take effect. @@ -171,15 +171,15 @@ group, as opposed to the full DN. 1. Edit `/home/git/gitlab/config/gitlab.yml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - group_base: ou=groups,dc=example,dc=com - admin_group: my_admin_group - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + group_base: ou=groups,dc=example,dc=com + admin_group: my_admin_group + ``` 1. [Restart GitLab][restart] for the changes to take effect. @@ -191,7 +191,6 @@ to lock down user abilities to invite new members to a group. When enabled follo 1. Only administrator can manage memberships of any group including access levels. 2. Users are not allowed to share project with other groups or invite members to a project created in a group. - ## Adjusting LDAP user sync schedule > Introduced in GitLab Enterprise Edition Starter. @@ -211,9 +210,9 @@ sync to run once every 12 hours at the top of the hour. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" - ``` + ```ruby + gitlab_rails['ldap_sync_worker_cron'] = "0 */12 * * *" + ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -221,11 +220,11 @@ sync to run once every 12 hours at the top of the hour. 1. Edit `config/gitlab.yaml`: - ```yaml - cron_jobs: - ldap_sync_worker_cron: - "0 */12 * * *" - ``` + ```yaml + cron_jobs: + ldap_sync_worker_cron: + "0 */12 * * *" + ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -252,9 +251,9 @@ sync to run once every 2 hours at the top of the hour. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" - ``` + ```ruby + gitlab_rails['ldap_group_sync_worker_cron'] = "0 */2 * * * *" + ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -262,11 +261,11 @@ sync to run once every 2 hours at the top of the hour. 1. Edit `config/gitlab.yaml`: - ```yaml - cron_jobs: - ldap_group_sync_worker_cron: - "*/30 * * * *" - ``` + ```yaml + cron_jobs: + ldap_group_sync_worker_cron: + "*/30 * * * *" + ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -283,20 +282,20 @@ task. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - ## snip... - ## - ## An array of CNs of groups containing users that should be considered external - ## - ## Ex. ['interns', 'contractors'] - ## - ## Note: Not `cn=interns` or the full DN - ## - external_groups: ['interns', 'contractors'] - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + ## snip... + ## + ## An array of CNs of groups containing users that should be considered external + ## + ## Ex. ['interns', 'contractors'] + ## + ## Note: Not `cn=interns` or the full DN + ## + external_groups: ['interns', 'contractors'] + EOS + ``` 1. [Reconfigure GitLab][reconfigure] for the changes to take effect. @@ -304,14 +303,14 @@ task. 1. Edit `config/gitlab.yaml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - external_groups: ['interns', 'contractors'] - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + external_groups: ['interns', 'contractors'] + ``` 1. [Restart GitLab][restart] for the changes to take effect. @@ -436,66 +435,71 @@ step of the sync. 1. Start a Rails console - ```bash - # For Omnibus installations - sudo gitlab-rails console + ```bash + # For Omnibus installations + sudo gitlab-rails console - # For installations from source - sudo -u git -H bundle exec rails console production - ``` + # For installations from source + sudo -u git -H bundle exec rails console production + ``` 1. Set the log level to debug (only for this session): - ```ruby - Rails.logger.level = Logger::DEBUG - ``` + ```ruby + Rails.logger.level = Logger::DEBUG + ``` + 1. Choose a GitLab group to test with. This group should have an LDAP group link already configured. If the output is `nil`, the group could not be found. If a bunch of group attributes are output, your group was found successfully. - ```ruby - group = Group.find_by(name: 'my_group') + ```ruby + group = Group.find_by(name: 'my_group') + + # Output + => #<Group:0x007fe825196558 id: 1234, name: "my_group"...> + ``` - # Output - => #<Group:0x007fe825196558 id: 1234, name: "my_group"...> - ``` 1. Run a group sync for this particular group. - ```ruby - EE::Gitlab::Auth::LDAP::Sync::Group.execute_all_providers(group) - ``` + ```ruby + EE::Gitlab::Auth::LDAP::Sync::Group.execute_all_providers(group) + ``` + 1. Look through the output of the sync. See [example log output](#example-log-output) below for more information about the output. 1. If you still aren't able to see why the user isn't being added, query the LDAP group directly to see what members are listed. Still in the Rails console, run the following query: - ```ruby - adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider - ldap_group = EE::Gitlab::Auth::LDAP::Group.find_by_cn('group_cn_here', adapter) + ```ruby + adapter = Gitlab::Auth::LDAP::Adapter.new('ldapmain') # If `main` is the LDAP provider + ldap_group = EE::Gitlab::Auth::LDAP::Group.find_by_cn('group_cn_here', adapter) + + # Output + => #<EE::Gitlab::Auth::LDAP::Group:0x007fcbdd0bb6d8 + ``` - # Output - => #<EE::Gitlab::Auth::LDAP::Group:0x007fcbdd0bb6d8 - ``` 1. Query the LDAP group's member DNs and see if the user's DN is in the list. One of the DNs here should match the 'Identifier' from the LDAP identity checked earlier. If it doesn't, the user does not appear to be in the LDAP group. - ```ruby - ldap_group.member_dns + ```ruby + ldap_group.member_dns + + # Output + => ["uid=john,ou=people,dc=example,dc=com", "uid=mary,ou=people,dc=example,dc=com"] + ``` - # Output - => ["uid=john,ou=people,dc=example,dc=com", "uid=mary,ou=people,dc=example,dc=com"] - ``` 1. Some LDAP servers don't store members by DN. Rather, they use UIDs instead. If you didn't see results from the last query, try querying by UIDs instead. - ```ruby - ldap_group.member_uids + ```ruby + ldap_group.member_uids - # Output - => ['john','mary'] - ``` + # Output + => ['john','mary'] + ``` #### Example log output diff --git a/doc/administration/auth/ldap.md b/doc/administration/auth/ldap.md index 2144f5753a8..86e6be5f4fa 100644 --- a/doc/administration/auth/ldap.md +++ b/doc/administration/auth/ldap.md @@ -398,30 +398,30 @@ The `user_filter` DN can contain special characters. For example: - A comma: - ``` - OU=GitLab, Inc,DC=gitlab,DC=com - ``` + ``` + OU=GitLab, Inc,DC=gitlab,DC=com + ``` - Open and close brackets: - ``` - OU=Gitlab (Inc),DC=gitlab,DC=com - ``` + ``` + OU=Gitlab (Inc),DC=gitlab,DC=com + ``` - These characters must be escaped as documented in - [RFC 4515](https://tools.ietf.org/search/rfc4515). + These characters must be escaped as documented in + [RFC 4515](https://tools.ietf.org/search/rfc4515). - Escape commas with `\2C`. For example: - ``` - OU=GitLab\2C Inc,DC=gitlab,DC=com - ``` + ``` + OU=GitLab\2C Inc,DC=gitlab,DC=com + ``` - Escape open and close brackets with `\28` and `\29`, respectively. For example: - ``` - OU=Gitlab \28Inc\29,DC=gitlab,DC=com - ``` + ``` + OU=Gitlab \28Inc\29,DC=gitlab,DC=com + ``` ## Enabling LDAP sign-in for existing GitLab users @@ -445,13 +445,13 @@ the configuration option `lowercase_usernames`. By default, this configuration o 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - # snip... - lowercase_usernames: true - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + # snip... + lowercase_usernames: true + EOS + ``` 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -459,14 +459,14 @@ the configuration option `lowercase_usernames`. By default, this configuration o 1. Edit `config/gitlab.yaml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - lowercase_usernames: true - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + lowercase_usernames: true + ``` 1. [Restart GitLab](../restart_gitlab.md#installations-from-source) for the changes to take effect. @@ -519,13 +519,13 @@ ldapsearch -H ldaps://$host:$port -D "$bind_dn" -y bind_dn_password.txt -b "$ba - Run the following check command to make sure that the LDAP settings are correct and GitLab can see your users: - ```bash - # For Omnibus installations - sudo gitlab-rake gitlab:ldap:check + ```bash + # For Omnibus installations + sudo gitlab-rake gitlab:ldap:check - # For installations from source - sudo -u git -H bundle exec rake gitlab:ldap:check RAILS_ENV=production - ``` + # For installations from source + sudo -u git -H bundle exec rake gitlab:ldap:check RAILS_ENV=production + ``` ### Connection Refused diff --git a/doc/administration/auth/oidc.md b/doc/administration/auth/oidc.md index 6e48add6930..454da8c2866 100644 --- a/doc/administration/auth/oidc.md +++ b/doc/administration/auth/oidc.md @@ -5,76 +5,76 @@ GitLab can use [OpenID Connect](https://openid.net/specs/openid-connect-core-1_0 To enable the OpenID Connect OmniAuth provider, you must register your application with an OpenID Connect provider. The OpenID Connect will provide you with a client details and secret for you to use. -1. On your GitLab server, open the configuration file. - - For Omnibus GitLab: - - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` - - For installations from source: - - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` - - See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. - -1. Add the provider configuration. - - For Omnibus GitLab: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { 'name' => 'openid_connect', - 'label' => '<your_oidc_label>', - 'args' => { - "name' => 'openid_connect', - 'scope' => ['openid','profile'], - 'response_type' => 'code', - 'issuer' => '<your_oidc_url>', - 'discovery' => true, - 'client_auth_method' => 'query', - 'uid_field' => '<uid_field>', - 'client_options' => { - 'identifier' => '<your_oidc_client_id>', - 'secret' => '<your_oidc_client_secret>', - 'redirect_uri' => '<your_gitlab_url>/users/auth/openid_connect/callback' - } - } - } - ] - ``` - - For installation from source: - - ```yaml - - { name: 'openid_connect', - label: '<your_oidc_label>', - args: { - name: 'openid_connect', - scope: ['openid','profile'], - response_type: 'code', - issuer: '<your_oidc_url>', - discovery: true, - client_auth_method: 'query', - uid_field: '<uid_field>', - client_options: { - identifier: '<your_oidc_client_id>', - secret: '<your_oidc_client_secret>', - redirect_uri: '<your_gitlab_url>/users/auth/openid_connect/callback' - } - } - } - ``` - - > **Note:** - > - > - For more information on each configuration option refer to - the [OmniAuth OpenID Connect usage documentation](https://github.com/m0n9oose/omniauth_openid_connect#usage) and - the [OpenID Connect Core 1.0 specification](https://openid.net/specs/openid-connect-core-1_0.html). +1. On your GitLab server, open the configuration file. + + For Omnibus GitLab: + + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` + + For installations from source: + + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` + + See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) for initial settings. + +1. Add the provider configuration. + + For Omnibus GitLab: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { 'name' => 'openid_connect', + 'label' => '<your_oidc_label>', + 'args' => { + "name' => 'openid_connect', + 'scope' => ['openid','profile'], + 'response_type' => 'code', + 'issuer' => '<your_oidc_url>', + 'discovery' => true, + 'client_auth_method' => 'query', + 'uid_field' => '<uid_field>', + 'client_options' => { + 'identifier' => '<your_oidc_client_id>', + 'secret' => '<your_oidc_client_secret>', + 'redirect_uri' => '<your_gitlab_url>/users/auth/openid_connect/callback' + } + } + } + ] + ``` + + For installation from source: + + ```yaml + - { name: 'openid_connect', + label: '<your_oidc_label>', + args: { + name: 'openid_connect', + scope: ['openid','profile'], + response_type: 'code', + issuer: '<your_oidc_url>', + discovery: true, + client_auth_method: 'query', + uid_field: '<uid_field>', + client_options: { + identifier: '<your_oidc_client_id>', + secret: '<your_oidc_client_secret>', + redirect_uri: '<your_gitlab_url>/users/auth/openid_connect/callback' + } + } + } + ``` + + > **Note:** + > + > - For more information on each configuration option refer to + the [OmniAuth OpenID Connect usage documentation](https://github.com/m0n9oose/omniauth_openid_connect#usage) and + the [OpenID Connect Core 1.0 specification](https://openid.net/specs/openid-connect-core-1_0.html). 1. For the configuration above, change the values for the provider to match your OpenID Connect client setup. Use the following as a guide: - `<your_oidc_label>` is the label that will be displayed on the login page. diff --git a/doc/administration/auth/okta.md b/doc/administration/auth/okta.md index aa4e1b0d2e0..566003ba708 100644 --- a/doc/administration/auth/okta.md +++ b/doc/administration/auth/okta.md @@ -16,7 +16,7 @@ The following documentation enables Okta as a SAML provider. 1. Next, you'll need the to fill in the SAML general config. Here's an example image. -  +  1. The last part of the configuration is the feedback section where you can just say you're a customer and creating an app for internal use. @@ -24,7 +24,7 @@ The following documentation enables Okta as a SAML provider. profile. Click on the SAML 2.0 config instructions button which should look like the following: -  +  1. On the screen that comes up take note of the **Identity Provider Single Sign-On URL** which you'll use for the @@ -38,112 +38,112 @@ Now that the Okta app is configured, it's time to enable it in GitLab. ## Configure GitLab -1. On your GitLab server, open the configuration file: - - **For Omnibus GitLab installations** - - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` - - **For installations from source** - - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` - -1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) - for initial settings. - -1. To allow your users to use Okta to sign up without having to manually create - an account first, don't forget to add the following values to your - configuration: - - **For Omnibus GitLab installations** - - ```ruby - gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'] - gitlab_rails['omniauth_block_auto_created_users'] = false - ``` - - **For installations from source** - - ```yaml - allow_single_sign_on: ["saml"] - block_auto_created_users: false - ``` - -1. You can also automatically link Okta users with existing GitLab users if - their email addresses match by adding the following setting: - - **For Omnibus GitLab installations** - - ```ruby - gitlab_rails['omniauth_auto_link_saml_user'] = true - ``` - - **For installations from source** - - ```yaml - auto_link_saml_user: true - ``` - -1. Add the provider configuration. - - >**Notes:** - > - >- Change the value for `assertion_consumer_service_url` to match the HTTPS endpoint - of GitLab (append `users/auth/saml/callback` to the HTTPS URL of your GitLab - installation to generate the correct value). - > - >- To get the `idp_cert_fingerprint` fingerprint, first download the - certificate from the Okta app you registered and then run: - `openssl x509 -in okta.cert -noout -fingerprint`. Substitute `okta.cert` - with the location of your certificate. - > - >- Change the value of `idp_sso_target_url`, with the value of the - **Identity Provider Single Sign-On URL** from the step when you - configured the Okta app. - > - >- Change the value of `issuer` to the value of the **Audience Restriction** from your Okta app configuration. This will identify GitLab - to the IdP. - > - >- Leave `name_identifier_format` as-is. - - **For Omnibus GitLab installations** - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - name: 'saml', - args: { - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', - idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', - idp_sso_target_url: 'https://gitlab.oktapreview.com/app/gitlabdev773716_gitlabsaml_1/exk8odl81tBrjpD4B0h7/sso/saml', - issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' - }, - label: 'Okta' # optional label for SAML login button, defaults to "Saml" - } - ] - ``` - - **For installations from source** - - ```yaml - - { - name: 'saml', - args: { - assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', - idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', - idp_sso_target_url: 'https://gitlab.oktapreview.com/app/gitlabdev773716_gitlabsaml_1/exk8odl81tBrjpD4B0h7/sso/saml', - issuer: 'https://gitlab.example.com', - name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' - }, - label: 'Okta' # optional label for SAML login button, defaults to "Saml" - } - ``` +1. On your GitLab server, open the configuration file: + + **For Omnibus GitLab installations** + + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` + + **For installations from source** + + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` + +1. See [Initial OmniAuth Configuration](../../integration/omniauth.md#initial-omniauth-configuration) + for initial settings. + +1. To allow your users to use Okta to sign up without having to manually create + an account first, don't forget to add the following values to your + configuration: + + **For Omnibus GitLab installations** + + ```ruby + gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'] + gitlab_rails['omniauth_block_auto_created_users'] = false + ``` + + **For installations from source** + + ```yaml + allow_single_sign_on: ["saml"] + block_auto_created_users: false + ``` + +1. You can also automatically link Okta users with existing GitLab users if + their email addresses match by adding the following setting: + + **For Omnibus GitLab installations** + + ```ruby + gitlab_rails['omniauth_auto_link_saml_user'] = true + ``` + + **For installations from source** + + ```yaml + auto_link_saml_user: true + ``` + +1. Add the provider configuration. + + >**Notes:** + > + >- Change the value for `assertion_consumer_service_url` to match the HTTPS endpoint + of GitLab (append `users/auth/saml/callback` to the HTTPS URL of your GitLab + installation to generate the correct value). + > + >- To get the `idp_cert_fingerprint` fingerprint, first download the + certificate from the Okta app you registered and then run: + `openssl x509 -in okta.cert -noout -fingerprint`. Substitute `okta.cert` + with the location of your certificate. + > + >- Change the value of `idp_sso_target_url`, with the value of the + **Identity Provider Single Sign-On URL** from the step when you + configured the Okta app. + > + >- Change the value of `issuer` to the value of the **Audience Restriction** from your Okta app configuration. This will identify GitLab + to the IdP. + > + >- Leave `name_identifier_format` as-is. + + **For Omnibus GitLab installations** + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + name: 'saml', + args: { + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', + idp_sso_target_url: 'https://gitlab.oktapreview.com/app/gitlabdev773716_gitlabsaml_1/exk8odl81tBrjpD4B0h7/sso/saml', + issuer: 'https://gitlab.example.com', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' + }, + label: 'Okta' # optional label for SAML login button, defaults to "Saml" + } + ] + ``` + + **For installations from source** + + ```yaml + - { + name: 'saml', + args: { + assertion_consumer_service_url: 'https://gitlab.example.com/users/auth/saml/callback', + idp_cert_fingerprint: '43:51:43:a1:b5:fc:8b:b7:0a:3a:a9:b1:0f:66:73:a8', + idp_sso_target_url: 'https://gitlab.oktapreview.com/app/gitlabdev773716_gitlabsaml_1/exk8odl81tBrjpD4B0h7/sso/saml', + issuer: 'https://gitlab.example.com', + name_identifier_format: 'urn:oasis:names:tc:SAML:2.0:nameid-format:transient' + }, + label: 'Okta' # optional label for SAML login button, defaults to "Saml" + } + ``` 1. [Reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart](../restart_gitlab.md#installations-from-source) GitLab for Omnibus and installations from source respectively for the changes to take effect. diff --git a/doc/administration/auth/smartcard.md b/doc/administration/auth/smartcard.md index a0d4e9ef3b5..e47751e0cc5 100644 --- a/doc/administration/auth/smartcard.md +++ b/doc/administration/auth/smartcard.md @@ -56,11 +56,11 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['smartcard_enabled'] = true - gitlab_rails['smartcard_ca_file'] = "/etc/ssl/certs/CA.pem" - gitlab_rails['smartcard_client_certificate_required_port'] = 3444 - ``` + ```ruby + gitlab_rails['smartcard_enabled'] = true + gitlab_rails['smartcard_ca_file'] = "/etc/ssl/certs/CA.pem" + gitlab_rails['smartcard_client_certificate_required_port'] = 3444 + ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. @@ -154,15 +154,15 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['ldap_servers'] = YAML.load <<-EOS - main: - # snip... - # Enable smartcard authentication against the LDAP server. Valid values - # are "false", "optional", and "required". - smartcard_auth: optional - EOS - ``` + ```ruby + gitlab_rails['ldap_servers'] = YAML.load <<-EOS + main: + # snip... + # Enable smartcard authentication against the LDAP server. Valid values + # are "false", "optional", and "required". + smartcard_auth: optional + EOS + ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. @@ -171,16 +171,16 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `config/gitlab.yml`: - ```yaml - production: - ldap: - servers: - main: - # snip... - # Enable smartcard authentication against the LDAP server. Valid values - # are "false", "optional", and "required". - smartcard_auth: optional - ``` + ```yaml + production: + ldap: + servers: + main: + # snip... + # Enable smartcard authentication against the LDAP server. Valid values + # are "false", "optional", and "required". + smartcard_auth: optional + ``` 1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. @@ -191,9 +191,9 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['smartcard_required_for_git_access'] = true - ``` + ```ruby + gitlab_rails['smartcard_required_for_git_access'] = true + ``` 1. Save the file and [reconfigure](../restart_gitlab.md#omnibus-gitlab-reconfigure) GitLab for the changes to take effect. @@ -202,13 +202,13 @@ attribute. As a prerequisite, you must use an LDAP server that: 1. Edit `config/gitlab.yml`: - ```yaml - ## Smartcard authentication settings - smartcard: - # snip... - # Browser session with smartcard sign-in is required for Git access - required_for_git_access: true - ``` + ```yaml + ## Smartcard authentication settings + smartcard: + # snip... + # Browser session with smartcard sign-in is required for Git access + required_for_git_access: true + ``` 1. Save the file and [restart](../restart_gitlab.md#installations-from-source) GitLab for the changes to take effect. diff --git a/doc/administration/high_availability/consul.md b/doc/administration/high_availability/consul.md index 49199b659bc..1f93c8130d3 100644 --- a/doc/administration/high_availability/consul.md +++ b/doc/administration/high_availability/consul.md @@ -26,27 +26,27 @@ On each Consul node perform the following: 1. Edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: - ```ruby - # Disable all components except Consul - roles ['consul_role'] - - # START user configuration - # Replace placeholders: - # - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses gathered for CONSUL_SERVER_NODES - consul['configuration'] = { - server: true, - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) - } - - # Disable auto migrations - gitlab_rails['auto_migrate'] = false - # - # END user configuration - ``` - - > `consul_role` was introduced with GitLab 10.3 + ```ruby + # Disable all components except Consul + roles ['consul_role'] + + # START user configuration + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + server: true, + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + + # Disable auto migrations + gitlab_rails['auto_migrate'] = false + # + # END user configuration + ``` + + > `consul_role` was introduced with GitLab 10.3 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure) for the changes to take effect. @@ -77,6 +77,7 @@ check the [Troubleshooting section](#troubleshooting) before proceeding. ### Checking cluster membership To see which nodes are part of the cluster, run the following on any member in the cluster + ``` # /opt/gitlab/embedded/bin/consul members Node Address Status Type Build Protocol DC @@ -112,18 +113,18 @@ You will see messages like the following in `gitlab-ctl tail consul` output if y 2017-09-25_19:53:41.74356 2017/09/25 19:53:41 [ERR] agent: failed to sync remote state: No cluster leader ``` - To fix this: 1. Pick an address on each node that all of the other nodes can reach this node through. 1. Update your `/etc/gitlab/gitlab.rb` - ```ruby - consul['configuration'] = { - ... - bind_addr: 'IP ADDRESS' - } - ``` + ```ruby + consul['configuration'] = { + ... + bind_addr: 'IP ADDRESS' + } + ``` + 1. Run `gitlab-ctl reconfigure` If you still see the errors, you may have to [erase the consul database and reinitialize](#recreate-from-scratch) on the affected node. @@ -144,12 +145,13 @@ To fix this: 1. Pick an address on the node that all of the other nodes can reach this node through. 1. Update your `/etc/gitlab/gitlab.rb` - ```ruby - consul['configuration'] = { - ... - bind_addr: 'IP ADDRESS' - } - ``` + ```ruby + consul['configuration'] = { + ... + bind_addr: 'IP ADDRESS' + } + ``` + 1. Run `gitlab-ctl reconfigure` ### Outage recovery @@ -157,6 +159,7 @@ To fix this: If you lost enough server agents in the cluster to break quorum, then the cluster is considered failed, and it will not function without manual intervenetion. #### Recreate from scratch + By default, GitLab does not store anything in the consul cluster that cannot be recreated. To erase the consul database and reinitialize ``` @@ -168,4 +171,5 @@ By default, GitLab does not store anything in the consul cluster that cannot be After this, the cluster should start back up, and the server agents rejoin. Shortly after that, the client agents should rejoin as well. #### Recover a failed cluster + If you have taken advantage of consul to store other data, and want to restore the failed cluster, please follow the [Consul guide](https://www.consul.io/docs/guides/outage.html) to recover a failed cluster. diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md index 0db9985acd9..1702a731647 100644 --- a/doc/administration/high_availability/database.md +++ b/doc/administration/high_availability/database.md @@ -33,15 +33,15 @@ deploy the bundled PostgreSQL. 1. SSH into the PostgreSQL server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using **steps 1 and 2** from the GitLab downloads page. - - Do not complete any other steps on the download page. + - Do not complete any other steps on the download page. 1. Generate a password hash for PostgreSQL. This assumes you will use the default username of `gitlab` (recommended). The command will request a password and confirmation. Use the value that is output by this command in the next step as the value of `POSTGRESQL_PASSWORD_HASH`. - ```sh - sudo gitlab-ctl pg-password-md5 gitlab - ``` + ```sh + sudo gitlab-ctl pg-password-md5 gitlab + ``` 1. Edit `/etc/gitlab/gitlab.rb` and add the contents below, updating placeholder values appropriately. @@ -51,32 +51,32 @@ deploy the bundled PostgreSQL. addresses of the GitLab application servers that will connect to the database. Example: `%w(123.123.123.123/32 123.123.123.234/32)` - ```ruby - # Disable all components except PostgreSQL - roles ['postgres_role'] - repmgr['enable'] = false - consul['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false - pgbouncer_exporter['enable'] = false - redis_exporter['enable'] = false - gitlab_monitor['enable'] = false - - postgresql['listen_address'] = '0.0.0.0' - postgresql['port'] = 5432 - - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - # ???? - postgresql['trust_auth_cidr_addresses'] = %w(APPLICATION_SERVER_IP_BLOCKS) - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - ``` + ```ruby + # Disable all components except PostgreSQL + roles ['postgres_role'] + repmgr['enable'] = false + consul['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + redis_exporter['enable'] = false + gitlab_monitor['enable'] = false + + postgresql['listen_address'] = '0.0.0.0' + postgresql['port'] = 5432 + + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + # ???? + postgresql['trust_auth_cidr_addresses'] = %w(APPLICATION_SERVER_IP_BLOCKS) + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + ``` - NOTE: **Note:** The role `postgres_role` was introduced with GitLab 10.3 + NOTE: **Note:** The role `postgres_role` was introduced with GitLab 10.3 1. [Reconfigure GitLab] for the changes to take effect. 1. Note the PostgreSQL node's IP address or hostname, port, and @@ -194,9 +194,9 @@ When using default setup, minimum configuration requires: - `CONSUL_PASSWORD_HASH`. This is a hash generated out of consul username/password pair. Can be generated with: - ```sh - sudo gitlab-ctl pg-password-md5 CONSUL_USERNAME - ``` + ```sh + sudo gitlab-ctl pg-password-md5 CONSUL_USERNAME + ``` - `CONSUL_SERVER_NODES`. The IP addresses or DNS records of the Consul server nodes. @@ -237,9 +237,9 @@ We will need the following password information for the application's database u - `POSTGRESQL_PASSWORD_HASH`. This is a hash generated out of the username/password pair. Can be generated with: - ```sh - sudo gitlab-ctl pg-password-md5 POSTGRESQL_USERNAME - ``` + ```sh + sudo gitlab-ctl pg-password-md5 POSTGRESQL_USERNAME + ``` ##### Pgbouncer information @@ -250,9 +250,9 @@ When using default setup, minimum configuration requires: - `PGBOUNCER_PASSWORD_HASH`. This is a hash generated out of pgbouncer username/password pair. Can be generated with: - ```sh - sudo gitlab-ctl pg-password-md5 PGBOUNCER_USERNAME - ``` + ```sh + sudo gitlab-ctl pg-password-md5 PGBOUNCER_USERNAME + ``` - `PGBOUNCER_NODE`, is the IP address or a FQDN of the node running Pgbouncer. @@ -288,7 +288,6 @@ Make sure you install the necessary dependencies from step 1, add GitLab package repository from step 2. When installing the GitLab package, do not supply `EXTERNAL_URL` value. - #### Configuring the Database nodes 1. Make sure to [configure the Consul nodes](consul.md). @@ -296,53 +295,54 @@ When installing the GitLab package, do not supply `EXTERNAL_URL` value. 1. On the master database node, edit `/etc/gitlab/gitlab.rb` replacing values noted in the `# START user configuration` section: - ```ruby - # Disable all components except PostgreSQL and Repmgr and Consul - roles ['postgres_role'] - - # PostgreSQL configuration - postgresql['listen_address'] = '0.0.0.0' - postgresql['hot_standby'] = 'on' - postgresql['wal_level'] = 'replica' - postgresql['shared_preload_libraries'] = 'repmgr_funcs' - - # Disable automatic database migrations - gitlab_rails['auto_migrate'] = false - - # Configure the consul agent - consul['services'] = %w(postgresql) - - # START user configuration - # Please set the real values as explained in Required Information section - # - # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value - postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' - # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value - postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' - # Replace X with value of number of db nodes + 1 - postgresql['max_wal_senders'] = X - - # Replace XXX.XXX.XXX.XXX/YY with Network Address - postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) - repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 XXX.XXX.XXX.XXX/YY) - - # Replace placeholders: - # - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses gathered for CONSUL_SERVER_NODES - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) - } - # - # END user configuration - ``` - - > `postgres_role` was introduced with GitLab 10.3 + ```ruby + # Disable all components except PostgreSQL and Repmgr and Consul + roles ['postgres_role'] + + # PostgreSQL configuration + postgresql['listen_address'] = '0.0.0.0' + postgresql['hot_standby'] = 'on' + postgresql['wal_level'] = 'replica' + postgresql['shared_preload_libraries'] = 'repmgr_funcs' + + # Disable automatic database migrations + gitlab_rails['auto_migrate'] = false + + # Configure the consul agent + consul['services'] = %w(postgresql) + + # START user configuration + # Please set the real values as explained in Required Information section + # + # Replace PGBOUNCER_PASSWORD_HASH with a generated md5 value + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_PASSWORD_HASH' + # Replace POSTGRESQL_PASSWORD_HASH with a generated md5 value + postgresql['sql_user_password'] = 'POSTGRESQL_PASSWORD_HASH' + # Replace X with value of number of db nodes + 1 + postgresql['max_wal_senders'] = X + + # Replace XXX.XXX.XXX.XXX/YY with Network Address + postgresql['trust_auth_cidr_addresses'] = %w(XXX.XXX.XXX.XXX/YY) + repmgr['trust_auth_cidr_addresses'] = %w(127.0.0.1/32 XXX.XXX.XXX.XXX/YY) + + # Replace placeholders: + # + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses gathered for CONSUL_SERVER_NODES + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z) + } + # + # END user configuration + ``` + + > `postgres_role` was introduced with GitLab 10.3 1. On secondary nodes, add all the configuration specified above for primary node to `/etc/gitlab/gitlab.rb`. In addition, append the following configuration to inform gitlab-ctl that they are standby nodes initially and it need not attempt to register them as primary node + ``` # HA setting to specify if a node should attempt to be master on initialization repmgr['master_on_initialization'] = false @@ -367,31 +367,31 @@ Select one node as a primary node. 1. Open a database prompt: - ```sh - gitlab-psql -d gitlabhq_production - ``` + ```sh + gitlab-psql -d gitlabhq_production + ``` 1. Enable the `pg_trgm` extension: - ```sh - CREATE EXTENSION pg_trgm; - ``` + ```sh + CREATE EXTENSION pg_trgm; + ``` 1. Exit the database prompt by typing `\q` and Enter. 1. Verify the cluster is initialized with one node: - ```sh - gitlab-ctl repmgr cluster show - ``` + ```sh + gitlab-ctl repmgr cluster show + ``` - The output should be similar to the following: + The output should be similar to the following: - ``` - Role | Name | Upstream | Connection String - ----------+----------|----------|---------------------------------------- - * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` + ``` + Role | Name | Upstream | Connection String + ----------+----------|----------|---------------------------------------- + * master | HOSTNAME | | host=HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` 1. Note down the hostname/ip in the connection string: `host=HOSTNAME`. We will refer to the hostname in the next section as `MASTER_NODE_NAME`. If the value @@ -402,43 +402,43 @@ Select one node as a primary node. 1. Set up the repmgr standby: - ```sh - gitlab-ctl repmgr standby setup MASTER_NODE_NAME - ``` - - Do note that this will remove the existing data on the node. The command - has a wait time. - - The output should be similar to the following: - - ```console - # gitlab-ctl repmgr standby setup MASTER_NODE_NAME - Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data - If this is not what you want, hit Ctrl-C now to exit - To skip waiting, rerun with the -w option - Sleeping for 30 seconds - Stopping the database - Removing the data - Cloning the data - Starting the database - Registering the node with the cluster - ok: run: repmgrd: (pid 19068) 0s - ``` + ```sh + gitlab-ctl repmgr standby setup MASTER_NODE_NAME + ``` + + Do note that this will remove the existing data on the node. The command + has a wait time. + + The output should be similar to the following: + + ```console + # gitlab-ctl repmgr standby setup MASTER_NODE_NAME + Doing this will delete the entire contents of /var/opt/gitlab/postgresql/data + If this is not what you want, hit Ctrl-C now to exit + To skip waiting, rerun with the -w option + Sleeping for 30 seconds + Stopping the database + Removing the data + Cloning the data + Starting the database + Registering the node with the cluster + ok: run: repmgrd: (pid 19068) 0s + ``` 1. Verify the node now appears in the cluster: - ```sh - gitlab-ctl repmgr cluster show - ``` + ```sh + gitlab-ctl repmgr cluster show + ``` - The output should be similar to the following: + The output should be similar to the following: - ``` - Role | Name | Upstream | Connection String - ----------+---------|-----------|------------------------------------------------ - * master | MASTER | | host=MASTER_NODE_NAME user=gitlab_repmgr dbname=gitlab_repmgr - standby | STANDBY | MASTER | host=STANDBY_HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr - ``` + ``` + Role | Name | Upstream | Connection String + ----------+---------|-----------|------------------------------------------------ + * master | MASTER | | host=MASTER_NODE_NAME user=gitlab_repmgr dbname=gitlab_repmgr + standby | STANDBY | MASTER | host=STANDBY_HOSTNAME user=gitlab_repmgr dbname=gitlab_repmgr + ``` Repeat the above steps on all secondary nodes. @@ -487,15 +487,15 @@ attributes set, but the following need to be set. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - # Disable PostgreSQL on the application node - postgresql['enable'] = false + ```ruby + # Disable PostgreSQL on the application node + postgresql['enable'] = false - gitlab_rails['db_host'] = 'PGBOUNCER_NODE' - gitlab_rails['db_port'] = 6432 - gitlab_rails['db_password'] = 'POSTGRESQL_USER_PASSWORD' - gitlab_rails['auto_migrate'] = false - ``` + gitlab_rails['db_host'] = 'PGBOUNCER_NODE' + gitlab_rails['db_port'] = 6432 + gitlab_rails['db_password'] = 'POSTGRESQL_USER_PASSWORD' + gitlab_rails['auto_migrate'] = false + ``` 1. [Reconfigure GitLab] for the changes to take effect. @@ -655,45 +655,45 @@ After deploying the configuration follow these steps: 1. On `10.6.0.21`, our primary database - Enable the `pg_trgm` extension + Enable the `pg_trgm` extension - ```sh - gitlab-psql -d gitlabhq_production - ``` + ```sh + gitlab-psql -d gitlabhq_production + ``` - ``` - CREATE EXTENSION pg_trgm; - ``` + ``` + CREATE EXTENSION pg_trgm; + ``` 1. On `10.6.0.22`, our first standby database - Make this node a standby of the primary + Make this node a standby of the primary - ```sh - gitlab-ctl repmgr standby setup 10.6.0.21 - ``` + ```sh + gitlab-ctl repmgr standby setup 10.6.0.21 + ``` 1. On `10.6.0.23`, our second standby database - Make this node a standby of the primary + Make this node a standby of the primary - ```sh - gitlab-ctl repmgr standby setup 10.6.0.21 - ``` + ```sh + gitlab-ctl repmgr standby setup 10.6.0.21 + ``` 1. On `10.6.0.31`, our application server - Set gitlab-consul's pgbouncer password to `toomanysecrets` + Set gitlab-consul's pgbouncer password to `toomanysecrets` - ```sh - gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul - ``` + ```sh + gitlab-ctl write-pgpass --host 127.0.0.1 --database pgbouncer --user pgbouncer --hostuser gitlab-consul + ``` - Run database migrations + Run database migrations - ```sh - gitlab-rake gitlab:db:configure - ``` + ```sh + gitlab-rake gitlab:db:configure + ``` #### Example minimal setup @@ -830,16 +830,16 @@ standby nodes. 1. Ensure the old master node is not still active. 1. Login to the server that should become the new master and run: - ```sh - gitlab-ctl repmgr standby promote - ``` + ```sh + gitlab-ctl repmgr standby promote + ``` 1. If there are any other standby servers in the cluster, have them follow the new master server: - ```sh - gitlab-ctl repmgr standby follow NEW_MASTER - ``` + ```sh + gitlab-ctl repmgr standby follow NEW_MASTER + ``` #### Restore procedure @@ -849,42 +849,42 @@ after it has been restored to service. - If you want to remove the node from the cluster, on any other node in the cluster, run: - ```sh - gitlab-ctl repmgr standby unregister --node=X - ``` + ```sh + gitlab-ctl repmgr standby unregister --node=X + ``` - where X is the value of node in `repmgr.conf` on the old server. + where X is the value of node in `repmgr.conf` on the old server. - To find this, you can use: + To find this, you can use: - ```sh - awk -F = '$1 == "node" { print $2 }' /var/opt/gitlab/postgresql/repmgr.conf - ``` + ```sh + awk -F = '$1 == "node" { print $2 }' /var/opt/gitlab/postgresql/repmgr.conf + ``` - It will output something like: + It will output something like: - ``` - 959789412 - ``` + ``` + 959789412 + ``` - Then you will use this id to unregister the node: + Then you will use this id to unregister the node: - ```sh - gitlab-ctl repmgr standby unregister --node=959789412 - ``` + ```sh + gitlab-ctl repmgr standby unregister --node=959789412 + ``` - To add the node as a standby server: - ```sh - gitlab-ctl repmgr standby follow NEW_MASTER - gitlab-ctl restart repmgrd - ``` + ```sh + gitlab-ctl repmgr standby follow NEW_MASTER + gitlab-ctl restart repmgrd + ``` - CAUTION: **Warning:** When the server is brought back online, and before - you switch it to a standby node, repmgr will report that there are two masters. - If there are any clients that are still attempting to write to the old master, - this will cause a split, and the old master will need to be resynced from - scratch by performing a `gitlab-ctl repmgr standby setup NEW_MASTER`. + CAUTION: **Warning:** When the server is brought back online, and before + you switch it to a standby node, repmgr will report that there are two masters. + If there are any clients that are still attempting to write to the old master, + this will cause a split, and the old master will need to be resynced from + scratch by performing a `gitlab-ctl repmgr standby setup NEW_MASTER`. #### Alternate configurations @@ -927,13 +927,13 @@ the previous section: 1. On the current master node, create a password for the `gitlab` and `gitlab_repmgr` user: - ```sh - gitlab-psql -d template1 - template1=# \password gitlab_repmgr - Enter password: **** - Confirm password: **** - template1=# \password gitlab - ``` + ```sh + gitlab-psql -d template1 + template1=# \password gitlab_repmgr + Enter password: **** + Confirm password: **** + template1=# \password gitlab + ``` 1. On each database node: @@ -947,9 +947,9 @@ the previous section: 1. Create a `.pgpass` file. Enter the `gitlab_repmgr` password twice to when asked: - ```sh - gitlab-ctl write-pgpass --user gitlab_repmgr --hostuser gitlab-psql --database '*' - ``` + ```sh + gitlab-ctl write-pgpass --user gitlab_repmgr --hostuser gitlab-psql --database '*' + ``` 1. On each pgbouncer node, edit `/etc/gitlab/gitlab.rb`: 1. Ensure `gitlab_rails['db_password']` is set to the plaintext password for @@ -977,7 +977,7 @@ If you enable Monitoring, it must be enabled on **all** database servers. ## Troubleshooting -### Consul and PostgreSQL changes not taking effect. +### Consul and PostgreSQL changes not taking effect Due to the potential impacts, `gitlab-ctl reconfigure` only reloads Consul and PostgreSQL, it will not restart the services. However, not all changes can be activated by reloading. diff --git a/doc/administration/high_availability/gitlab.md b/doc/administration/high_availability/gitlab.md index 9b1b7142e83..83838928519 100644 --- a/doc/administration/high_availability/gitlab.md +++ b/doc/administration/high_availability/gitlab.md @@ -7,33 +7,33 @@ 1. If necessary, install the NFS client utility packages using the following commands: - ``` - # Ubuntu/Debian - apt-get install nfs-common + ``` + # Ubuntu/Debian + apt-get install nfs-common - # CentOS/Red Hat - yum install nfs-utils nfs-utils-lib - ``` + # CentOS/Red Hat + yum install nfs-utils nfs-utils-lib + ``` 1. Specify the necessary NFS shares. Mounts are specified in `/etc/fstab`. The exact contents of `/etc/fstab` will depend on how you chose to configure your NFS server. See [NFS documentation](nfs.md) for the various options. Here is an example snippet to add to `/etc/fstab`: - ``` - 10.1.0.1:/var/opt/gitlab/.ssh /var/opt/gitlab/.ssh nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/uploads nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-rails/shared nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/gitlab-ci/builds nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - 10.1.0.1:/var/opt/gitlab/git-data /var/opt/gitlab/git-data nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 - ``` + ``` + 10.1.0.1:/var/opt/gitlab/.ssh /var/opt/gitlab/.ssh nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/uploads nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-rails/shared nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/gitlab-ci/builds nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + 10.1.0.1:/var/opt/gitlab/git-data /var/opt/gitlab/git-data nfs4 defaults,soft,rsize=1048576,wsize=1048576,noatime,nofail,lookupcache=positive 0 2 + ``` 1. Create the shared directories. These may be different depending on your NFS mount locations. - ``` - mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data - ``` + ``` + mkdir -p /var/opt/gitlab/.ssh /var/opt/gitlab/gitlab-rails/uploads /var/opt/gitlab/gitlab-rails/shared /var/opt/gitlab/gitlab-ci/builds /var/opt/gitlab/git-data + ``` 1. Download/install GitLab Omnibus using **steps 1 and 2** from [GitLab downloads](https://about.gitlab.com/downloads). Do not complete other @@ -46,52 +46,52 @@ added NFS mounts in the default data locations. Additionally the UID and GIDs given are just examples and you should configure with your preferred values. - ```ruby - external_url 'https://gitlab.example.com' - - # Prevent GitLab from starting if NFS data mounts are not available - high_availability['mountpoint'] = '/var/opt/gitlab/git-data' - - # Disable components that will not be on the GitLab application server - roles ['application_role'] - nginx['enable'] = true - - # PostgreSQL connection details - gitlab_rails['db_adapter'] = 'postgresql' - gitlab_rails['db_encoding'] = 'unicode' - gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server - gitlab_rails['db_password'] = 'DB password' - - # Redis connection details - gitlab_rails['redis_port'] = '6379' - gitlab_rails['redis_host'] = '10.1.0.6' # IP/hostname of Redis server - gitlab_rails['redis_password'] = 'Redis Password' - - # Ensure UIDs and GIDs match between servers for permissions via NFS - user['uid'] = 9000 - user['gid'] = 9000 - web_server['uid'] = 9001 - web_server['gid'] = 9001 - registry['uid'] = 9002 - registry['gid'] = 9002 - ``` + ```ruby + external_url 'https://gitlab.example.com' + + # Prevent GitLab from starting if NFS data mounts are not available + high_availability['mountpoint'] = '/var/opt/gitlab/git-data' + + # Disable components that will not be on the GitLab application server + roles ['application_role'] + nginx['enable'] = true + + # PostgreSQL connection details + gitlab_rails['db_adapter'] = 'postgresql' + gitlab_rails['db_encoding'] = 'unicode' + gitlab_rails['db_host'] = '10.1.0.5' # IP/hostname of database server + gitlab_rails['db_password'] = 'DB password' + + # Redis connection details + gitlab_rails['redis_port'] = '6379' + gitlab_rails['redis_host'] = '10.1.0.6' # IP/hostname of Redis server + gitlab_rails['redis_password'] = 'Redis Password' + + # Ensure UIDs and GIDs match between servers for permissions via NFS + user['uid'] = 9000 + user['gid'] = 9000 + web_server['uid'] = 9001 + web_server['gid'] = 9001 + registry['uid'] = 9002 + registry['gid'] = 9002 + ``` 1. [Enable monitoring](#enable-monitoring) - > **Note:** To maintain uniformity of links across HA clusters, the `external_url` - on the first application server as well as the additional application - servers should point to the external url that users will use to access GitLab. - In a typical HA setup, this will be the url of the load balancer which will - route traffic to all GitLab application servers in the HA cluster. - > - > **Note:** When you specify `https` in the `external_url`, as in the example - above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If - certificates are not present, Nginx will fail to start. See - [Nginx documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) - for more information. - > - > **Note:** It is best to set the `uid` and `gid`s prior to the initial reconfigure - of GitLab. Omnibus will not recursively `chown` directories if set after the initial reconfigure. + > **Note:** To maintain uniformity of links across HA clusters, the `external_url` + on the first application server as well as the additional application + servers should point to the external url that users will use to access GitLab. + In a typical HA setup, this will be the url of the load balancer which will + route traffic to all GitLab application servers in the HA cluster. + > + > **Note:** When you specify `https` in the `external_url`, as in the example + above, GitLab assumes you have SSL certificates in `/etc/gitlab/ssl/`. If + certificates are not present, Nginx will fail to start. See + [Nginx documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https) + for more information. + > + > **Note:** It is best to set the `uid` and `gid`s prior to the initial reconfigure + of GitLab. Omnibus will not recursively `chown` directories if set after the initial reconfigure. ## First GitLab application server @@ -114,12 +114,12 @@ need some extra configuration. secondary servers **prior to** running the first `reconfigure` in the steps above. - ```ruby - gitlab_shell['secret_token'] = 'fbfb19c355066a9afb030992231c4a363357f77345edd0f2e772359e5be59b02538e1fa6cae8f93f7d23355341cea2b93600dab6d6c3edcdced558fc6d739860' - gitlab_rails['otp_key_base'] = 'b719fe119132c7810908bba18315259ed12888d4f5ee5430c42a776d840a396799b0a5ef0a801348c8a357f07aa72bbd58e25a84b8f247a25c72f539c7a6c5fa' - gitlab_rails['secret_key_base'] = '6e657410d57c71b4fc3ed0d694e7842b1895a8b401d812c17fe61caf95b48a6d703cb53c112bc01ebd197a85da81b18e29682040e99b4f26594772a4a2c98c6d' - gitlab_rails['db_key_base'] = 'bf2e47b68d6cafaef1d767e628b619365becf27571e10f196f98dc85e7771042b9203199d39aff91fcb6837c8ed83f2a912b278da50999bb11a2fbc0fba52964' - ``` + ```ruby + gitlab_shell['secret_token'] = 'fbfb19c355066a9afb030992231c4a363357f77345edd0f2e772359e5be59b02538e1fa6cae8f93f7d23355341cea2b93600dab6d6c3edcdced558fc6d739860' + gitlab_rails['otp_key_base'] = 'b719fe119132c7810908bba18315259ed12888d4f5ee5430c42a776d840a396799b0a5ef0a801348c8a357f07aa72bbd58e25a84b8f247a25c72f539c7a6c5fa' + gitlab_rails['secret_key_base'] = '6e657410d57c71b4fc3ed0d694e7842b1895a8b401d812c17fe61caf95b48a6d703cb53c112bc01ebd197a85da81b18e29682040e99b4f26594772a4a2c98c6d' + gitlab_rails['db_key_base'] = 'bf2e47b68d6cafaef1d767e628b619365becf27571e10f196f98dc85e7771042b9203199d39aff91fcb6837c8ed83f2a912b278da50999bb11a2fbc0fba52964' + ``` 1. Run `touch /etc/gitlab/skip-auto-reconfigure` to prevent database migrations from running on upgrade. Only the primary GitLab application server should diff --git a/doc/administration/high_availability/monitoring_node.md b/doc/administration/high_availability/monitoring_node.md index 385e7441ac9..cbc1d4bcd52 100644 --- a/doc/administration/high_availability/monitoring_node.md +++ b/doc/administration/high_availability/monitoring_node.md @@ -4,7 +4,7 @@ ## Standalone Monitoring node using GitLab Omnibus -The GitLab Omnibus package can be used to configure a standalone Monitoring node running Prometheus and Grafana. +The GitLab Omnibus package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md). The monitoring node is not highly available. See [Scaling and High Availability](README.md) for an overview of GitLab scaling and high availability options. @@ -20,44 +20,44 @@ Omnibus: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - ```ruby - external_url 'http://gitlab.example.com' - - # Enable Prometheus - prometheus['enable'] = true - prometheus['listen_address'] = '0.0.0.0:9090' - prometheus['monitor_kubernetes'] = false - - # Enable Grafana - grafana['enable'] = true - grafana['admin_password'] = 'toomanysecrets' - - # Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true - - # Replace placeholders - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses of the Consul server nodes - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), - } - - # Disable all other services - gitlab_rails['auto_migrate'] = false - alertmanager['enable'] = false - gitaly['enable'] = false - gitlab_monitor['enable'] = false - gitlab_workhorse['enable'] = false - nginx['enable'] = true - postgres_exporter['enable'] = false - postgresql['enable'] = false - redis['enable'] = false - redis_exporter['enable'] = false - sidekiq['enable'] = false - unicorn['enable'] = false - node_exporter['enable'] = false - ``` + ```ruby + external_url 'http://gitlab.example.com' + + # Enable Prometheus + prometheus['enable'] = true + prometheus['listen_address'] = '0.0.0.0:9090' + prometheus['monitor_kubernetes'] = false + + # Enable Grafana + grafana['enable'] = true + grafana['admin_password'] = 'toomanysecrets' + + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true + + # Replace placeholders + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses of the Consul server nodes + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), + } + + # Disable all other services + gitlab_rails['auto_migrate'] = false + alertmanager['enable'] = false + gitaly['enable'] = false + gitlab_monitor['enable'] = false + gitlab_workhorse['enable'] = false + nginx['enable'] = true + postgres_exporter['enable'] = false + postgresql['enable'] = false + redis['enable'] = false + redis_exporter['enable'] = false + sidekiq['enable'] = false + unicorn['enable'] = false + node_exporter['enable'] = false + ``` 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md index 561ba214686..6ab6b8bed30 100644 --- a/doc/administration/high_availability/nfs.md +++ b/doc/administration/high_availability/nfs.md @@ -42,8 +42,8 @@ maintaining ID mapping without LDAP, in most cases you should enable numeric UID and GIDs (which is off by default in some cases) for simplified permission management between systems: - - [NetApp instructions](https://library.netapp.com/ecmdocs/ECMP1401220/html/GUID-24367A9F-E17B-4725-ADC1-02D86F56F78E.html) - - For non-NetApp devices, disable NFSv4 `idmapping` by performing opposite of [enable NFSv4 idmapper](https://wiki.archlinux.org/index.php/NFS#Enabling_NFSv4_idmapping) +- [NetApp instructions](https://library.netapp.com/ecmdocs/ECMP1401220/html/GUID-24367A9F-E17B-4725-ADC1-02D86F56F78E.html) +- For non-NetApp devices, disable NFSv4 `idmapping` by performing opposite of [enable NFSv4 idmapper](https://wiki.archlinux.org/index.php/NFS#Enabling_NFSv4_idmapping) ### Improving NFS performance with GitLab @@ -87,10 +87,10 @@ on an Linux NFS server, do the following: 1. On the NFS server, run: - ```sh - echo 0 > /proc/sys/fs/leases-enable - sysctl -w fs.leases-enable=0 - ``` + ```sh + echo 0 > /proc/sys/fs/leases-enable + sysctl -w fs.leases-enable=0 + ``` 1. Restart the NFS server process. For example, on CentOS run `service nfs restart`. diff --git a/doc/administration/high_availability/pgbouncer.md b/doc/administration/high_availability/pgbouncer.md index 2788b087628..6890b0f7db7 100644 --- a/doc/administration/high_availability/pgbouncer.md +++ b/doc/administration/high_availability/pgbouncer.md @@ -105,39 +105,39 @@ It is recommended to run pgbouncer alongside the `gitlab-rails` service, or on i 1. On your database node, ensure the following is set in your `/etc/gitlab/gitlab.rb` - ```ruby - postgresql['pgbouncer_user_password'] = 'PGBOUNCER_USER_PASSWORD_HASH' - postgresql['sql_user_password'] = 'SQL_USER_PASSWORD_HASH' - postgresql['listen_address'] = 'XX.XX.XX.Y' # Where XX.XX.XX.Y is the ip address on the node postgresql should listen on - postgresql['md5_auth_cidr_addresses'] = %w(AA.AA.AA.B/32) # Where AA.AA.AA.B is the IP address of the pgbouncer node - ``` + ```ruby + postgresql['pgbouncer_user_password'] = 'PGBOUNCER_USER_PASSWORD_HASH' + postgresql['sql_user_password'] = 'SQL_USER_PASSWORD_HASH' + postgresql['listen_address'] = 'XX.XX.XX.Y' # Where XX.XX.XX.Y is the ip address on the node postgresql should listen on + postgresql['md5_auth_cidr_addresses'] = %w(AA.AA.AA.B/32) # Where AA.AA.AA.B is the IP address of the pgbouncer node + ``` 1. Run `gitlab-ctl reconfigure` - **Note:** If the database was already running, it will need to be restarted after reconfigure by running `gitlab-ctl restart postgresql`. + **Note:** If the database was already running, it will need to be restarted after reconfigure by running `gitlab-ctl restart postgresql`. 1. On the node you are running pgbouncer on, make sure the following is set in `/etc/gitlab/gitlab.rb` - ```ruby - pgbouncer['enable'] = true - pgbouncer['databases'] = { - gitlabhq_production: { - host: 'DATABASE_HOST', - user: 'pgbouncer', - password: 'PGBOUNCER_USER_PASSWORD_HASH' - } - } - ``` + ```ruby + pgbouncer['enable'] = true + pgbouncer['databases'] = { + gitlabhq_production: { + host: 'DATABASE_HOST', + user: 'pgbouncer', + password: 'PGBOUNCER_USER_PASSWORD_HASH' + } + } + ``` 1. Run `gitlab-ctl reconfigure` 1. On the node running unicorn, make sure the following is set in `/etc/gitlab/gitlab.rb` - ```ruby - gitlab_rails['db_host'] = 'PGBOUNCER_HOST' - gitlab_rails['db_port'] = '6432' - gitlab_rails['db_password'] = 'SQL_USER_PASSWORD' - ``` + ```ruby + gitlab_rails['db_host'] = 'PGBOUNCER_HOST' + gitlab_rails['db_port'] = '6432' + gitlab_rails['db_password'] = 'SQL_USER_PASSWORD' + ``` 1. Run `gitlab-ctl reconfigure` @@ -147,28 +147,28 @@ It is recommended to run pgbouncer alongside the `gitlab-rails` service, or on i > [Introduced](https://gitlab.com/gitlab-org/omnibus-gitlab/issues/3786) in GitLab 12.0. - If you enable Monitoring, it must be enabled on **all** pgbouncer servers. +If you enable Monitoring, it must be enabled on **all** pgbouncer servers. - 1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: +1. Create/edit `/etc/gitlab/gitlab.rb` and add the following configuration: - ```ruby - # Enable service discovery for Prometheus - consul['enable'] = true - consul['monitoring_service_discovery'] = true + ```ruby + # Enable service discovery for Prometheus + consul['enable'] = true + consul['monitoring_service_discovery'] = true - # Replace placeholders - # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z - # with the addresses of the Consul server nodes - consul['configuration'] = { - retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), - } + # Replace placeholders + # Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z + # with the addresses of the Consul server nodes + consul['configuration'] = { + retry_join: %w(Y.Y.Y.Y consul1.gitlab.example.com Z.Z.Z.Z), + } - # Set the network addresses that the exporters will listen on - node_exporter['listen_address'] = '0.0.0.0:9100' - pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' - ``` + # Set the network addresses that the exporters will listen on + node_exporter['listen_address'] = '0.0.0.0:9100' + pgbouncer_exporter['listen_address'] = '0.0.0.0:9188' + ``` - 1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. +1. Run `sudo gitlab-ctl reconfigure` to compile the configuration. ### Interacting with pgbouncer @@ -190,6 +190,7 @@ pgbouncer=# The password you will be prompted for is the PGBOUNCER_USER_PASSWORD To get some basic information about the instance, run + ```shell pgbouncer=# show databases; show clients; show servers; name | host | port | database | force_user | pool_size | reserve_pool | pool_mode | max_connections | current_connections diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md index 27310c59755..c29514ed9f6 100644 --- a/doc/administration/high_availability/redis.md +++ b/doc/administration/high_availability/redis.md @@ -47,28 +47,28 @@ Omnibus: 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - ```ruby - ## Enable Redis - redis['enable'] = true - - ## Disable all other services - sidekiq['enable'] = false - gitlab_workhorse['enable'] = false - unicorn['enable'] = false - postgresql['enable'] = false - nginx['enable'] = false - prometheus['enable'] = false - alertmanager['enable'] = false - pgbouncer_exporter['enable'] = false - gitlab_monitor['enable'] = false - gitaly['enable'] = false - - redis['bind'] = '0.0.0.0' - redis['port'] = '6379' - redis['password'] = 'SECRET_PASSWORD_HERE' - - gitlab_rails['auto_migrate'] = false - ``` + ```ruby + ## Enable Redis + redis['enable'] = true + + ## Disable all other services + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false + unicorn['enable'] = false + postgresql['enable'] = false + nginx['enable'] = false + prometheus['enable'] = false + alertmanager['enable'] = false + pgbouncer_exporter['enable'] = false + gitlab_monitor['enable'] = false + gitaly['enable'] = false + + redis['bind'] = '0.0.0.0' + redis['port'] = '6379' + redis['password'] = 'SECRET_PASSWORD_HERE' + + gitlab_rails['auto_migrate'] = false + ``` 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. 1. Note the Redis node's IP address or hostname, port, and @@ -359,37 +359,37 @@ The prerequisites for a HA Redis setup are the following: 1. SSH into the **master** Redis server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - ```ruby - # Specify server role as 'redis_master_role' - roles ['redis_master_role'] + ```ruby + # Specify server role as 'redis_master_role' + roles ['redis_master_role'] - # IP address pointing to a local IP that the other machines can reach to. - # You can also set bind to '0.0.0.0' which listen in all interfaces. - # If you really need to bind to an external accessible IP, make - # sure you add extra firewall rules to prevent unauthorized access. - redis['bind'] = '10.0.0.1' + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.0.0.1' - # Define a port so Redis can listen for TCP requests which will allow other - # machines to connect to it. - redis['port'] = 6379 + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 - # Set up password authentication for Redis (use the same password in all nodes). - redis['password'] = 'redis-password-goes-here' - ``` + # Set up password authentication for Redis (use the same password in all nodes). + redis['password'] = 'redis-password-goes-here' + ``` 1. Only the primary GitLab application server should handle migrations. To prevent database migrations from running on upgrade, add the following configuration to your `/etc/gitlab/gitlab.rb` file: - ``` - gitlab_rails['auto_migrate'] = false - ``` + ``` + gitlab_rails['auto_migrate'] = false + ``` 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. @@ -402,42 +402,42 @@ The prerequisites for a HA Redis setup are the following: 1. SSH into the **slave** Redis server. 1. [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab package you want using **steps 1 and 2** from the GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - and type (Community, Enterprise editions) of your current install. - - Do not complete any other steps on the download page. + - Make sure you select the correct Omnibus package, with the same version + and type (Community, Enterprise editions) of your current install. + - Do not complete any other steps on the download page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents: - ```ruby - # Specify server role as 'redis_slave_role' - roles ['redis_slave_role'] + ```ruby + # Specify server role as 'redis_slave_role' + roles ['redis_slave_role'] - # IP address pointing to a local IP that the other machines can reach to. - # You can also set bind to '0.0.0.0' which listen in all interfaces. - # If you really need to bind to an external accessible IP, make - # sure you add extra firewall rules to prevent unauthorized access. - redis['bind'] = '10.0.0.2' + # IP address pointing to a local IP that the other machines can reach to. + # You can also set bind to '0.0.0.0' which listen in all interfaces. + # If you really need to bind to an external accessible IP, make + # sure you add extra firewall rules to prevent unauthorized access. + redis['bind'] = '10.0.0.2' - # Define a port so Redis can listen for TCP requests which will allow other - # machines to connect to it. - redis['port'] = 6379 + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 - # The same password for Redis authentication you set up for the master node. - redis['password'] = 'redis-password-goes-here' + # The same password for Redis authentication you set up for the master node. + redis['password'] = 'redis-password-goes-here' - # The IP of the master Redis node. - redis['master_ip'] = '10.0.0.1' + # The IP of the master Redis node. + redis['master_ip'] = '10.0.0.1' - # Port of master Redis server, uncomment to change to non default. Defaults - # to `6379`. - #redis['master_port'] = 6379 - ``` + # Port of master Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + ``` 1. To prevent reconfigure from running automatically on upgrade, run: - ``` - sudo touch /etc/gitlab/skip-auto-reconfigure - ``` + ``` + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. 1. Go through the steps again for all the other slave nodes. @@ -487,89 +487,89 @@ multiple machines with the Sentinel daemon. 1. **You can omit this step if the Sentinels will be hosted in the same node as the other Redis instances.** - [Download/install](https://about.gitlab.com/downloads-ee) the - Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the - GitLab downloads page. - - Make sure you select the correct Omnibus package, with the same version - the GitLab application is running. - - Do not complete any other steps on the download page. + [Download/install](https://about.gitlab.com/downloads-ee) the + Omnibus GitLab Enterprise Edition package using **steps 1 and 2** from the + GitLab downloads page. + - Make sure you select the correct Omnibus package, with the same version + the GitLab application is running. + - Do not complete any other steps on the download page. 1. Edit `/etc/gitlab/gitlab.rb` and add the contents (if you are installing the Sentinels in the same node as the other Redis instances, some values might be duplicate below): - ```ruby - roles ['redis_sentinel_role'] - - # Must be the same in every sentinel node - redis['master_name'] = 'gitlab-redis' - - # The same password for Redis authentication you set up for the master node. - redis['master_password'] = 'redis-password-goes-here' - - # The IP of the master Redis node. - redis['master_ip'] = '10.0.0.1' - - # Define a port so Redis can listen for TCP requests which will allow other - # machines to connect to it. - redis['port'] = 6379 - - # Port of master Redis server, uncomment to change to non default. Defaults - # to `6379`. - #redis['master_port'] = 6379 - - ## Configure Sentinel - sentinel['bind'] = '10.0.0.1' - - # Port that Sentinel listens on, uncomment to change to non default. Defaults - # to `26379`. - # sentinel['port'] = 26379 - - ## Quorum must reflect the amount of voting sentinels it take to start a failover. - ## Value must NOT be greater then the amount of sentinels. - ## - ## The quorum can be used to tune Sentinel in two ways: - ## 1. If a the quorum is set to a value smaller than the majority of Sentinels - ## we deploy, we are basically making Sentinel more sensible to master failures, - ## triggering a failover as soon as even just a minority of Sentinels is no longer - ## able to talk with the master. - ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are - ## making Sentinel able to failover only when there are a very large number (larger - ## than majority) of well connected Sentinels which agree about the master being down.s - sentinel['quorum'] = 2 - - ## Consider unresponsive server down after x amount of ms. - # sentinel['down_after_milliseconds'] = 10000 - - ## Specifies the failover timeout in milliseconds. It is used in many ways: - ## - ## - The time needed to re-start a failover after a previous failover was - ## already tried against the same master by a given Sentinel, is two - ## times the failover timeout. - ## - ## - The time needed for a slave replicating to a wrong master according - ## to a Sentinel current configuration, to be forced to replicate - ## with the right master, is exactly the failover timeout (counting since - ## the moment a Sentinel detected the misconfiguration). - ## - ## - The time needed to cancel a failover that is already in progress but - ## did not produced any configuration change (SLAVEOF NO ONE yet not - ## acknowledged by the promoted slave). - ## - ## - The maximum time a failover in progress waits for all the slaves to be - ## reconfigured as slaves of the new master. However even after this time - ## the slaves will be reconfigured by the Sentinels anyway, but not with - ## the exact parallel-syncs progression as specified. - # sentinel['failover_timeout'] = 60000 - ``` + ```ruby + roles ['redis_sentinel_role'] + + # Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + # The same password for Redis authentication you set up for the master node. + redis['master_password'] = 'redis-password-goes-here' + + # The IP of the master Redis node. + redis['master_ip'] = '10.0.0.1' + + # Define a port so Redis can listen for TCP requests which will allow other + # machines to connect to it. + redis['port'] = 6379 + + # Port of master Redis server, uncomment to change to non default. Defaults + # to `6379`. + #redis['master_port'] = 6379 + + ## Configure Sentinel + sentinel['bind'] = '10.0.0.1' + + # Port that Sentinel listens on, uncomment to change to non default. Defaults + # to `26379`. + # sentinel['port'] = 26379 + + ## Quorum must reflect the amount of voting sentinels it take to start a failover. + ## Value must NOT be greater then the amount of sentinels. + ## + ## The quorum can be used to tune Sentinel in two ways: + ## 1. If a the quorum is set to a value smaller than the majority of Sentinels + ## we deploy, we are basically making Sentinel more sensible to master failures, + ## triggering a failover as soon as even just a minority of Sentinels is no longer + ## able to talk with the master. + ## 1. If a quorum is set to a value greater than the majority of Sentinels, we are + ## making Sentinel able to failover only when there are a very large number (larger + ## than majority) of well connected Sentinels which agree about the master being down.s + sentinel['quorum'] = 2 + + ## Consider unresponsive server down after x amount of ms. + # sentinel['down_after_milliseconds'] = 10000 + + ## Specifies the failover timeout in milliseconds. It is used in many ways: + ## + ## - The time needed to re-start a failover after a previous failover was + ## already tried against the same master by a given Sentinel, is two + ## times the failover timeout. + ## + ## - The time needed for a slave replicating to a wrong master according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right master, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## - The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (SLAVEOF NO ONE yet not + ## acknowledged by the promoted slave). + ## + ## - The maximum time a failover in progress waits for all the slaves to be + ## reconfigured as slaves of the new master. However even after this time + ## the slaves will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + # sentinel['failover_timeout'] = 60000 + ``` 1. To prevent database migrations from running on upgrade, run: - ``` - sudo touch /etc/gitlab/skip-auto-reconfigure - ``` + ``` + sudo touch /etc/gitlab/skip-auto-reconfigure + ``` - Only the primary GitLab application server should handle migrations. + Only the primary GitLab application server should handle migrations. 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. 1. Go through the steps again for all the other Sentinel nodes. @@ -593,20 +593,20 @@ which ideally should not have Redis or Sentinels on it for a HA setup. 1. SSH into the server where the GitLab application is installed. 1. Edit `/etc/gitlab/gitlab.rb` and add/change the following lines: - ``` - ## Must be the same in every sentinel node - redis['master_name'] = 'gitlab-redis' - - ## The same password for Redis authentication you set up for the master node. - redis['master_password'] = 'redis-password-goes-here' - - ## A list of sentinels with `host` and `port` - gitlab_rails['redis_sentinels'] = [ - {'host' => '10.0.0.1', 'port' => 26379}, - {'host' => '10.0.0.2', 'port' => 26379}, - {'host' => '10.0.0.3', 'port' => 26379} - ] - ``` + ```ruby + ## Must be the same in every sentinel node + redis['master_name'] = 'gitlab-redis' + + ## The same password for Redis authentication you set up for the master node. + redis['master_password'] = 'redis-password-goes-here' + + ## A list of sentinels with `host` and `port` + gitlab_rails['redis_sentinels'] = [ + {'host' => '10.0.0.1', 'port' => 26379}, + {'host' => '10.0.0.2', 'port' => 26379}, + {'host' => '10.0.0.3', 'port' => 26379} + ] + ``` 1. [Reconfigure Omnibus GitLab][reconfigure] for the changes to take effect. @@ -791,31 +791,34 @@ cache, queues, and shared_state. To make this work with Sentinel: 1. Set the appropriate variable in `/etc/gitlab/gitlab.rb` for each instance you are using: - ```ruby - gitlab_rails['redis_cache_instance'] = REDIS_CACHE_URL - gitlab_rails['redis_queues_instance'] = REDIS_QUEUES_URL - gitlab_rails['redis_shared_state_instance'] = REDIS_SHARED_STATE_URL - ``` + ```ruby + gitlab_rails['redis_cache_instance'] = REDIS_CACHE_URL + gitlab_rails['redis_queues_instance'] = REDIS_QUEUES_URL + gitlab_rails['redis_shared_state_instance'] = REDIS_SHARED_STATE_URL + ``` + **Note**: Redis URLs should be in the format: `redis://:PASSWORD@SENTINEL_MASTER_NAME` - 1. PASSWORD is the plaintext password for the Redis instance - 1. SENTINEL_MASTER_NAME is the Sentinel master name (e.g. `gitlab-redis-cache`) + 1. PASSWORD is the plaintext password for the Redis instance + 1. SENTINEL_MASTER_NAME is the Sentinel master name (e.g. `gitlab-redis-cache`) + 1. Include an array of hashes with host/port combinations, such as the following: - ```ruby - gitlab_rails['redis_cache_sentinels'] = [ - { host: REDIS_CACHE_SENTINEL_HOST, port: PORT1 }, - { host: REDIS_CACHE_SENTINEL_HOST2, port: PORT2 } - ] - gitlab_rails['redis_queues_sentinels'] = [ - { host: REDIS_QUEUES_SENTINEL_HOST, port: PORT1 }, - { host: REDIS_QUEUES_SENTINEL_HOST2, port: PORT2 } - ] - gitlab_rails['redis_shared_state_sentinels'] = [ - { host: SHARED_STATE_SENTINEL_HOST, port: PORT1 }, - { host: SHARED_STATE_SENTINEL_HOST2, port: PORT2 } - ] - ``` + ```ruby + gitlab_rails['redis_cache_sentinels'] = [ + { host: REDIS_CACHE_SENTINEL_HOST, port: PORT1 }, + { host: REDIS_CACHE_SENTINEL_HOST2, port: PORT2 } + ] + gitlab_rails['redis_queues_sentinels'] = [ + { host: REDIS_QUEUES_SENTINEL_HOST, port: PORT1 }, + { host: REDIS_QUEUES_SENTINEL_HOST2, port: PORT2 } + ] + gitlab_rails['redis_shared_state_sentinels'] = [ + { host: SHARED_STATE_SENTINEL_HOST, port: PORT1 }, + { host: SHARED_STATE_SENTINEL_HOST2, port: PORT2 } + ] + ``` + 1. Note that for each persistence class, GitLab will default to using the configuration specified in `gitlab_rails['redis_sentinels']` unless overridden by the settings above. @@ -879,12 +882,12 @@ in order for the HA setup to work as expected. Before proceeding with the troubleshooting below, check your firewall rules: - Redis machines - - Accept TCP connection in `6379` - - Connect to the other Redis machines via TCP in `6379` + - Accept TCP connection in `6379` + - Connect to the other Redis machines via TCP in `6379` - Sentinel machines - - Accept TCP connection in `26379` - - Connect to other Sentinel machines via TCP in `26379` - - Connect to the Redis machines via TCP in `6379` + - Accept TCP connection in `26379` + - Connect to other Sentinel machines via TCP in `26379` + - Connect to the Redis machines via TCP in `6379` ### Troubleshooting Redis replication @@ -952,38 +955,38 @@ To make sure your configuration is correct: 1. SSH into your GitLab application server 1. Enter the Rails console: - ``` - # For Omnibus installations - sudo gitlab-rails console + ``` + # For Omnibus installations + sudo gitlab-rails console - # For source installations - sudo -u git rails console production - ``` + # For source installations + sudo -u git rails console production + ``` 1. Run in the console: - ```ruby - redis = Redis.new(Gitlab::Redis::SharedState.params) - redis.info - ``` + ```ruby + redis = Redis.new(Gitlab::Redis::SharedState.params) + redis.info + ``` - Keep this screen open and try to simulate a failover below. + Keep this screen open and try to simulate a failover below. 1. To simulate a failover on master Redis, SSH into the Redis server and run: - ```bash - # port must match your master redis port, and the sleep time must be a few seconds bigger than defined one - redis-cli -h localhost -p 6379 DEBUG sleep 20 - ``` + ```bash + # port must match your master redis port, and the sleep time must be a few seconds bigger than defined one + redis-cli -h localhost -p 6379 DEBUG sleep 20 + ``` 1. Then back in the Rails console from the first step, run: - ``` - redis.info - ``` + ``` + redis.info + ``` - You should see a different port after a few seconds delay - (the failover/reconnect time). + You should see a different port after a few seconds delay + (the failover/reconnect time). ## Changelog diff --git a/doc/administration/high_availability/redis_source.md b/doc/administration/high_availability/redis_source.md index be6b547372a..a5463e5128c 100644 --- a/doc/administration/high_availability/redis_source.md +++ b/doc/administration/high_availability/redis_source.md @@ -49,22 +49,22 @@ Assuming that the Redis master instance IP is `10.0.0.1`: 1. [Install Redis](../../install/installation.md#7-redis). 1. Edit `/etc/redis/redis.conf`: - ```conf - ## Define a `bind` address pointing to a local IP that your other machines - ## can reach you. If you really need to bind to an external accessible IP, make - ## sure you add extra firewall rules to prevent unauthorized access: - bind 10.0.0.1 - - ## Define a `port` to force redis to listen on TCP so other machines can - ## connect to it (default port is `6379`). - port 6379 - - ## Set up password authentication (use the same password in all nodes). - ## The password should be defined equal for both `requirepass` and `masterauth` - ## when setting up Redis to use with Sentinel. - requirepass redis-password-goes-here - masterauth redis-password-goes-here - ``` + ```conf + ## Define a `bind` address pointing to a local IP that your other machines + ## can reach you. If you really need to bind to an external accessible IP, make + ## sure you add extra firewall rules to prevent unauthorized access: + bind 10.0.0.1 + + ## Define a `port` to force redis to listen on TCP so other machines can + ## connect to it (default port is `6379`). + port 6379 + + ## Set up password authentication (use the same password in all nodes). + ## The password should be defined equal for both `requirepass` and `masterauth` + ## when setting up Redis to use with Sentinel. + requirepass redis-password-goes-here + masterauth redis-password-goes-here + ``` 1. Restart the Redis service for the changes to take effect. @@ -75,25 +75,25 @@ Assuming that the Redis slave instance IP is `10.0.0.2`: 1. [Install Redis](../../install/installation.md#7-redis). 1. Edit `/etc/redis/redis.conf`: - ```conf - ## Define a `bind` address pointing to a local IP that your other machines - ## can reach you. If you really need to bind to an external accessible IP, make - ## sure you add extra firewall rules to prevent unauthorized access: - bind 10.0.0.2 + ```conf + ## Define a `bind` address pointing to a local IP that your other machines + ## can reach you. If you really need to bind to an external accessible IP, make + ## sure you add extra firewall rules to prevent unauthorized access: + bind 10.0.0.2 - ## Define a `port` to force redis to listen on TCP so other machines can - ## connect to it (default port is `6379`). - port 6379 + ## Define a `port` to force redis to listen on TCP so other machines can + ## connect to it (default port is `6379`). + port 6379 - ## Set up password authentication (use the same password in all nodes). - ## The password should be defined equal for both `requirepass` and `masterauth` - ## when setting up Redis to use with Sentinel. - requirepass redis-password-goes-here - masterauth redis-password-goes-here + ## Set up password authentication (use the same password in all nodes). + ## The password should be defined equal for both `requirepass` and `masterauth` + ## when setting up Redis to use with Sentinel. + requirepass redis-password-goes-here + masterauth redis-password-goes-here - ## Define `slaveof` pointing to the Redis master instance with IP and port. - slaveof 10.0.0.1 6379 - ``` + ## Define `slaveof` pointing to the Redis master instance with IP and port. + slaveof 10.0.0.1 6379 + ``` 1. Restart the Redis service for the changes to take effect. 1. Go through the steps again for all the other slave nodes. @@ -110,56 +110,57 @@ master with IP `10.0.0.1` (some settings might overlap with the master): 1. [Install Redis Sentinel](https://redis.io/topics/sentinel) 1. Edit `/etc/redis/sentinel.conf`: - ```conf - ## Define a `bind` address pointing to a local IP that your other machines - ## can reach you. If you really need to bind to an external accessible IP, make - ## sure you add extra firewall rules to prevent unauthorized access: - bind 10.0.0.1 - - ## Define a `port` to force Sentinel to listen on TCP so other machines can - ## connect to it (default port is `6379`). - port 26379 - - ## Set up password authentication (use the same password in all nodes). - ## The password should be defined equal for both `requirepass` and `masterauth` - ## when setting up Redis to use with Sentinel. - requirepass redis-password-goes-here - masterauth redis-password-goes-here - - ## Define with `sentinel auth-pass` the same shared password you have - ## defined for both Redis master and slaves instances. - sentinel auth-pass gitlab-redis redis-password-goes-here - - ## Define with `sentinel monitor` the IP and port of the Redis - ## master node, and the quorum required to start a failover. - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - - ## Define with `sentinel down-after-milliseconds` the time in `ms` - ## that an unresponsive server will be considered down. - sentinel down-after-milliseconds gitlab-redis 10000 - - ## Define a value for `sentinel failover_timeout` in `ms`. This has multiple - ## meanings: - ## - ## * The time needed to re-start a failover after a previous failover was - ## already tried against the same master by a given Sentinel, is two - ## times the failover timeout. - ## - ## * The time needed for a slave replicating to a wrong master according - ## to a Sentinel current configuration, to be forced to replicate - ## with the right master, is exactly the failover timeout (counting since - ## the moment a Sentinel detected the misconfiguration). - ## - ## * The time needed to cancel a failover that is already in progress but - ## did not produced any configuration change (SLAVEOF NO ONE yet not - ## acknowledged by the promoted slave). - ## - ## * The maximum time a failover in progress waits for all the slaves to be - ## reconfigured as slaves of the new master. However even after this time - ## the slaves will be reconfigured by the Sentinels anyway, but not with - ## the exact parallel-syncs progression as specified. - sentinel failover_timeout 30000 - ``` + ```conf + ## Define a `bind` address pointing to a local IP that your other machines + ## can reach you. If you really need to bind to an external accessible IP, make + ## sure you add extra firewall rules to prevent unauthorized access: + bind 10.0.0.1 + + ## Define a `port` to force Sentinel to listen on TCP so other machines can + ## connect to it (default port is `6379`). + port 26379 + + ## Set up password authentication (use the same password in all nodes). + ## The password should be defined equal for both `requirepass` and `masterauth` + ## when setting up Redis to use with Sentinel. + requirepass redis-password-goes-here + masterauth redis-password-goes-here + + ## Define with `sentinel auth-pass` the same shared password you have + ## defined for both Redis master and slaves instances. + sentinel auth-pass gitlab-redis redis-password-goes-here + + ## Define with `sentinel monitor` the IP and port of the Redis + ## master node, and the quorum required to start a failover. + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + + ## Define with `sentinel down-after-milliseconds` the time in `ms` + ## that an unresponsive server will be considered down. + sentinel down-after-milliseconds gitlab-redis 10000 + + ## Define a value for `sentinel failover_timeout` in `ms`. This has multiple + ## meanings: + ## + ## * The time needed to re-start a failover after a previous failover was + ## already tried against the same master by a given Sentinel, is two + ## times the failover timeout. + ## + ## * The time needed for a slave replicating to a wrong master according + ## to a Sentinel current configuration, to be forced to replicate + ## with the right master, is exactly the failover timeout (counting since + ## the moment a Sentinel detected the misconfiguration). + ## + ## * The time needed to cancel a failover that is already in progress but + ## did not produced any configuration change (SLAVEOF NO ONE yet not + ## acknowledged by the promoted slave). + ## + ## * The maximum time a failover in progress waits for all the slaves to be + ## reconfigured as slaves of the new master. However even after this time + ## the slaves will be reconfigured by the Sentinels anyway, but not with + ## the exact parallel-syncs progression as specified. + sentinel failover_timeout 30000 + ``` + 1. Restart the Redis service for the changes to take effect. 1. Go through the steps again for all the other Sentinel nodes. @@ -180,21 +181,21 @@ setup: [resque.yml.example][resque], and uncomment the Sentinel lines, pointing to the correct server credentials: - ```yaml - # resque.yaml - production: - url: redis://:redi-password-goes-here@gitlab-redis/ - sentinels: - - - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port - ``` + ```yaml + # resque.yaml + production: + url: redis://:redi-password-goes-here@gitlab-redis/ + sentinels: + - + host: 10.0.0.1 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.2 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.3 + port: 26379 # point to sentinel, not to redis port + ``` 1. [Restart GitLab][restart] for the changes to take effect. @@ -232,23 +233,23 @@ or a failover promotes a different **Master** node. 1. In `/etc/redis/redis.conf`: - ```conf - bind 10.0.0.1 - port 6379 - requirepass redis-password-goes-here - masterauth redis-password-goes-here - ``` + ```conf + bind 10.0.0.1 + port 6379 + requirepass redis-password-goes-here + masterauth redis-password-goes-here + ``` 1. In `/etc/redis/sentinel.conf`: - ```conf - bind 10.0.0.1 - port 26379 - sentinel auth-pass gitlab-redis redis-password-goes-here - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - sentinel down-after-milliseconds gitlab-redis 10000 - sentinel failover_timeout 30000 - ``` + ```conf + bind 10.0.0.1 + port 26379 + sentinel auth-pass gitlab-redis redis-password-goes-here + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + sentinel down-after-milliseconds gitlab-redis 10000 + sentinel failover_timeout 30000 + ``` 1. Restart the Redis service for the changes to take effect. @@ -256,24 +257,24 @@ or a failover promotes a different **Master** node. 1. In `/etc/redis/redis.conf`: - ```conf - bind 10.0.0.2 - port 6379 - requirepass redis-password-goes-here - masterauth redis-password-goes-here - slaveof 10.0.0.1 6379 - ``` + ```conf + bind 10.0.0.2 + port 6379 + requirepass redis-password-goes-here + masterauth redis-password-goes-here + slaveof 10.0.0.1 6379 + ``` 1. In `/etc/redis/sentinel.conf`: - ```conf - bind 10.0.0.2 - port 26379 - sentinel auth-pass gitlab-redis redis-password-goes-here - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - sentinel down-after-milliseconds gitlab-redis 10000 - sentinel failover_timeout 30000 - ``` + ```conf + bind 10.0.0.2 + port 26379 + sentinel auth-pass gitlab-redis redis-password-goes-here + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + sentinel down-after-milliseconds gitlab-redis 10000 + sentinel failover_timeout 30000 + ``` 1. Restart the Redis service for the changes to take effect. @@ -281,24 +282,24 @@ or a failover promotes a different **Master** node. 1. In `/etc/redis/redis.conf`: - ```conf - bind 10.0.0.3 - port 6379 - requirepass redis-password-goes-here - masterauth redis-password-goes-here - slaveof 10.0.0.1 6379 - ``` + ```conf + bind 10.0.0.3 + port 6379 + requirepass redis-password-goes-here + masterauth redis-password-goes-here + slaveof 10.0.0.1 6379 + ``` 1. In `/etc/redis/sentinel.conf`: - ```conf - bind 10.0.0.3 - port 26379 - sentinel auth-pass gitlab-redis redis-password-goes-here - sentinel monitor gitlab-redis 10.0.0.1 6379 2 - sentinel down-after-milliseconds gitlab-redis 10000 - sentinel failover_timeout 30000 - ``` + ```conf + bind 10.0.0.3 + port 26379 + sentinel auth-pass gitlab-redis redis-password-goes-here + sentinel monitor gitlab-redis 10.0.0.1 6379 2 + sentinel down-after-milliseconds gitlab-redis 10000 + sentinel failover_timeout 30000 + ``` 1. Restart the Redis service for the changes to take effect. @@ -306,20 +307,20 @@ or a failover promotes a different **Master** node. 1. Edit `/home/git/gitlab/config/resque.yml`: - ```yaml - production: - url: redis://:redi-password-goes-here@gitlab-redis/ - sentinels: - - - host: 10.0.0.1 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.2 - port: 26379 # point to sentinel, not to redis port - - - host: 10.0.0.3 - port: 26379 # point to sentinel, not to redis port - ``` + ```yaml + production: + url: redis://:redi-password-goes-here@gitlab-redis/ + sentinels: + - + host: 10.0.0.1 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.2 + port: 26379 # point to sentinel, not to redis port + - + host: 10.0.0.3 + port: 26379 # point to sentinel, not to redis port + ``` 1. [Restart GitLab][restart] for the changes to take effect. diff --git a/doc/administration/integration/plantuml.md b/doc/administration/integration/plantuml.md index 7f0ec82248d..6dab46cbe6f 100644 --- a/doc/administration/integration/plantuml.md +++ b/doc/administration/integration/plantuml.md @@ -69,11 +69,12 @@ our AsciiDoc snippets, wikis and repos using delimited blocks: - **Markdown** <pre> + ````markdown ```plantuml Bob -> Alice : hello Alice -> Bob : Go Away - ```</pre> - + ``` + ```` - **AsciiDoc** diff --git a/doc/administration/operations/filesystem_benchmarking.md b/doc/administration/operations/filesystem_benchmarking.md index 4a6e22bdb84..b5922d9d99d 100644 --- a/doc/administration/operations/filesystem_benchmarking.md +++ b/doc/administration/operations/filesystem_benchmarking.md @@ -106,9 +106,9 @@ important metric is the `real` time. ```sh $ time for i in {0..1000}; do echo 'test' > "test${i}.txt"; done -real 0m0.116s -user 0m0.025s -sys 0m0.091s +real 0m0.116s +user 0m0.025s +sys 0m0.091s $ time for i in {0..1000}; do cat "test${i}.txt" > /dev/null; done diff --git a/doc/administration/pseudonymizer.md b/doc/administration/pseudonymizer.md index 78b2751da13..716a4259a64 100644 --- a/doc/administration/pseudonymizer.md +++ b/doc/administration/pseudonymizer.md @@ -65,8 +65,8 @@ To configure the pseudonymizer, you need to: ```yaml pseudonymizer: - manifest: config/pseudonymizer.yml - upload: + manifest: config/pseudonymizer.yml + upload: remote_directory: 'gitlab-elt' # bucket name connection: provider: AWS diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index c6529812ec3..277d42d06c6 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -1,11 +1,10 @@ # Uploads administration ->**Notes:** Uploads represent all user data that may be sent to GitLab as a single file. As an example, avatars and notes' attachments are uploads. Uploads are integral to GitLab functionality, and therefore cannot be disabled. ## Using local storage ->**Notes:** +NOTE: **Note:** This is the default configuration To change the location where the uploads are stored locally, follow the steps @@ -15,7 +14,7 @@ below. **In Omnibus installations:** ->**Notes:** +NOTE: **Note:** For historical reasons, uploads are stored into a base directory, which by default is `uploads/-/system`. It is strongly discouraged to change this configuration option on an existing GitLab installation. _The uploads are stored by default in `/var/opt/gitlab/gitlab-rails/uploads`._ @@ -42,8 +41,8 @@ _The uploads are stored by default in ```yaml uploads: - storage_path: /mnt/storage - base_dir: uploads + storage_path: /mnt/storage + base_dir: uploads ``` 1. Save the file and [restart GitLab][] for the changes to take effect. @@ -108,8 +107,8 @@ _The uploads are stored by default in } ``` - >**Note:** - >If you are using AWS IAM profiles, be sure to omit the AWS access key and secret access key/value pairs. + NOTE: **Note:** + If you are using AWS IAM profiles, be sure to omit the AWS access key and secret access key/value pairs. ```ruby gitlab_rails['uploads_object_store_connection'] = { diff --git a/doc/api/graphql/index.md b/doc/api/graphql/index.md index 4e4b9418e47..bdc7c1959d2 100644 --- a/doc/api/graphql/index.md +++ b/doc/api/graphql/index.md @@ -47,6 +47,12 @@ info about multiplexed queries is also available for [graphql-ruby](https://graphql-ruby.org/queries/multiplex.html) the library GitLab uses on the backend. +## Reference + +GitLab's GraphQL reference [is available](reference/index.md). + +It is automatically generated from GitLab's GraphQL schema and embedded in a Markdown file. + ## GraphiQL The API can be explored by using the GraphiQL IDE, it is available on your diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md new file mode 100644 index 00000000000..2d3bec4ff67 --- /dev/null +++ b/doc/api/graphql/reference/index.md @@ -0,0 +1,507 @@ +<!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check compile_docs task on lib/tasks/gitlab/graphql.rake. +---> + +# GraphQL API Resources + +This documentation is self-generated based on GitLab current GraphQL schema. + +The API can be explored interactively using the [GraphiQL IDE](../index.md#graphiql). + +## Objects + +### AddAwardEmojiPayload + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Reasons why the mutation failed. | +| `awardEmoji` | AwardEmoji | The award emoji after mutation | + +### AwardEmoji + +| Name | Type | Description | +| --- | ---- | ---------- | +| `name` | String! | The emoji name | +| `description` | String! | The emoji description | +| `unicode` | String! | The emoji in unicode | +| `emoji` | String! | The emoji as an icon | +| `unicodeVersion` | String! | The unicode version for this emoji | +| `user` | User! | The user who awarded the emoji | + +### Blob + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `type` | EntryType! | | +| `path` | String! | | +| `flatPath` | String! | | +| `webUrl` | String | | +| `lfsOid` | String | | + +### Commit + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `sha` | String! | | +| `title` | String | | +| `description` | String | | +| `message` | String | | +| `authoredDate` | Time | | +| `webUrl` | String! | | +| `author` | User | | +| `latestPipeline` | Pipeline | Latest pipeline for this commit | + +### DetailedStatus + +| Name | Type | Description | +| --- | ---- | ---------- | +| `group` | String! | | +| `icon` | String! | | +| `favicon` | String! | | +| `detailsPath` | String! | | +| `hasDetails` | Boolean! | | +| `label` | String! | | +| `text` | String! | | +| `tooltip` | String! | | + +### DiffPosition + +| Name | Type | Description | +| --- | ---- | ---------- | +| `headSha` | String! | The sha of the head at the time the comment was made | +| `baseSha` | String | The merge base of the branch the comment was made on | +| `startSha` | String! | The sha of the branch being compared against | +| `filePath` | String! | The path of the file that was changed | +| `oldPath` | String | The path of the file on the start sha. | +| `newPath` | String | The path of the file on the head sha. | +| `positionType` | DiffPositionType! | | +| `oldLine` | Int | The line on start sha that was changed | +| `newLine` | Int | The line on head sha that was changed | +| `x` | Int | The X postion on which the comment was made | +| `y` | Int | The Y position on which the comment was made | +| `width` | Int | The total width of the image | +| `height` | Int | The total height of the image | + +### Discussion + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `createdAt` | Time! | | + +### Group + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `path` | String! | | +| `fullName` | String! | | +| `fullPath` | ID! | | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `visibility` | String | | +| `lfsEnabled` | Boolean | | +| `requestAccessEnabled` | Boolean | | +| `userPermissions` | GroupPermissions! | Permissions for the current user on the resource | +| `webUrl` | String! | | +| `avatarUrl` | String | | +| `parent` | Group | | + +### GroupPermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `readGroup` | Boolean! | Whether or not a user can perform `read_group` on this resource | + +### Issue + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | IssuePermissions! | Permissions for the current user on the resource | +| `iid` | ID! | | +| `title` | String! | | +| `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `state` | IssueState! | | +| `reference` | String! | | +| `author` | User! | | +| `milestone` | Milestone | | +| `dueDate` | Time | | +| `confidential` | Boolean! | | +| `discussionLocked` | Boolean! | | +| `upvotes` | Int! | | +| `downvotes` | Int! | | +| `userNotesCount` | Int! | | +| `webPath` | String! | | +| `webUrl` | String! | | +| `relativePosition` | Int | | +| `closedAt` | Time | | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | +| `taskCompletionStatus` | TaskCompletionStatus! | | + +### IssuePermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `readIssue` | Boolean! | Whether or not a user can perform `read_issue` on this resource | +| `adminIssue` | Boolean! | Whether or not a user can perform `admin_issue` on this resource | +| `updateIssue` | Boolean! | Whether or not a user can perform `update_issue` on this resource | +| `createNote` | Boolean! | Whether or not a user can perform `create_note` on this resource | +| `reopenIssue` | Boolean! | Whether or not a user can perform `reopen_issue` on this resource | + +### Label + +| Name | Type | Description | +| --- | ---- | ---------- | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `title` | String! | | +| `color` | String! | | +| `textColor` | String! | | + +### MergeRequest + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | MergeRequestPermissions! | Permissions for the current user on the resource | +| `id` | ID! | | +| `iid` | String! | | +| `title` | String! | | +| `titleHtml` | String | The GitLab Flavored Markdown rendering of `title` | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `state` | MergeRequestState! | | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | +| `sourceProject` | Project | | +| `targetProject` | Project! | | +| `project` | Project! | | +| `projectId` | Int! | | +| `sourceProjectId` | Int | | +| `targetProjectId` | Int! | | +| `sourceBranch` | String! | | +| `targetBranch` | String! | | +| `workInProgress` | Boolean! | | +| `mergeWhenPipelineSucceeds` | Boolean | | +| `diffHeadSha` | String | | +| `mergeCommitSha` | String | | +| `userNotesCount` | Int | | +| `shouldRemoveSourceBranch` | Boolean | | +| `forceRemoveSourceBranch` | Boolean | | +| `mergeStatus` | String | | +| `inProgressMergeCommitSha` | String | | +| `mergeError` | String | | +| `allowCollaboration` | Boolean | | +| `shouldBeRebased` | Boolean! | | +| `rebaseCommitSha` | String | | +| `rebaseInProgress` | Boolean! | | +| `mergeCommitMessage` | String | | +| `defaultMergeCommitMessage` | String | | +| `mergeOngoing` | Boolean! | | +| `sourceBranchExists` | Boolean! | | +| `mergeableDiscussionsState` | Boolean | | +| `webUrl` | String | | +| `upvotes` | Int! | | +| `downvotes` | Int! | | +| `subscribed` | Boolean! | | +| `headPipeline` | Pipeline | | +| `taskCompletionStatus` | TaskCompletionStatus! | | + +### MergeRequestPermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `readMergeRequest` | Boolean! | Whether or not a user can perform `read_merge_request` on this resource | +| `adminMergeRequest` | Boolean! | Whether or not a user can perform `admin_merge_request` on this resource | +| `updateMergeRequest` | Boolean! | Whether or not a user can perform `update_merge_request` on this resource | +| `createNote` | Boolean! | Whether or not a user can perform `create_note` on this resource | +| `pushToSourceBranch` | Boolean! | Whether or not a user can perform `push_to_source_branch` on this resource | +| `removeSourceBranch` | Boolean! | Whether or not a user can perform `remove_source_branch` on this resource | +| `cherryPickOnCurrentMergeRequest` | Boolean! | Whether or not a user can perform `cherry_pick_on_current_merge_request` on this resource | +| `revertOnCurrentMergeRequest` | Boolean! | Whether or not a user can perform `revert_on_current_merge_request` on this resource | + +### MergeRequestSetWipPayload + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Reasons why the mutation failed. | +| `mergeRequest` | MergeRequest | The merge request after mutation | + +### Metadata + +| Name | Type | Description | +| --- | ---- | ---------- | +| `version` | String! | | +| `revision` | String! | | + +### Milestone + +| Name | Type | Description | +| --- | ---- | ---------- | +| `description` | String | | +| `title` | String! | | +| `state` | String! | | +| `dueDate` | Time | | +| `startDate` | Time | | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | + +### Namespace + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `path` | String! | | +| `fullName` | String! | | +| `fullPath` | ID! | | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `visibility` | String | | +| `lfsEnabled` | Boolean | | +| `requestAccessEnabled` | Boolean | | + +### Note + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | NotePermissions! | Permissions for the current user on the resource | +| `id` | ID! | | +| `project` | Project | The project this note is associated to | +| `author` | User! | The user who wrote this note | +| `resolvedBy` | User | The user that resolved the discussion | +| `system` | Boolean! | Whether or not this note was created by the system or by a user | +| `body` | String! | The content note itself | +| `bodyHtml` | String | The GitLab Flavored Markdown rendering of `note` | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | +| `discussion` | Discussion | The discussion this note is a part of | +| `resolvable` | Boolean! | | +| `resolvedAt` | Time | The time the discussion was resolved | +| `position` | DiffPosition | The position of this note on a diff | + +### NotePermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `readNote` | Boolean! | Whether or not a user can perform `read_note` on this resource | +| `createNote` | Boolean! | Whether or not a user can perform `create_note` on this resource | +| `adminNote` | Boolean! | Whether or not a user can perform `admin_note` on this resource | +| `resolveNote` | Boolean! | Whether or not a user can perform `resolve_note` on this resource | +| `awardEmoji` | Boolean! | Whether or not a user can perform `award_emoji` on this resource | + +### PageInfo + +| Name | Type | Description | +| --- | ---- | ---------- | +| `hasNextPage` | Boolean! | When paginating forwards, are there more items? | +| `hasPreviousPage` | Boolean! | When paginating backwards, are there more items? | +| `startCursor` | String | When paginating backwards, the cursor to continue. | +| `endCursor` | String | When paginating forwards, the cursor to continue. | + +### Pipeline + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | PipelinePermissions! | Permissions for the current user on the resource | +| `id` | ID! | | +| `iid` | String! | | +| `sha` | String! | | +| `beforeSha` | String | | +| `status` | PipelineStatusEnum! | | +| `detailedStatus` | DetailedStatus! | | +| `duration` | Int | Duration of the pipeline in seconds | +| `coverage` | Float | Coverage percentage | +| `createdAt` | Time! | | +| `updatedAt` | Time! | | +| `startedAt` | Time | | +| `finishedAt` | Time | | +| `committedAt` | Time | | + +### PipelinePermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `updatePipeline` | Boolean! | Whether or not a user can perform `update_pipeline` on this resource | +| `adminPipeline` | Boolean! | Whether or not a user can perform `admin_pipeline` on this resource | +| `destroyPipeline` | Boolean! | Whether or not a user can perform `destroy_pipeline` on this resource | + +### Project + +| Name | Type | Description | +| --- | ---- | ---------- | +| `userPermissions` | ProjectPermissions! | Permissions for the current user on the resource | +| `id` | ID! | | +| `fullPath` | ID! | | +| `path` | String! | | +| `nameWithNamespace` | String! | | +| `name` | String! | | +| `description` | String | | +| `descriptionHtml` | String | The GitLab Flavored Markdown rendering of `description` | +| `tagList` | String | | +| `sshUrlToRepo` | String | | +| `httpUrlToRepo` | String | | +| `webUrl` | String | | +| `starCount` | Int! | | +| `forksCount` | Int! | | +| `createdAt` | Time | | +| `lastActivityAt` | Time | | +| `archived` | Boolean | | +| `visibility` | String | | +| `containerRegistryEnabled` | Boolean | | +| `sharedRunnersEnabled` | Boolean | | +| `lfsEnabled` | Boolean | | +| `mergeRequestsFfOnlyEnabled` | Boolean | | +| `avatarUrl` | String | | +| `issuesEnabled` | Boolean | | +| `mergeRequestsEnabled` | Boolean | | +| `wikiEnabled` | Boolean | | +| `snippetsEnabled` | Boolean | | +| `jobsEnabled` | Boolean | | +| `publicJobs` | Boolean | | +| `openIssuesCount` | Int | | +| `importStatus` | String | | +| `onlyAllowMergeIfPipelineSucceeds` | Boolean | | +| `requestAccessEnabled` | Boolean | | +| `onlyAllowMergeIfAllDiscussionsAreResolved` | Boolean | | +| `printingMergeRequestLinkEnabled` | Boolean | | +| `namespace` | Namespace | | +| `group` | Group | | +| `statistics` | ProjectStatistics | | +| `repository` | Repository | | +| `mergeRequest` | MergeRequest | | +| `issue` | Issue | | + +### ProjectPermissions + +| Name | Type | Description | +| --- | ---- | ---------- | +| `changeNamespace` | Boolean! | Whether or not a user can perform `change_namespace` on this resource | +| `changeVisibilityLevel` | Boolean! | Whether or not a user can perform `change_visibility_level` on this resource | +| `renameProject` | Boolean! | Whether or not a user can perform `rename_project` on this resource | +| `removeProject` | Boolean! | Whether or not a user can perform `remove_project` on this resource | +| `archiveProject` | Boolean! | Whether or not a user can perform `archive_project` on this resource | +| `removeForkProject` | Boolean! | Whether or not a user can perform `remove_fork_project` on this resource | +| `removePages` | Boolean! | Whether or not a user can perform `remove_pages` on this resource | +| `readProject` | Boolean! | Whether or not a user can perform `read_project` on this resource | +| `createMergeRequestIn` | Boolean! | Whether or not a user can perform `create_merge_request_in` on this resource | +| `readWiki` | Boolean! | Whether or not a user can perform `read_wiki` on this resource | +| `readProjectMember` | Boolean! | Whether or not a user can perform `read_project_member` on this resource | +| `createIssue` | Boolean! | Whether or not a user can perform `create_issue` on this resource | +| `uploadFile` | Boolean! | Whether or not a user can perform `upload_file` on this resource | +| `readCycleAnalytics` | Boolean! | Whether or not a user can perform `read_cycle_analytics` on this resource | +| `downloadCode` | Boolean! | Whether or not a user can perform `download_code` on this resource | +| `downloadWikiCode` | Boolean! | Whether or not a user can perform `download_wiki_code` on this resource | +| `forkProject` | Boolean! | Whether or not a user can perform `fork_project` on this resource | +| `createProjectSnippet` | Boolean! | Whether or not a user can perform `create_project_snippet` on this resource | +| `readCommitStatus` | Boolean! | Whether or not a user can perform `read_commit_status` on this resource | +| `requestAccess` | Boolean! | Whether or not a user can perform `request_access` on this resource | +| `createPipeline` | Boolean! | Whether or not a user can perform `create_pipeline` on this resource | +| `createPipelineSchedule` | Boolean! | Whether or not a user can perform `create_pipeline_schedule` on this resource | +| `createMergeRequestFrom` | Boolean! | Whether or not a user can perform `create_merge_request_from` on this resource | +| `createWiki` | Boolean! | Whether or not a user can perform `create_wiki` on this resource | +| `pushCode` | Boolean! | Whether or not a user can perform `push_code` on this resource | +| `createDeployment` | Boolean! | Whether or not a user can perform `create_deployment` on this resource | +| `pushToDeleteProtectedBranch` | Boolean! | Whether or not a user can perform `push_to_delete_protected_branch` on this resource | +| `adminWiki` | Boolean! | Whether or not a user can perform `admin_wiki` on this resource | +| `adminProject` | Boolean! | Whether or not a user can perform `admin_project` on this resource | +| `updatePages` | Boolean! | Whether or not a user can perform `update_pages` on this resource | +| `adminRemoteMirror` | Boolean! | Whether or not a user can perform `admin_remote_mirror` on this resource | +| `createLabel` | Boolean! | Whether or not a user can perform `create_label` on this resource | +| `updateWiki` | Boolean! | Whether or not a user can perform `update_wiki` on this resource | +| `destroyWiki` | Boolean! | Whether or not a user can perform `destroy_wiki` on this resource | +| `createPages` | Boolean! | Whether or not a user can perform `create_pages` on this resource | +| `destroyPages` | Boolean! | Whether or not a user can perform `destroy_pages` on this resource | +| `readPagesContent` | Boolean! | Whether or not a user can perform `read_pages_content` on this resource | + +### ProjectStatistics + +| Name | Type | Description | +| --- | ---- | ---------- | +| `commitCount` | Int! | | +| `storageSize` | Int! | | +| `repositorySize` | Int! | | +| `lfsObjectsSize` | Int! | | +| `buildArtifactsSize` | Int! | | +| `packagesSize` | Int! | | +| `wikiSize` | Int | | + +### RemoveAwardEmojiPayload + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Reasons why the mutation failed. | +| `awardEmoji` | AwardEmoji | The award emoji after mutation | + +### Repository + +| Name | Type | Description | +| --- | ---- | ---------- | +| `rootRef` | String | | +| `empty` | Boolean! | | +| `exists` | Boolean! | | +| `tree` | Tree | | + +### Submodule + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `type` | EntryType! | | +| `path` | String! | | +| `flatPath` | String! | | + +### TaskCompletionStatus + +| Name | Type | Description | +| --- | ---- | ---------- | +| `count` | Int! | | +| `completedCount` | Int! | | + +### ToggleAwardEmojiPayload + +| Name | Type | Description | +| --- | ---- | ---------- | +| `clientMutationId` | String | A unique identifier for the client performing the mutation. | +| `errors` | String! => Array | Reasons why the mutation failed. | +| `awardEmoji` | AwardEmoji | The award emoji after mutation | +| `toggledOn` | Boolean! | True when the emoji was awarded, false when it was removed | + +### Tree + +| Name | Type | Description | +| --- | ---- | ---------- | +| `lastCommit` | Commit | | + +### TreeEntry + +| Name | Type | Description | +| --- | ---- | ---------- | +| `id` | ID! | | +| `name` | String! | | +| `type` | EntryType! | | +| `path` | String! | | +| `flatPath` | String! | | +| `webUrl` | String | | + +### User + +| Name | Type | Description | +| --- | ---- | ---------- | +| `name` | String! | | +| `username` | String! | | +| `avatarUrl` | String! | | +| `webUrl` | String! | | + diff --git a/doc/api/repositories.md b/doc/api/repositories.md index ffae5c17310..4aff79c9c62 100644 --- a/doc/api/repositories.md +++ b/doc/api/repositories.md @@ -235,17 +235,17 @@ Example response: ```json { - "id": "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863", - "short_id": "1a0b36b3", - "title": "Initial commit", - "created_at": "2014-02-27T08:03:18.000Z", - "parent_ids": [], - "message": "Initial commit\n", - "author_name": "Dmitriy Zaporozhets", - "author_email": "dmitriy.zaporozhets@gmail.com", - "authored_date": "2014-02-27T08:03:18.000Z", - "committer_name": "Dmitriy Zaporozhets", - "committer_email": "dmitriy.zaporozhets@gmail.com", - "committed_date": "2014-02-27T08:03:18.000Z" + "id": "1a0b36b3cdad1d2ee32457c102a8c0b7056fa863", + "short_id": "1a0b36b3", + "title": "Initial commit", + "created_at": "2014-02-27T08:03:18.000Z", + "parent_ids": [], + "message": "Initial commit\n", + "author_name": "Dmitriy Zaporozhets", + "author_email": "dmitriy.zaporozhets@gmail.com", + "authored_date": "2014-02-27T08:03:18.000Z", + "committer_name": "Dmitriy Zaporozhets", + "committer_email": "dmitriy.zaporozhets@gmail.com", + "committed_date": "2014-02-27T08:03:18.000Z" } ``` diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 42dd4f08ed8..4d6ca8cff6d 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -273,6 +273,7 @@ export CI_RUNNER_ID="10" export CI_RUNNER_DESCRIPTION="my runner" export CI_RUNNER_TAGS="docker, linux" export CI_SERVER="yes" +export CI_SERVER_HOST="example.com" export CI_SERVER_NAME="GitLab" export CI_SERVER_REVISION="70606bf" export CI_SERVER_VERSION="8.9.0" @@ -644,6 +645,8 @@ Running on runner-8a2f473d-project-1796893-concurrent-0 via runner-8a2f473d-mach ++ CI_PROJECT_DIR=/builds/gitlab-examples/ci-debug-trace ++ export CI_SERVER=yes ++ CI_SERVER=yes +++ export 'CI_SERVER_HOST=example.com' +++ CI_SERVER_HOST='example.com' ++ export 'CI_SERVER_NAME=GitLab CI' ++ CI_SERVER_NAME='GitLab CI' ++ export CI_SERVER_VERSION= @@ -678,6 +681,8 @@ Running on runner-8a2f473d-project-1796893-concurrent-0 via runner-8a2f473d-mach ++ CI_JOB_NAME=debug_trace ++ export CI_JOB_STAGE=test ++ CI_JOB_STAGE=test +++ export CI_SERVER_HOST=example.com +++ CI_SERVER_HOST=example.com ++ export CI_SERVER_NAME=GitLab ++ CI_SERVER_NAME=GitLab ++ export CI_SERVER_VERSION=8.14.3-ee diff --git a/doc/ci/variables/predefined_variables.md b/doc/ci/variables/predefined_variables.md index e911e97d3c8..49543c57886 100644 --- a/doc/ci/variables/predefined_variables.md +++ b/doc/ci/variables/predefined_variables.md @@ -101,6 +101,7 @@ future GitLab releases.** | `CI_RUNNER_TAGS` | 8.10 | 0.5 | The defined runner tags | | `CI_RUNNER_VERSION` | all | 10.6 | GitLab Runner version that is executing the current job | | `CI_SERVER` | all | all | Mark that job is executed in CI environment | +| `CI_SERVER_HOST` | 12.1 | all | Host component of the GitLab instance URL, without protocol and port (like gitlab.example.com) | | `CI_SERVER_NAME` | all | all | The name of CI server that is used to coordinate jobs | | `CI_SERVER_REVISION` | all | all | GitLab revision that is used to schedule jobs | | `CI_SERVER_VERSION` | all | all | GitLab version that is used to schedule jobs | diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 474db05de06..c2ef58acf15 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -2829,7 +2829,8 @@ Alternatively, one can pass the `ci.skip` [Git push option][push-option] if using Git 2.10 or newer: ```sh -git push -o ci.skip +git push --push-option=ci.skip # using git 2.10+ +git push -o ci.skip # using git 2.18+ ``` <!-- ## Troubleshooting diff --git a/doc/development/README.md b/doc/development/README.md index 1566173992a..a74770ae383 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -22,7 +22,7 @@ description: 'Learn how to contribute to GitLab.' - [Security process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) - [Requesting access to Chatops on GitLab.com](chatops_on_gitlabcom.md#requesting-access) (for GitLabbers) -## UX and frontend guides +## UX and Frontend guides - [GitLab Design System](https://design.gitlab.com/) for building GitLab with existing CSS styles and elements - [Frontend guidelines](fe_guide/index.md) diff --git a/doc/development/chaos_endpoints.md b/doc/development/chaos_endpoints.md index 403a5b21827..b3406275937 100644 --- a/doc/development/chaos_endpoints.md +++ b/doc/development/chaos_endpoints.md @@ -15,23 +15,19 @@ Currently, there are four endpoints for simulating the following conditions: ## Enabling chaos endpoints -For obvious reasons, these endpoints are not enabled by default. They can be enabled by setting the `GITLAB_ENABLE_CHAOS_ENDPOINTS` environment variable to `1`. - -For example, if you're using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) this can be done with the following command: - -```bash -GITLAB_ENABLE_CHAOS_ENDPOINTS=1 gdk run -``` - -## Securing the chaos endpoints +For obvious reasons, these endpoints are not enabled by default on `production`. +They are enabled by default on **development** environments. DANGER: **Danger:** -It is highly recommended that you secure access to the chaos endpoints using a secret token. This is recommended when enabling these endpoints locally and essential when running in a staging or other shared environment. You should not enable them in production unless you absolutely know what you're doing. +It is required that you secure access to the chaos endpoints using a secret token. +You should not enable them in production unless you absolutely know what you're doing. -A secret token can be set through the `GITLAB_CHAOS_SECRET` environment variable. For example, when using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) this can be done with the following command: +A secret token can be set through the `GITLAB_CHAOS_SECRET` environment variable. +For example, when using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) +this can be done with the following command: ```bash -GITLAB_ENABLE_CHAOS_ENDPOINTS=1 GITLAB_CHAOS_SECRET=secret gdk run +GITLAB_CHAOS_SECRET=secret gdk run ``` Replace `secret` with your own secret token. @@ -56,10 +52,11 @@ GET /-/chaos/leakmem?memory_mb=1024&duration_s=50 | Attribute | Type | Required | Description | | ------------ | ------- | -------- | ---------------------------------------------------------------------------------- | | `memory_mb` | integer | no | How much memory, in MB, should be leaked. Defaults to 100MB. | -| `duration_s` | integer | no | Minimum duration, in seconds, that the memory should be retained. Defaults to 30s. | +| `duration_s` | integer | no | Minimum duration_s, in seconds, that the memory should be retained. Defaults to 30s. | ```bash curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024&duration_s=10 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024&duration_s=10&token=secret ``` ## CPU spin @@ -70,23 +67,47 @@ Depending on your rack server setup, your request may timeout after a predermine If you're using Unicorn, this is done by killing the worker process. ``` -GET /-/chaos/cpuspin -GET /-/chaos/cpuspin?duration_s=50 +GET /-/chaos/cpu_spin +GET /-/chaos/cpu_spin?duration_s=50 +``` + +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | --------------------------------------------------------------------- | +| `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | + +```bash +curl http://localhost:3000/-/chaos/cpu_spin?duration_s=60 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/cpu_spin?duration_s=60&token=secret +``` + +## DB spin + +This endpoint attempts to fully utilise a single core, and interleave it with DB request, for the given period. +This endpoint can be used to model yielding execution to another threads when running concurrently. + +Depending on your rack server setup, your request may timeout after a predermined period (normally 60 seconds). +If you're using Unicorn, this is done by killing the worker process. + +``` +GET /-/chaos/db_spin +GET /-/chaos/db_spin?duration_s=50 ``` | Attribute | Type | Required | Description | | ------------ | ------- | -------- | --------------------------------------------------------------------- | +| `interval_s` | float | no | Interval, in seconds, for every DB request. Defaults to 1s | | `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | ```bash -curl http://localhost:3000/-/chaos/cpuspin?duration_s=60 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/db_spin?interval_s=1&duration_s=60 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/db_spin?interval_s=1&duration_s=60&token=secret ``` ## Sleep -This endpoint is similar to the CPU Spin endpoint but simulates off-processor activity, such as network calls to backend services. It will sleep for a given duration. +This endpoint is similar to the CPU Spin endpoint but simulates off-processor activity, such as network calls to backend services. It will sleep for a given duration_s. -As with the CPU Spin endpoint, this may lead to your request timing out if duration exceeds the configured limit. +As with the CPU Spin endpoint, this may lead to your request timing out if duration_s exceeds the configured limit. ``` GET /-/chaos/sleep @@ -99,6 +120,7 @@ GET /-/chaos/sleep?duration_s=50 ```bash curl http://localhost:3000/-/chaos/sleep?duration_s=60 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/sleep?duration_s=60&token=secret ``` ## Kill @@ -114,4 +136,5 @@ GET /-/chaos/kill ```bash curl http://localhost:3000/-/chaos/kill --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/kill?token=secret ``` diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index 5552d5d37b4..2ade59b76ed 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -237,24 +237,23 @@ Here are the steps to gate a new feature in Gitaly behind a feature flag. 1. Create prometheus metrics: ```go - var findAllTagsRequests = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "gitaly_find_all_tags_requests_total", - Help: "Counter of go vs ruby implementation of FindAllTags", - }, - []string{"implementation"}, - ) + var findAllTagsRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "gitaly_find_all_tags_requests_total", + Help: "Counter of go vs ruby implementation of FindAllTags", + }, + []string{"implementation"}, ) func init() { - prometheus.Register(findAllTagsRequests) + prometheus.Register(findAllTagsRequests) } if featureflag.IsEnabled(ctx, findAllTagsFeatureFlag) { - findAllTagsRequests.WithLabelValues("go").Inc() + findAllTagsRequests.WithLabelValues("go").Inc() // go implementation } else { - findAllTagsRequests.WithLabelValues("ruby").Inc() + findAllTagsRequests.WithLabelValues("ruby").Inc() // ruby implementation } ``` diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 73eaf758923..fed3b1ca595 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -59,10 +59,11 @@ Here's a list of the AWS services we will use, with links to pricing information - **ElastiCache**: An in-memory cache environment will be used to provide a High Availability Redis configuration. See the [Amazon ElastiCache pricing](https://aws.amazon.com/elasticache/pricing/). - + NOTE: **Note:** Please note that while we will be using EBS for storage, we do not recommend using EFS as it may negatively impact GitLab's performance. You can review the [relevant documentation](../../administration/high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. ## Creating an IAM EC2 instance role and profile + To minimize the permissions of the user, we'll create a new [IAM](https://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html) role with limited access: @@ -90,7 +91,7 @@ We'll now create a VPC, a virtual networking environment that you'll control: `10.0.0.0/16`. If you don't require dedicated hardware, you can leave "Tenancy" as default. Click **Yes, Create** when ready. -  +  ### Subnets @@ -107,16 +108,16 @@ RDS instances as well: for example `gitlab-public-10.0.0.0`, select the VPC we created previously, and at the IPv4 CIDR block let's give it a 24 subnet `10.0.0.0/24`: -  +  1. Follow the same steps to create all subnets: - | Name tag | Type |Availability Zone | CIDR block | - | -------- | ---- | ---------------- | ---------- | - | gitlab-public-10.0.0.0 | public | us-west-2a | 10.0.0.0 | - | gitlab-private-10.0.1.0 | private | us-west-2a | 10.0.1.0 | - | gitlab-public-10.0.2.0 | public | us-west-2b | 10.0.2.0 | - | gitlab-private-10.0.3.0 | private | us-west-2b | 10.0.3.0 | + | Name tag | Type |Availability Zone | CIDR block | + | -------- | ---- | ---------------- | ---------- | + | gitlab-public-10.0.0.0 | public | us-west-2a | 10.0.0.0 | + | gitlab-private-10.0.1.0 | private | us-west-2a | 10.0.1.0 | + | gitlab-public-10.0.2.0 | public | us-west-2b | 10.0.2.0 | + | gitlab-private-10.0.3.0 | private | us-west-2b | 10.0.3.0 | ### Route Table @@ -139,7 +140,7 @@ create a new one: 1. Select it from the table, and then under the **Actions** dropdown choose "Attach to VPC". -  +  1. Choose `gitlab-vpc` from the list and hit **Attach**. @@ -154,14 +155,14 @@ it receive traffic from any destination. as destination. In the target, select the `gitlab-gateway` we created previously. Hit **Save** once done. -  +  Next, we must associate the **public** subnets to the route table: 1. Select the **Subnet Associations** tab and hit **Edit**. 1. Check only the public subnet and hit **Save**. -  +  --- @@ -178,12 +179,12 @@ The security group is basically the firewall: Inbound Rules tab. You will need to open the SSH, HTTP, and HTTPS ports. Set the source to `0.0.0.0/0`. -  +  - TIP: **Tip:** - Based on best practices, you should allow SSH traffic from only a known - host or CIDR block. In that case, change the SSH source to be custom and give - it the IP you want to SSH from. + TIP: **Tip:** + Based on best practices, you should allow SSH traffic from only a known + host or CIDR block. In that case, change the SSH source to be custom and give + it the IP you want to SSH from. 1. When done, click **Save**. @@ -204,7 +205,7 @@ create the actual RDS instance. we defined them in the [subnets section](#subnets)). Click **Create** when ready. -  +  ### Creating the database @@ -214,27 +215,27 @@ Now, it's time to create the database: 1. Select PostgreSQL and click **Next**. 1. Since this is a production server, let's choose "Production". Click **Next**. 1. Let's see the instance specifications: - 1. Leave the license model as is (`postgresql-license`). - 1. For the version, select the latest of the 9.6 series (check the - [database requirements](../../install/requirements.md#postgresql-requirements)) - if there are any updates on this). - 1. For the size, let's select a `t2.medium` instance. - 1. Multi-AZ-deployment is recommended as redundancy, so choose "Create - replica in different zone". Read more at - [High Availability (Multi-AZ)](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZ.html). - 1. A Provisioned IOPS (SSD) storage type is best suited for HA (though you can - choose a General Purpose (SSD) to reduce the costs). Read more about it at - [Storage for Amazon RDS](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html). - -1. The rest of the settings on this page request a DB isntance identifier, username + 1. Leave the license model as is (`postgresql-license`). + 1. For the version, select the latest of the 9.6 series (check the + [database requirements](../../install/requirements.md#postgresql-requirements)) + if there are any updates on this). + 1. For the size, let's select a `t2.medium` instance. + 1. Multi-AZ-deployment is recommended as redundancy, so choose "Create + replica in different zone". Read more at + [High Availability (Multi-AZ)](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.MultiAZ.html). + 1. A Provisioned IOPS (SSD) storage type is best suited for HA (though you can + choose a General Purpose (SSD) to reduce the costs). Read more about it at + [Storage for Amazon RDS](http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Storage.html). + +1. The rest of the settings on this page request a DB instance identifier, username and a master password. We've chosen to use `gitlab-db-ha`, `gitlab` and a very secure password respectively. Keep these in hand for later. 1. Click **Next** to proceed to the advanced settings. 1. Make sure to choose our gitlab VPC, our subnet group, set public accessibility to **No**, and to leave it to create a new security group. The only additional - change which will be helpful is the database name for which we can use - `gitlabhq_production`. At the very bottom, there's an option to enable - auto updates to minor versions. You may want to turn it off. + change which will be helpful is the database name for which we can use + `gitlabhq_production`. At the very bottom, there's an option to enable + auto updates to minor versions. You may want to turn it off. 1. When done, click **Create database**. ### Installing the `pg_trgm` extension for PostgreSQL @@ -276,7 +277,7 @@ To set up Redis: Make sure to select our VPC and its [private subnets](#subnets). Click **Create** when ready. -  +  1. Select **Redis** on the left menu and click **Create** to create a new Redis cluster. Depending on your load, you can choose whether to enable @@ -284,16 +285,16 @@ To set up Redis: chance to deploy Redis in multi availability zones. In this guide, we chose not to enable it. 1. In the settings section: - 1. Give the cluster a name (`gitlab-redis`) and a description. - 1. For the version, select the latest of `3.2` series (e.g., `3.2.10`). - 1. Select the node type and the number of replicas. + 1. Give the cluster a name (`gitlab-redis`) and a description. + 1. For the version, select the latest of `3.2` series (e.g., `3.2.10`). + 1. Select the node type and the number of replicas. 1. In the advanced settings section: 1. Select the multi-AZ auto-failover option. 1. Select the subnet group we created previously. 1. Manually select the preferred availability zones, and under "Replica 2" choose a different zone than the other two. -  +  1. In the security settings, edit the security groups and choose the `gitlab-security-group` we had previously created. @@ -316,11 +317,11 @@ and add a custom TCP rule for port `6379` accessible within itself. On the EC2 dashboard, look for Load Balancer on the left column: 1. Click the **Create Load Balancer** button. - 1. Choose the Application Load Balancer. - 1. Give it a name (`gitlab-loadbalancer`) and set the scheme to "internet-facing". - 1. In the "Listeners" section, make sure it has HTTP and HTTPS. - 1. In the "Availability Zones" section, select the `gitlab-vpc` we have created - and associate the **public subnets**. + 1. Choose the Application Load Balancer. + 1. Give it a name (`gitlab-loadbalancer`) and set the scheme to "internet-facing". + 1. In the "Listeners" section, make sure it has HTTP and HTTPS. + 1. In the "Availability Zones" section, select the `gitlab-vpc` we have created + and associate the **public subnets**. 1. Click **Configure Security Settings** to go to the next section to select the TLS certificate. When done, go to the next step. 1. In the "Security Groups" section, create a new one by giving it a name @@ -355,7 +356,7 @@ Choose the AMI: where `<version>` the latest version as seen on the [releases page](https://about.gitlab.com/releases/). -  +  ### Choose an instance type @@ -504,19 +505,19 @@ The EBS volume will host the Git repositories data: 1. Tell GitLab to store its data in the new directory by editing `/etc/gitlab/gitlab.rb` with your editor: - ```ruby - git_data_dirs({ - "default" => { "path" => "/mnt/gitlab-data" } - }) - ``` + ```ruby + git_data_dirs({ + "default" => { "path" => "/mnt/gitlab-data" } + }) + ``` - where `/mnt/gitlab-data` the location where you will store the Git data. + where `/mnt/gitlab-data` the location where you will store the Git data. 1. Save the file and reconfigure GitLab: - ```sh - sudo gitlab-ctl reconfigure - ``` + ```sh + sudo gitlab-ctl reconfigure + ``` TIP: **Tip:** If you wish to add more than one data volumes to store the Git repositories, @@ -549,15 +550,15 @@ After you SSH into the instance, configure the domain name: 1. Open `/etc/gitlab/gitlab.rb` with your preferred editor. 1. Edit the `external_url` value: - ```ruby - external_url 'http://example.com' - ``` + ```ruby + external_url 'http://example.com' + ``` 1. Reconfigure GitLab: - ```sh - sudo gitlab-ctl reconfigure - ``` + ```sh + sudo gitlab-ctl reconfigure + ``` You should now be able to reach GitLab at the URL you defined. To use HTTPS (recommended), see the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). @@ -608,9 +609,9 @@ To back up GitLab: 1. SSH into your instance. 1. Take a backup: - ```sh - sudo gitlab-rake gitlab:backup:create - ``` + ```sh + sudo gitlab-rake gitlab:backup:create + ``` ### Restoring GitLab from a backup @@ -626,16 +627,16 @@ released, you can update your GitLab instance: 1. SSH into your instance 1. Take a backup: - ```sh - sudo gitlab-rake gitlab:backup:create - ``` + ```sh + sudo gitlab-rake gitlab:backup:create + ``` 1. Update the repositories and install GitLab: - ```sh - sudo apt update - sudo apt install gitlab-ee - ``` + ```sh + sudo apt update + sudo apt install gitlab-ee + ``` After a few minutes, the new version should be up and running. diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index b1f79893baf..c0e1b0ebbc8 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -67,18 +67,19 @@ The first items we need to configure are the basic settings of the underlying vi 1. Enter a `User name` - e.g. **"gitlab-admin"** 1. Select an `Authentication type`, either **SSH public key** or **Password**: - > **Note:** if you're unsure which authentication type to use, select **Password** + > **Note:** if you're unsure which authentication type to use, select **Password** + + 1. If you chose **SSH public key** - enter your `SSH public key` into the field provided + _(read the [SSH documentation][GitLab-Docs-SSH] to learn more about how to set up SSH + public keys)_ + 1. If you chose **Password** - enter the password you wish to use _(this is the password that you + will use later in this tutorial to [SSH] into the VM, so make sure it's a strong password/passphrase)_ - 1. If you chose **SSH public key** - enter your `SSH public key` into the field provided - _(read the [SSH documentation][GitLab-Docs-SSH] to learn more about how to set up SSH - public keys)_ - 1. If you chose **Password** - enter the password you wish to use _(this is the password that you - will use later in this tutorial to [SSH] into the VM, so make sure it's a strong password/passphrase)_ 1. Choose the appropriate `Subscription` tier for your Azure account 1. Choose an existing `Resource Group` or create a new one - e.g. **"GitLab-CE-Azure"** - > **Note:** a "Resource group" is a way to group related resources together for easier administration. - > We chose "GitLab-CE-Azure", but your resource group can have the same name as your VM. + > **Note:** a "Resource group" is a way to group related resources together for easier administration. + > We chose "GitLab-CE-Azure", but your resource group can have the same name as your VM. 1. Choose a `Location` - if you're unsure, select the default location @@ -248,6 +249,7 @@ rules in the list:  ## Connecting to GitLab + Use the domain name you set up earlier (or the public IP address) to visit your new GitLab instance in your browser. If everything has gone according to plan you should be presented with the following page, asking you to set a _new_ password for the administrator account automatically @@ -348,6 +350,7 @@ your VM, you can use the IP address in its place in the following command: ```bash ssh username@your-azure-domain-name.com ``` + Provide your password at the prompt to authenticate. #### SSH from Windows (PuTTY) @@ -411,12 +414,12 @@ Check out our other [Technical Articles][GitLab-Technical-Articles] or browse th - [GitLab Community Edition][CE] - [GitLab Enterprise Edition][EE] - [Microsoft Azure][Azure] - - [Azure - Free Account FAQ][Azure-Free-Account-FAQ] - - [Azure - Marketplace][Azure-Marketplace] - - [Azure Portal][Azure-Portal] - - [Azure - Pricing Calculator][Azure-Pricing-Calculator] - - [Azure - Troubleshoot SSH Connections to an Azure Linux VM][Azure-Troubleshoot-SSH-Connection] - - [Azure - Properly Shutdown an Azure VM][Azure-Properly-Shutdown-VM] + - [Azure - Free Account FAQ][Azure-Free-Account-FAQ] + - [Azure - Marketplace][Azure-Marketplace] + - [Azure Portal][Azure-Portal] + - [Azure - Pricing Calculator][Azure-Pricing-Calculator] + - [Azure - Troubleshoot SSH Connections to an Azure Linux VM][Azure-Troubleshoot-SSH-Connection] + - [Azure - Properly Shutdown an Azure VM][Azure-Properly-Shutdown-VM] - [SSH], [PuTTY] and [Using SSH in PuTTY][Using-SSH-In-Putty] [Original-Blog-Post]: https://about.gitlab.com/2016/07/13/how-to-setup-a-gitlab-instance-on-microsoft-azure/ "How to Set up a GitLab Instance on Microsoft Azure" diff --git a/doc/install/digitaloceandocker.md b/doc/install/digitaloceandocker.md index 63bb941ad47..b6bf7c95527 100644 --- a/doc/install/digitaloceandocker.md +++ b/doc/install/digitaloceandocker.md @@ -36,30 +36,30 @@ The rest of the steps are identical for macOS and Linux. 1. Login to Digital Ocean. 1. Generate a new API token at <https://cloud.digitalocean.com/settings/api/tokens>. - This command will create a new DO droplet called `gitlab-test-env-do` that will act as a docker host. + This command will create a new DO droplet called `gitlab-test-env-do` that will act as a docker host. - NOTE: **Note:** - 4GB is the minimum requirement for a Docker host that will run more than one GitLab instance. + NOTE: **Note:** + 4GB is the minimum requirement for a Docker host that will run more than one GitLab instance. - - RAM: 4GB - - Name: `gitlab-test-env-do` - - Driver: `digitalocean` + - RAM: 4GB + - Name: `gitlab-test-env-do` + - Driver: `digitalocean` 1. Set the DO token: - ```sh - export DOTOKEN=<your generated token> - ``` + ```sh + export DOTOKEN=<your generated token> + ``` 1. Create the machine: - ```sh - docker-machine create \ - --driver digitalocean \ - --digitalocean-access-token=$DOTOKEN \ - --digitalocean-size "4gb" \ - gitlab-test-env-do - ``` + ```sh + docker-machine create \ + --driver digitalocean \ + --digitalocean-access-token=$DOTOKEN \ + --digitalocean-size "4gb" \ + gitlab-test-env-do + ``` Resource: <https://docs.docker.com/machine/drivers/digital-ocean/>. diff --git a/doc/install/google_cloud_platform/index.md b/doc/install/google_cloud_platform/index.md index 14bf7012c01..be29bcc7cd7 100644 --- a/doc/install/google_cloud_platform/index.md +++ b/doc/install/google_cloud_platform/index.md @@ -30,16 +30,16 @@ To deploy GitLab on GCP you first need to create a virtual machine: 1. Go to <https://console.cloud.google.com/compute/instances> and log in with your Google credentials. 1. Click on **Create** -  +  -1. On the next page, you can select the type of VM as well as the +1. On the next page, you can select the type of VM as well as the estimated costs. Provide the name of the instance, desired datacenter, and machine type. Note that GitLab recommends at least 2 vCPU's and 4GB of RAM. -  +  1. Click **Change** under Boot disk to select the size, type, and desired operating system. GitLab supports a [variety of linux operating systems][req], including Ubuntu and Debian. Click **Select** when finished. -  +  1. As a last step allow HTTP and HTTPS traffic, then click **Create**. The process will finish in a few seconds. @@ -53,13 +53,13 @@ After a few seconds, the instance will be created and available to log in. The n 1. Click on the SSH button to connect to the instance. 1. A new window will appear, with you logged into the instance. -  +  1. Next, follow the instructions for installing GitLab for the operating system you choose, at <https://about.gitlab.com/install/>. You can use the IP address from the step above, as the hostname. 1. Congratulations! GitLab is now installed and you can access it via your browser. To finish installation, open the URL in your browser and provide the initial administrator password. The username for this account is `root`. -  +  ## Next steps @@ -83,31 +83,31 @@ here's how you configure GitLab to be aware of the change: 1. SSH into the VM. You can easily use the **SSH** button in the Google console and a new window will pop up. -  +  - In the future you might want to set up [connecting with an SSH key][ssh] - instead. + In the future you might want to set up [connecting with an SSH key][ssh] + instead. 1. Edit the config file of Omnibus GitLab using your favorite text editor: - ``` - sudo vim /etc/gitlab/gitlab.rb - ``` + ``` + sudo vim /etc/gitlab/gitlab.rb + ``` 1. Set the `external_url` value to the domain name you wish GitLab to have **without** `https`: - ``` - external_url 'http://gitlab.example.com' - ``` + ``` + external_url 'http://gitlab.example.com' + ``` - We will set up HTTPS in the next step, no need to do this now. + We will set up HTTPS in the next step, no need to do this now. 1. Reconfigure GitLab for the changes to take effect: - ``` - sudo gitlab-ctl reconfigure - ``` + ``` + sudo gitlab-ctl reconfigure + ``` 1. You can now visit GitLab using the domain name. diff --git a/doc/install/installation.md b/doc/install/installation.md index e9206469e5d..06ec00cecc4 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -299,57 +299,57 @@ use of extensions and concurrent index removal, you need at least PostgreSQL 9.2 1. Install the database packages: - ```sh - sudo apt-get install -y postgresql postgresql-client libpq-dev postgresql-contrib - ``` + ```sh + sudo apt-get install -y postgresql postgresql-client libpq-dev postgresql-contrib + ``` 1. Create a database user for GitLab: - ```sh - sudo -u postgres psql -d template1 -c "CREATE USER git CREATEDB;" - ``` + ```sh + sudo -u postgres psql -d template1 -c "CREATE USER git CREATEDB;" + ``` 1. Create the `pg_trgm` extension (required for GitLab 8.6+): - ```sh - sudo -u postgres psql -d template1 -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" - ``` + ```sh + sudo -u postgres psql -d template1 -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" + ``` 1. Create the GitLab production database and grant all privileges on database: - ```sh - sudo -u postgres psql -d template1 -c "CREATE DATABASE gitlabhq_production OWNER git;" - ``` + ```sh + sudo -u postgres psql -d template1 -c "CREATE DATABASE gitlabhq_production OWNER git;" + ``` 1. Try connecting to the new database with the new user: - ```sh - sudo -u git -H psql -d gitlabhq_production - ``` + ```sh + sudo -u git -H psql -d gitlabhq_production + ``` 1. Check if the `pg_trgm` extension is enabled: - ```sh - SELECT true AS enabled - FROM pg_available_extensions - WHERE name = 'pg_trgm' - AND installed_version IS NOT NULL; - ``` + ```sh + SELECT true AS enabled + FROM pg_available_extensions + WHERE name = 'pg_trgm' + AND installed_version IS NOT NULL; + ``` - If the extension is enabled this will produce the following output: + If the extension is enabled this will produce the following output: - ``` - enabled - --------- - t - (1 row) - ``` + ``` + enabled + --------- + t + (1 row) + ``` 1. Quit the database session: - ```sh - gitlabhq_production> \q - ``` + ```sh + gitlabhq_production> \q + ``` ## 7. Redis @@ -831,26 +831,27 @@ how to configure GitLab with a relative URL. To use GitLab with HTTPS: 1. In `gitlab.yml`: - 1. Set the `port` option in section 1 to `443`. - 1. Set the `https` option in section 1 to `true`. + 1. Set the `port` option in section 1 to `443`. + 1. Set the `https` option in section 1 to `true`. 1. In the `config.yml` of gitlab-shell: - 1. Set `gitlab_url` option to the HTTPS endpoint of GitLab (e.g. `https://git.example.com`). - 1. Set the certificates using either the `ca_file` or `ca_path` option. + 1. Set `gitlab_url` option to the HTTPS endpoint of GitLab (e.g. `https://git.example.com`). + 1. Set the certificates using either the `ca_file` or `ca_path` option. 1. Use the `gitlab-ssl` Nginx example config instead of the `gitlab` config. - 1. Update `YOUR_SERVER_FQDN`. - 1. Update `ssl_certificate` and `ssl_certificate_key`. - 1. Review the configuration file and consider applying other security and performance enhancing features. + 1. Update `YOUR_SERVER_FQDN`. + 1. Update `ssl_certificate` and `ssl_certificate_key`. + 1. Review the configuration file and consider applying other security and performance enhancing features. Using a self-signed certificate is discouraged but if you must use it, follow the normal directions. Then: 1. Generate a self-signed SSL certificate: - ```sh - mkdir -p /etc/nginx/ssl/ - cd /etc/nginx/ssl/ - sudo openssl req -newkey rsa:2048 -x509 -nodes -days 3560 -out gitlab.crt -keyout gitlab.key - sudo chmod o-r gitlab.key - ``` + ```sh + mkdir -p /etc/nginx/ssl/ + cd /etc/nginx/ssl/ + sudo openssl req -newkey rsa:2048 -x509 -nodes -days 3560 -out gitlab.crt -keyout gitlab.key + sudo chmod o-r gitlab.key + ``` + 1. In the `config.yml` of gitlab-shell set `self_signed_cert` to `true`. ### Enable Reply by email diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md index e4a2d9ecd68..fbbe2a34952 100644 --- a/doc/install/openshift_and_gitlab/index.md +++ b/doc/install/openshift_and_gitlab/index.md @@ -70,17 +70,17 @@ In short: 1. Open a terminal and in a new directory run: - ```sh - vagrant init openshift/origin-all-in-one - ``` + ```sh + vagrant init openshift/origin-all-in-one + ``` 1. This will generate a Vagrantfile based on the all-in-one VM image 1. In the same directory where you generated the Vagrantfile enter: - ```sh - vagrant up - ``` + ```sh + vagrant up + ``` This will download the VirtualBox image and fire up the VM with some preconfigured values as you can see in the Vagrantfile. As you may have noticed, you need @@ -195,22 +195,22 @@ In that case, the OpenShift service might not be running, so in order to fix it: 1. SSH into the VM by going to the directory where the Vagrantfile is and then run: - ```sh - vagrant ssh - ``` + ```sh + vagrant ssh + ``` 1. Run `systemctl` and verify by the output that the `openshift` service is not running (it will be in red color). If that's the case start the service with: - ```sh - sudo systemctl start openshift - ``` + ```sh + sudo systemctl start openshift + ``` 1. Verify the service is up with: - ```sh - systemctl status openshift -l - ``` + ```sh + systemctl status openshift -l + ``` Now you will be able to login using `oc` (like we did before) and visit the web console. @@ -393,55 +393,55 @@ Let's see how to do that using the following steps. 1. Make sure you are in the `gitlab` project: - ```sh - oc project gitlab - ``` + ```sh + oc project gitlab + ``` 1. See what services are used for this project: - ```sh - oc get svc - ``` + ```sh + oc get svc + ``` - The output will be similar to: + The output will be similar to: - ``` - NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE - gitlab-ce 172.30.243.177 <none> 22/TCP,80/TCP 5d - gitlab-ce-postgresql 172.30.116.75 <none> 5432/TCP 5d - gitlab-ce-redis 172.30.105.88 <none> 6379/TCP 5d - ``` + ``` + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + gitlab-ce 172.30.243.177 <none> 22/TCP,80/TCP 5d + gitlab-ce-postgresql 172.30.116.75 <none> 5432/TCP 5d + gitlab-ce-redis 172.30.105.88 <none> 6379/TCP 5d + ``` 1. We need to see the replication controllers of the `gitlab-ce` service. Get a detailed view of the current ones: - ```sh - oc describe rc gitlab-ce - ``` + ```sh + oc describe rc gitlab-ce + ``` - This will return a large detailed list of the current replication controllers. - Search for the name of the GitLab controller, usually `gitlab-ce-1` or if - that failed at some point and you spawned another one, it will be named - `gitlab-ce-2`. + This will return a large detailed list of the current replication controllers. + Search for the name of the GitLab controller, usually `gitlab-ce-1` or if + that failed at some point and you spawned another one, it will be named + `gitlab-ce-2`. 1. Scale GitLab using the previous information: - ```sh - oc scale --replicas=2 replicationcontrollers gitlab-ce-2 - ``` + ```sh + oc scale --replicas=2 replicationcontrollers gitlab-ce-2 + ``` 1. Get the new replicas number to make sure scaling worked: - ```sh - oc get rc gitlab-ce-2 - ``` + ```sh + oc get rc gitlab-ce-2 + ``` - which will return something like: + which will return something like: - ``` - NAME DESIRED CURRENT AGE - gitlab-ce-2 2 2 5d - ``` + ``` + NAME DESIRED CURRENT AGE + gitlab-ce-2 2 2 5d + ``` And that's it! We successfully scaled the replicas to 2 using the CLI. @@ -478,13 +478,13 @@ For OpenShift v3.0, you will need to do this manually: 1. Edit the Security Context: - ```sh - oc edit scc anyuid - ``` + ```sh + oc edit scc anyuid + ``` 1. Add `system:serviceaccount:<project>:gitlab-ce-user` to the `users` section. If you changed the Application Name from the default the user will - will be `<app-name>-user` instead of `gitlab-ce-user` + will be `<app-name>-user` instead of `gitlab-ce-user` 1. Save and exit the editor diff --git a/doc/install/relative_url.md b/doc/install/relative_url.md index b53624a33bf..bc6364f57f7 100644 --- a/doc/install/relative_url.md +++ b/doc/install/relative_url.md @@ -58,59 +58,59 @@ assumptions are made: Make sure to follow all steps below: -1. (Optional) If you run short on resources, you can temporarily free up some - memory by shutting down the GitLab service with the following command: +1. (Optional) If you run short on resources, you can temporarily free up some + memory by shutting down the GitLab service with the following command: - ```shell - sudo service gitlab stop - ``` + ```shell + sudo service gitlab stop + ``` -1. Create `/home/git/gitlab/config/initializers/relative_url.rb` +1. Create `/home/git/gitlab/config/initializers/relative_url.rb` - ```shell - cp /home/git/gitlab/config/initializers/relative_url.rb.sample \ - /home/git/gitlab/config/initializers/relative_url.rb - ``` + ```shell + cp /home/git/gitlab/config/initializers/relative_url.rb.sample \ + /home/git/gitlab/config/initializers/relative_url.rb + ``` - and change the following line: + and change the following line: - ```ruby - config.relative_url_root = "/gitlab" - ``` + ```ruby + config.relative_url_root = "/gitlab" + ``` -1. Edit `/home/git/gitlab/config/gitlab.yml` and uncomment/change the - following line: +1. Edit `/home/git/gitlab/config/gitlab.yml` and uncomment/change the + following line: - ```yaml - relative_url_root: /gitlab - ``` + ```yaml + relative_url_root: /gitlab + ``` -1. Edit `/home/git/gitlab/config/unicorn.rb` and uncomment/change the - following line: +1. Edit `/home/git/gitlab/config/unicorn.rb` and uncomment/change the + following line: - ```ruby - ENV['RAILS_RELATIVE_URL_ROOT'] = "/gitlab" - ``` + ```ruby + ENV['RAILS_RELATIVE_URL_ROOT'] = "/gitlab" + ``` -1. Edit `/home/git/gitlab-shell/config.yml` and append the relative path to - the following line: +1. Edit `/home/git/gitlab-shell/config.yml` and append the relative path to + the following line: - ```yaml - gitlab_url: http://127.0.0.1/gitlab - ``` + ```yaml + gitlab_url: http://127.0.0.1/gitlab + ``` -1. Make sure you have copied the supplied init script and the defaults file - as stated in the [installation guide](installation.md#install-init-script). - Then, edit `/etc/default/gitlab` and set in `gitlab_workhorse_options` the - `-authBackend` setting to read like: +1. Make sure you have copied the supplied init script and the defaults file + as stated in the [installation guide](installation.md#install-init-script). + Then, edit `/etc/default/gitlab` and set in `gitlab_workhorse_options` the + `-authBackend` setting to read like: - ```shell - -authBackend http://127.0.0.1:8080/gitlab - ``` + ```shell + -authBackend http://127.0.0.1:8080/gitlab + ``` - **Note:** - If you are using a custom init script, make sure to edit the above - gitlab-workhorse setting as needed. + **Note:** + If you are using a custom init script, make sure to edit the above + gitlab-workhorse setting as needed. 1. [Restart GitLab][] for the changes to take effect. @@ -118,9 +118,9 @@ Make sure to follow all steps below: To disable the relative URL: -1. Remove `/home/git/gitlab/config/initializers/relative_url.rb` +1. Remove `/home/git/gitlab/config/initializers/relative_url.rb` -1. Follow the same as above starting from 2. and set up the +1. Follow the same as above starting from 2. and set up the GitLab URL to one that doesn't contain a relative path. [omnibus-rel]: https://docs.gitlab.com/omnibus/settings/configuration.html#configuring-a-relative-url-for-gitlab "How to set up relative URL in Omnibus GitLab" diff --git a/doc/integration/auth0.md b/doc/integration/auth0.md index c67375ede50..5061b863e79 100644 --- a/doc/integration/auth0.md +++ b/doc/integration/auth0.md @@ -16,64 +16,64 @@ application. 1. At the top of the Settings screen, you should see your Domain, Client ID and Client Secret. Take note of these as you'll need to put them in the configuration file. For example: - - Domain: `test1234.auth0.com` - - Client ID: `t6X8L2465bNePWLOvt9yi41i` - - Client Secret: `KbveM3nqfjwCbrhaUy_gDu2dss8TIlHIdzlyf33pB7dEK5u_NyQdp65O_o02hXs2` + - Domain: `test1234.auth0.com` + - Client ID: `t6X8L2465bNePWLOvt9yi41i` + - Client Secret: `KbveM3nqfjwCbrhaUy_gDu2dss8TIlHIdzlyf33pB7dEK5u_NyQdp65O_o02hXs2` 1. Fill in the Allowed Callback URLs: - - `http://YOUR_GITLAB_URL/users/auth/auth0/callback` (or) - - `https://YOUR_GITLAB_URL/users/auth/auth0/callback` + - `http://YOUR_GITLAB_URL/users/auth/auth0/callback` (or) + - `https://YOUR_GITLAB_URL/users/auth/auth0/callback` 1. Fill in the Allowed Origins (CORS): - - `http://YOUR_GITLAB_URL` (or) - - `https://YOUR_GITLAB_URL` + - `http://YOUR_GITLAB_URL` (or) + - `https://YOUR_GITLAB_URL` 1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. 1. Add the provider configuration: - For omnibus package: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "auth0", - "args" => { client_id: 'YOUR_AUTH0_CLIENT_ID', - client_secret: 'YOUR_AUTH0_CLIENT_SECRET', - domain: 'YOUR_AUTH0_DOMAIN', - scope: 'openid profile email' - } - } - ] - ``` - - For installations from source: - - ```yaml - - { name: 'auth0', - args: { - client_id: 'YOUR_AUTH0_CLIENT_ID', - client_secret: 'YOUR_AUTH0_CLIENT_SECRET', - domain: 'YOUR_AUTH0_DOMAIN', - scope: 'openid profile email' } - } - ``` + For omnibus package: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "auth0", + "args" => { client_id: 'YOUR_AUTH0_CLIENT_ID', + client_secret: 'YOUR_AUTH0_CLIENT_SECRET', + domain: 'YOUR_AUTH0_DOMAIN', + scope: 'openid profile email' + } + } + ] + ``` + + For installations from source: + + ```yaml + - { name: 'auth0', + args: { + client_id: 'YOUR_AUTH0_CLIENT_ID', + client_secret: 'YOUR_AUTH0_CLIENT_SECRET', + domain: 'YOUR_AUTH0_DOMAIN', + scope: 'openid profile email' } + } + ``` 1. Change `YOUR_AUTH0_CLIENT_ID` to the client ID from the Auth0 Console page from step 5. @@ -81,8 +81,8 @@ application. 1. Change `YOUR_AUTH0_CLIENT_SECRET` to the client secret from the Auth0 Console page from step 5. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be an Auth0 icon below the regular sign in form. Click the icon to begin the authentication process. Auth0 will ask the diff --git a/doc/integration/bitbucket.md b/doc/integration/bitbucket.md index 68ec8c4b5c2..5d8f2ebcb8b 100644 --- a/doc/integration/bitbucket.md +++ b/doc/integration/bitbucket.md @@ -30,97 +30,97 @@ To enable the Bitbucket OmniAuth provider you must register your application with Bitbucket.org. Bitbucket will generate an application ID and secret key for you to use. -1. Sign in to [Bitbucket.org](https://bitbucket.org). -1. Navigate to your individual user settings (**Bitbucket settings**) or a team's - settings (**Manage team**), depending on how you want the application registered. - It does not matter if the application is registered as an individual or a - team, that is entirely up to you. -1. Select **OAuth** in the left menu under "Access Management". -1. Select **Add consumer**. -1. Provide the required details: - - | Item | Description | - | :--- | :---------- | - | **Name** | This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. | - | **Application description** | Fill this in if you wish. | - | **Callback URL** | The URL to your GitLab installation, e.g., `https://gitlab.example.com/users/auth`. | - | **URL** | The URL to your GitLab installation, e.g., `https://gitlab.example.com`. | - - NOTE: Be sure to append `/users/auth` to the end of the callback URL - to prevent a [OAuth2 convert - redirect](http://tetraph.com/covert_redirect/) vulnerability. - - NOTE: Starting in GitLab 8.15, you MUST specify a callback URL, or you will - see an "Invalid redirect_uri" message. For more details, see [the - Bitbucket documentation](https://confluence.atlassian.com/bitbucket/oauth-faq-338365710.html). - - And grant at least the following permissions: - - ``` - Account: Email, Read - Projects: Read - Repositories: Read - Pull Requests: Read - Issues: Read - Wiki: Read and Write - ``` - -  - -1. Select **Save**. -1. Select your newly created OAuth consumer and you should now see a Key and - Secret in the list of OAuth consumers. Keep this page open as you continue - the configuration. - -  - -1. On your GitLab server, open the configuration file: - - ``` - # For Omnibus packages - sudo editor /etc/gitlab/gitlab.rb - - # For installations from source - sudo -u git -H editor /home/git/gitlab/config/gitlab.yml - ``` - -1. Add the Bitbucket provider configuration: - - For Omnibus packages: - - ```ruby - gitlab_rails['omniauth_enabled'] = true - - gitlab_rails['omniauth_providers'] = [ - { - "name" => "bitbucket", - "app_id" => "BITBUCKET_APP_KEY", - "app_secret" => "BITBUCKET_APP_SECRET", - "url" => "https://bitbucket.org/" - } - ] - ``` - - For installations from source: - - ```yaml - omniauth: - enabled: true - providers: - - { name: 'bitbucket', - app_id: 'BITBUCKET_APP_KEY', - app_secret: 'BITBUCKET_APP_SECRET', - url: 'https://bitbucket.org/' } - ``` - - --- - - Where `BITBUCKET_APP_KEY` is the Key and `BITBUCKET_APP_SECRET` the Secret - from the Bitbucket application page. - -1. Save the configuration file. -1. For the changes to take effect, [reconfigure GitLab][] if you installed via - Omnibus, or [restart][] if installed from source. +1. Sign in to [Bitbucket.org](https://bitbucket.org). +1. Navigate to your individual user settings (**Bitbucket settings**) or a team's + settings (**Manage team**), depending on how you want the application registered. + It does not matter if the application is registered as an individual or a + team, that is entirely up to you. +1. Select **OAuth** in the left menu under "Access Management". +1. Select **Add consumer**. +1. Provide the required details: + + | Item | Description | + | :--- | :---------- | + | **Name** | This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. | + | **Application description** | Fill this in if you wish. | + | **Callback URL** | The URL to your GitLab installation, e.g., `https://gitlab.example.com/users/auth`. | + | **URL** | The URL to your GitLab installation, e.g., `https://gitlab.example.com`. | + + NOTE: Be sure to append `/users/auth` to the end of the callback URL + to prevent a [OAuth2 convert + redirect](http://tetraph.com/covert_redirect/) vulnerability. + + NOTE: Starting in GitLab 8.15, you MUST specify a callback URL, or you will + see an "Invalid redirect_uri" message. For more details, see [the + Bitbucket documentation](https://confluence.atlassian.com/bitbucket/oauth-faq-338365710.html). + + And grant at least the following permissions: + + ``` + Account: Email, Read + Projects: Read + Repositories: Read + Pull Requests: Read + Issues: Read + Wiki: Read and Write + ``` + +  + +1. Select **Save**. +1. Select your newly created OAuth consumer and you should now see a Key and + Secret in the list of OAuth consumers. Keep this page open as you continue + the configuration. + +  + +1. On your GitLab server, open the configuration file: + + ``` + # For Omnibus packages + sudo editor /etc/gitlab/gitlab.rb + + # For installations from source + sudo -u git -H editor /home/git/gitlab/config/gitlab.yml + ``` + +1. Add the Bitbucket provider configuration: + + For Omnibus packages: + + ```ruby + gitlab_rails['omniauth_enabled'] = true + + gitlab_rails['omniauth_providers'] = [ + { + "name" => "bitbucket", + "app_id" => "BITBUCKET_APP_KEY", + "app_secret" => "BITBUCKET_APP_SECRET", + "url" => "https://bitbucket.org/" + } + ] + ``` + + For installations from source: + + ```yaml + omniauth: + enabled: true + providers: + - { name: 'bitbucket', + app_id: 'BITBUCKET_APP_KEY', + app_secret: 'BITBUCKET_APP_SECRET', + url: 'https://bitbucket.org/' } + ``` + + --- + + Where `BITBUCKET_APP_KEY` is the Key and `BITBUCKET_APP_SECRET` the Secret + from the Bitbucket application page. + +1. Save the configuration file. +1. For the changes to take effect, [reconfigure GitLab][] if you installed via + Omnibus, or [restart][] if installed from source. On the sign in page there should now be a Bitbucket icon below the regular sign in form. Click the icon to begin the authentication process. Bitbucket will ask diff --git a/doc/integration/cas.md b/doc/integration/cas.md index c6178fa44f0..f99337376a8 100644 --- a/doc/integration/cas.md +++ b/doc/integration/cas.md @@ -2,63 +2,63 @@ To enable the CAS OmniAuth provider you must register your application with your CAS instance. This requires the service URL GitLab will supply to CAS. It should be something like: `https://gitlab.example.com:443/users/auth/cas3/callback?url`. By default handling for SLO is enabled, you only need to configure CAS for backchannel logout. -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: +1. Add the provider configuration: - For omnibus package: + For omnibus package: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name"=> "cas3", - "label"=> "cas", - "args"=> { - "url"=> 'CAS_SERVER', - "login_url"=> '/CAS_PATH/login', - "service_validate_url"=> '/CAS_PATH/p3/serviceValidate', - "logout_url"=> '/CAS_PATH/logout' - } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name"=> "cas3", + "label"=> "cas", + "args"=> { + "url"=> 'CAS_SERVER', + "login_url"=> '/CAS_PATH/login', + "service_validate_url"=> '/CAS_PATH/p3/serviceValidate', + "logout_url"=> '/CAS_PATH/logout' + } + } + ] + ``` - For installations from source: + For installations from source: - ``` - - { name: 'cas3', - label: 'cas', - args: { - url: 'CAS_SERVER', - login_url: '/CAS_PATH/login', - service_validate_url: '/CAS_PATH/p3/serviceValidate', - logout_url: '/CAS_PATH/logout'} } - ``` + ``` + - { name: 'cas3', + label: 'cas', + args: { + url: 'CAS_SERVER', + login_url: '/CAS_PATH/login', + service_validate_url: '/CAS_PATH/p3/serviceValidate', + logout_url: '/CAS_PATH/logout'} } + ``` -1. Change 'CAS_PATH' to the root of your CAS instance (ie. `cas`). +1. Change 'CAS_PATH' to the root of your CAS instance (ie. `cas`). -1. If your CAS instance does not use default TGC lifetimes, update the `cas3.session_duration` to at least the current TGC maximum lifetime. To explicitly disable SLO, regardless of CAS settings, set this to 0. +1. If your CAS instance does not use default TGC lifetimes, update the `cas3.session_duration` to at least the current TGC maximum lifetime. To explicitly disable SLO, regardless of CAS settings, set this to 0. -1. Save the configuration file. +1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a CAS tab in the sign in form. diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index da1df07a75d..fff06254da7 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -12,6 +12,7 @@ special searches: - [Advanced Syntax Search](../user/search/advanced_search_syntax.md) ## Version Requirements + <!-- Please remember to update ee/lib/system_check/app/elasticsearch_check.rb if this changes --> | GitLab version | Elasticsearch version | @@ -424,91 +425,90 @@ Here are some common pitfalls and how to overcome them: - **How can I verify my GitLab instance is using Elasticsearch?** - The easiest method is via the rails console (`sudo gitlab-rails console`) by running the following: + The easiest method is via the rails console (`sudo gitlab-rails console`) by running the following: - ```ruby - u = User.find_by_username('your-username') - s = SearchService.new(u, {:search => 'search_term'}) - pp s.search_objects.class.name - ``` + ```ruby + u = User.find_by_username('your-username') + s = SearchService.new(u, {:search => 'search_term'}) + pp s.search_objects.class.name + ``` - If you see `Elasticsearch::Model::Response::Records`, you are using Elasticsearch. + If you see `Elasticsearch::Model::Response::Records`, you are using Elasticsearch. - **I updated GitLab and now I can't find anything** - We continuously make updates to our indexing strategies and aim to support - newer versions of Elasticsearch. When indexing changes are made, it may - be necessary for you to [reindex](#adding-gitlabs-data-to-the-elasticsearch-index) after updating GitLab. + We continuously make updates to our indexing strategies and aim to support + newer versions of Elasticsearch. When indexing changes are made, it may + be necessary for you to [reindex](#adding-gitlabs-data-to-the-elasticsearch-index) after updating GitLab. - **I indexed all the repositories but I can't find anything** - Make sure you indexed all the database data [as stated above](#adding-gitlabs-data-to-the-elasticsearch-index). + Make sure you indexed all the database data [as stated above](#adding-gitlabs-data-to-the-elasticsearch-index). - Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side. + Beyond that, check via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html) to see if the data shows up on the Elasticsearch side. - If it shows up via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html), check that it shows up via the rails console (`sudo gitlab-rails console`): + If it shows up via the [Elasticsearch Search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-search.html), check that it shows up via the rails console (`sudo gitlab-rails console`): - ```ruby - u = User.find_by_username('your-username') - s = SearchService.new(u, {:search => 'search_term', :scope => ‘blobs’}) - pp s.search_objects.to_a - ``` + ```ruby + u = User.find_by_username('your-username') + s = SearchService.new(u, {:search => 'search_term', :scope => ‘blobs’}) + pp s.search_objects.to_a + ``` - See [Elasticsearch Index Scopes](elasticsearch.md#elasticsearch-index-scopes) for more information on searching for specific types of data. + See [Elasticsearch Index Scopes](elasticsearch.md#elasticsearch-index-scopes) for more information on searching for specific types of data. - **I indexed all the repositories but then switched Elasticsearch servers and now I can't find anything** - You will need to re-run all the rake tasks to re-index the database, repositories, and wikis. + You will need to re-run all the rake tasks to re-index the database, repositories, and wikis. - **The indexing process is taking a very long time** - The more data present in your GitLab instance, the longer the indexing process takes. + The more data present in your GitLab instance, the longer the indexing process takes. - **No new data is added to the Elasticsearch index when I push code** - When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could - happen that an error during the process causes one or multiple projects to remain locked. In order to unlock them, - run the `gitlab:elastic:clear_locked_projects` rake task. + When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could + happen that an error during the process causes one or multiple projects to remain locked. In order to unlock them, + run the `gitlab:elastic:clear_locked_projects` rake task. - **"Can't specify parent if no parent field has been configured"** - If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indexes you will get - exception in lots of different cases: - - ```text - Elasticsearch::Transport::Transport::Errors::BadRequest([400] { - "error": { - "root_cause": [{ - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }], - "type": "illegal_argument_exception", - "reason": "Can't specify parent if no parent field has been configured" - }, - "status": 400 - }): - ``` - - This is because we changed the index mapping in GitLab 8.12 and the old indexes should be removed and built from scratch again, - see details in the [8-11-to-8-12 update guide](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/doc/update/8.11-to-8.12.md#11-elasticsearch-index-update-if-you-currently-use-elasticsearch). + If you enabled Elasticsearch before GitLab 8.12 and have not rebuilt indexes you will get + exception in lots of different cases: + + ```text + Elasticsearch::Transport::Transport::Errors::BadRequest([400] { + "error": { + "root_cause": [{ + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }], + "type": "illegal_argument_exception", + "reason": "Can't specify parent if no parent field has been configured" + }, + "status": 400 + }): + ``` + + This is because we changed the index mapping in GitLab 8.12 and the old indexes should be removed and built from scratch again, + see details in the [8-11-to-8-12 update guide](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/doc/update/8.11-to-8.12.md#11-elasticsearch-index-update-if-you-currently-use-elasticsearch). - Exception `Elasticsearch::Transport::Transport::Errors::BadRequest` - If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](#system-requirements). - There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command. + If you have this exception (just like in the case above but the actual message is different) please check if you have the correct Elasticsearch version and you met the other [requirements](#system-requirements). + There is also an easy way to check it automatically with `sudo gitlab-rake gitlab:check` command. - Exception `Elasticsearch::Transport::Transport::Errors::RequestEntityTooLarge` - ```text - [413] {"Message":"Request size exceeded 10485760 bytes"} - ``` - - This exception is seen when your Elasticsearch cluster is configured to reject - requests above a certain size (10MiB in this case). This corresponds to the - `http.max_content_length` setting in `elasticsearch.yml`. Increase it to a - larger size and restart your Elasticsearch cluster. + ```text + [413] {"Message":"Request size exceeded 10485760 bytes"} + ``` - AWS has [fixed limits](http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html) - for this setting ("Maximum Size of HTTP Request Payloads"), based on the size of - the underlying instance. + This exception is seen when your Elasticsearch cluster is configured to reject + requests above a certain size (10MiB in this case). This corresponds to the + `http.max_content_length` setting in `elasticsearch.yml`. Increase it to a + larger size and restart your Elasticsearch cluster. + AWS has [fixed limits](http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html) + for this setting ("Maximum Size of HTTP Request Payloads"), based on the size of + the underlying instance. diff --git a/doc/integration/facebook.md b/doc/integration/facebook.md index fe789a80eed..837434da737 100644 --- a/doc/integration/facebook.md +++ b/doc/integration/facebook.md @@ -2,7 +2,7 @@ To enable the Facebook OmniAuth provider you must register your application with Facebook. Facebook will generate an app ID and secret key for you to use. -1. Sign in to the [Facebook Developer Platform](https://developers.facebook.com/). +1. Sign in to the [Facebook Developer Platform](https://developers.facebook.com/). 1. Choose "My Apps" > "Add a New App" @@ -47,53 +47,53 @@ To enable the Facebook OmniAuth provider you must register your application with  -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: +1. Add the provider configuration: - For omnibus package: + For omnibus package: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "facebook", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET" - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "facebook", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET" + } + ] + ``` - For installations from source: + For installations from source: - ``` - - { name: 'facebook', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET' } - ``` + ``` + - { name: 'facebook', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET' } + ``` -1. Change 'YOUR_APP_ID' to the API key from Facebook page in step 10. +1. Change 'YOUR_APP_ID' to the API key from Facebook page in step 10. -1. Change 'YOUR_APP_SECRET' to the API secret from the Facebook page in step 10. +1. Change 'YOUR_APP_SECRET' to the API secret from the Facebook page in step 10. -1. Save the configuration file. +1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Facebook icon below the regular sign in form. Click the icon to begin the authentication process. Facebook will ask the user to sign in and authorize the GitLab application. If everything goes well the user will be returned to GitLab and will be signed in. diff --git a/doc/integration/github.md b/doc/integration/github.md index 5b01dd9feb7..c8dbae65465 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -7,111 +7,111 @@ You can integrate your GitLab instance with GitHub.com as well as GitHub Enterpr To enable GitHub OmniAuth provider, you must use GitHub's credentials for your GitLab instance. To get the credentials (a pair of Client ID and Client Secret), you must register an application as an OAuth App on GitHub. -1. Sign in to GitHub. +1. Sign in to GitHub. -1. Navigate to your individual user or organization settings, depending on how you want the application registered. It does not matter if the application is registered as an individual or an organization - that is entirely up to you. +1. Navigate to your individual user or organization settings, depending on how you want the application registered. It does not matter if the application is registered as an individual or an organization - that is entirely up to you. - - For individual accounts, select **Developer settings** from the left menu, then select **OAuth Apps**. - - For organization accounts, directly select **OAuth Apps** from the left menu. + - For individual accounts, select **Developer settings** from the left menu, then select **OAuth Apps**. + - For organization accounts, directly select **OAuth Apps** from the left menu. -1. Select **Register an application** (if you don't have any OAuth App) or **New OAuth App** (if you already have OAuth Apps). -  +1. Select **Register an application** (if you don't have any OAuth App) or **New OAuth App** (if you already have OAuth Apps). +  -1. Provide the required details. - - Application name: This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. - - Homepage URL: The URL of your GitLab installation. For example, `https://gitlab.example.com`. - - Application description: Fill this in if you wish. - - Authorization callback URL: `http(s)://${YOUR_DOMAIN}/users/auth`. Please make sure the port is included if your GitLab instance is not configured on default port. -  +1. Provide the required details. + - Application name: This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. + - Homepage URL: The URL of your GitLab installation. For example, `https://gitlab.example.com`. + - Application description: Fill this in if you wish. + - Authorization callback URL: `http(s)://${YOUR_DOMAIN}/users/auth`. Please make sure the port is included if your GitLab instance is not configured on default port. +  - NOTE: Be sure to append `/users/auth` to the end of the callback URL - to prevent a [OAuth2 convert - redirect](http://tetraph.com/covert_redirect/) vulnerability. + NOTE: Be sure to append `/users/auth` to the end of the callback URL + to prevent a [OAuth2 convert + redirect](http://tetraph.com/covert_redirect/) vulnerability. -1. Select **Register application**. +1. Select **Register application**. -1. You should now see a pair of **Client ID** and **Client Secret** near the top right of the page (see screenshot). - Keep this page open as you continue configuration. -  +1. You should now see a pair of **Client ID** and **Client Secret** near the top right of the page (see screenshot). + Keep this page open as you continue configuration. +  -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: +1. Add the provider configuration: - For omnibus package: + For omnibus package: - For GitHub.com: + For GitHub.com: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "github", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "args" => { "scope" => "user:email" } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "github", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "args" => { "scope" => "user:email" } + } + ] + ``` - For GitHub Enterprise: + For GitHub Enterprise: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "github", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "url" => "https://github.example.com/", - "args" => { "scope" => "user:email" } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "github", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "url" => "https://github.example.com/", + "args" => { "scope" => "user:email" } + } + ] + ``` - For installation from source: + For installation from source: - For GitHub.com: + For GitHub.com: - ``` - - { name: 'github', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - args: { scope: 'user:email' } } - ``` + ``` + - { name: 'github', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { scope: 'user:email' } } + ``` - For GitHub Enterprise: + For GitHub Enterprise: - ``` - - { name: 'github', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - url: "https://github.example.com/", - args: { scope: 'user:email' } } - ``` + ``` + - { name: 'github', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + url: "https://github.example.com/", + args: { scope: 'user:email' } } + ``` - __Replace `https://github.example.com/` with your GitHub URL.__ + __Replace `https://github.example.com/` with your GitHub URL.__ -1. Change `YOUR_APP_ID` to the Client ID from the GitHub application page from step 6. +1. Change `YOUR_APP_ID` to the Client ID from the GitHub application page from step 6. -1. Change `YOUR_APP_SECRET` to the Client Secret from the GitHub application page from step 6. +1. Change `YOUR_APP_SECRET` to the Client Secret from the GitHub application page from step 6. -1. Save the configuration file. +1. Save the configuration file. -1. [Reconfigure GitLab][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure GitLab][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a GitHub icon below the regular sign in form. Click the icon to begin the authentication process. GitHub will ask the user to sign in and authorize the GitLab application. @@ -127,16 +127,16 @@ and changing the global Git `sslVerify` option to `false` in the GitLab server. For omnibus package: ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "github", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "url" => "https://github.example.com/", - "verify_ssl" => false, - "args" => { "scope" => "user:email" } - } - ] +gitlab_rails['omniauth_providers'] = [ + { + "name" => "github", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "url" => "https://github.example.com/", + "verify_ssl" => false, + "args" => { "scope" => "user:email" } + } +] ``` You will also need to disable Git SSL verification on the server hosting GitLab. @@ -148,11 +148,11 @@ omnibus_gitconfig['system'] = { "http" => ["sslVerify = false"] } For installation from source: ``` - - { name: 'github', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - url: "https://github.example.com/", - verify_ssl: false, - args: { scope: 'user:email' } } +- { name: 'github', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + url: "https://github.example.com/", + verify_ssl: false, + args: { scope: 'user:email' } } ``` You will also need to disable Git SSL verification on the server hosting GitLab. diff --git a/doc/integration/gitlab.md b/doc/integration/gitlab.md index 70087576678..46da3d88d90 100644 --- a/doc/integration/gitlab.md +++ b/doc/integration/gitlab.md @@ -5,78 +5,78 @@ Import projects from GitLab.com and login to your GitLab instance with your GitL To enable the GitLab.com OmniAuth provider you must register your application with GitLab.com. GitLab.com will generate an application ID and secret key for you to use. -1. Sign in to GitLab.com +1. Sign in to GitLab.com 1. On the upper right corner, click on your avatar and go to your **Settings**. -1. Select **Applications** in the left menu. +1. Select **Applications** in the left menu. -1. Provide the required details for **Add new application**. - - Name: This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. - - Redirect URI: +1. Provide the required details for **Add new application**. + - Name: This can be anything. Consider something like `<Organization>'s GitLab` or `<Your Name>'s GitLab` or something else descriptive. + - Redirect URI: - ``` - http://your-gitlab.example.com/import/gitlab/callback - http://your-gitlab.example.com/users/auth/gitlab/callback - ``` + ``` + http://your-gitlab.example.com/import/gitlab/callback + http://your-gitlab.example.com/users/auth/gitlab/callback + ``` - The first link is required for the importer and second for the authorization. + The first link is required for the importer and second for the authorization. -1. Select **Save application**. +1. Select **Save application**. -1. You should now see a **Application Id** and **Secret** near the top right of the page (see screenshot). - Keep this page open as you continue configuration. -  +1. You should now see a **Application Id** and **Secret** near the top right of the page (see screenshot). + Keep this page open as you continue configuration. +  -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. - For omnibus package: + For omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: +1. Add the provider configuration: - For omnibus package: + For omnibus package: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "gitlab", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "args" => { "scope" => "api" } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "gitlab", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "args" => { "scope" => "api" } + } + ] + ``` - For installations from source: + For installations from source: - ``` - - { name: 'gitlab', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - args: { scope: 'api' } } - ``` + ``` + - { name: 'gitlab', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { scope: 'api' } } + ``` -1. Change 'YOUR_APP_ID' to the Application ID from the GitLab.com application page. +1. Change 'YOUR_APP_ID' to the Application ID from the GitLab.com application page. -1. Change 'YOUR_APP_SECRET' to the secret from the GitLab.com application page. +1. Change 'YOUR_APP_SECRET' to the secret from the GitLab.com application page. -1. Save the configuration file. +1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a GitLab.com icon below the regular sign in form. Click the icon to begin the authentication process. GitLab.com will ask the user to sign in and authorize the GitLab application. diff --git a/doc/integration/google.md b/doc/integration/google.md index d2b4e119978..4f6999571b6 100644 --- a/doc/integration/google.md +++ b/doc/integration/google.md @@ -10,10 +10,10 @@ In Google's side: 1. Navigate to the [cloud resource manager](https://console.cloud.google.com/cloud-resource-manager) page 1. Select **Create Project** 1. Provide the project information: - - **Project name** - "GitLab" works just fine here. - - **Project ID** - Must be unique to all Google Developer registered applications. - Google provides a randomly generated Project ID by default. You can use - the randomly generated ID or choose a new one. + - **Project name** - "GitLab" works just fine here. + - **Project ID** - Must be unique to all Google Developer registered applications. + Google provides a randomly generated Project ID by default. You can use + the randomly generated ID or choose a new one. 1. Refresh the page and you should see your new project in the list 1. Go to the [Google API Console](https://console.developers.google.com/apis/dashboard) 1. Select the previously created project form the upper left corner @@ -21,17 +21,17 @@ In Google's side: 1. Select **OAuth consent screen** and fill the form with the required information 1. In the **Credentials** tab, select **Create credentials > OAuth client ID** 1. Fill in the required information - - **Application type** - Choose "Web Application" - - **Name** - Use the default one or provide your own - - **Authorized JavaScript origins** -This isn't really used by GitLab but go - ahead and put `https://gitlab.example.com` - - **Authorized redirect URIs** - Enter your domain name followed by the - callback URIs one at a time: - - ``` - https://gitlab.example.com/users/auth/google_oauth2/callback - https://gitlab.example.com/-/google_api/auth/callback - ``` + - **Application type** - Choose "Web Application" + - **Name** - Use the default one or provide your own + - **Authorized JavaScript origins** -This isn't really used by GitLab but go + ahead and put `https://gitlab.example.com` + - **Authorized redirect URIs** - Enter your domain name followed by the + callback URIs one at a time: + + ``` + https://gitlab.example.com/users/auth/google_oauth2/callback + https://gitlab.example.com/-/google_api/auth/callback + ``` 1. You should now be able to see a Client ID and Client secret. Note them down or keep this page open as you will need them later. @@ -45,64 +45,64 @@ On your GitLab server: 1. Open the configuration file. - For Omnibus GitLab: + For Omnibus GitLab: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. 1. Add the provider configuration: - For Omnibus GitLab: + For Omnibus GitLab: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "google_oauth2", - "app_id" => "YOUR_APP_ID", - "app_secret" => "YOUR_APP_SECRET", - "args" => { "access_type" => "offline", "approval_prompt" => '' } - } - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "google_oauth2", + "app_id" => "YOUR_APP_ID", + "app_secret" => "YOUR_APP_SECRET", + "args" => { "access_type" => "offline", "approval_prompt" => '' } + } + ] + ``` - For installations from source: + For installations from source: - ```yaml - - { name: 'google_oauth2', app_id: 'YOUR_APP_ID', - app_secret: 'YOUR_APP_SECRET', - args: { access_type: 'offline', approval_prompt: '' } } - ``` + ```yaml + - { name: 'google_oauth2', app_id: 'YOUR_APP_ID', + app_secret: 'YOUR_APP_SECRET', + args: { access_type: 'offline', approval_prompt: '' } } + ``` 1. Change `YOUR_APP_ID` to the client ID from the Google Developer page 1. Similarly, change `YOUR_APP_SECRET` to the client secret 1. Make sure that you configure GitLab to use an FQDN as Google will not accept raw IP addresses. - For Omnibus packages: + For Omnibus packages: - ```ruby - external_url 'https://gitlab.example.com' - ``` + ```ruby + external_url 'https://gitlab.example.com' + ``` - For installations from source: + For installations from source: - ```yaml - gitlab: - host: https://gitlab.example.com - ``` + ```yaml + gitlab: + host: https://gitlab.example.com + ``` -1. Save the configuration file. -1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. Save the configuration file. +1. [Reconfigure][] or [restart GitLab][] for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page there should now be a Google icon below the regular sign in form. Click the icon to begin the authentication process. Google will ask the diff --git a/doc/integration/kerberos.md b/doc/integration/kerberos.md index cf2ef511264..b4f2025265e 100644 --- a/doc/integration/kerberos.md +++ b/doc/integration/kerberos.md @@ -50,20 +50,20 @@ For source installations, make sure the `kerberos` gem group authentication. In most cases, you only need to enable Kerberos and specify the location of the keytab: - ```yaml - omniauth: - enabled: true - allow_single_sign_on: ['kerberos'] + ```yaml + omniauth: + enabled: true + allow_single_sign_on: ['kerberos'] - kerberos: - # Allow the HTTP Negotiate authentication method for Git clients - enabled: true + kerberos: + # Allow the HTTP Negotiate authentication method for Git clients + enabled: true - # Kerberos 5 keytab file. The keytab file must be readable by the GitLab user, - # and should be different from other keytabs in the system. - # (default: use default keytab from Krb5 config) - keytab: /etc/http.keytab - ``` + # Kerberos 5 keytab file. The keytab file must be readable by the GitLab user, + # and should be different from other keytabs in the system. + # (default: use default keytab from Krb5 config) + keytab: /etc/http.keytab + ``` 1. [Restart GitLab] for the changes to take effect. @@ -73,13 +73,13 @@ For source installations, make sure the `kerberos` gem group 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['omniauth_enabled'] = true - gitlab_rails['omniauth_allow_single_sign_on'] = ['kerberos'] + ```ruby + gitlab_rails['omniauth_enabled'] = true + gitlab_rails['omniauth_allow_single_sign_on'] = ['kerberos'] - gitlab_rails['kerberos_enabled'] = true - gitlab_rails['kerberos_keytab'] = "/etc/http.keytab" - ``` + gitlab_rails['kerberos_enabled'] = true + gitlab_rails['kerberos_keytab'] = "/etc/http.keytab" + ``` 1. [Reconfigure GitLab] for the changes to take effect. @@ -149,26 +149,26 @@ keep offering only `basic` authentication. (e.g., `/etc/nginx/sites-available/gitlab-ssl`) and configure NGINX to listen to port `8443` in addition to the standard HTTPS port: - ```conf - server { - listen 0.0.0.0:443 ssl; - listen [::]:443 ipv6only=on ssl default_server; - listen 0.0.0.0:8443 ssl; - listen [::]:8443 ipv6only=on ssl; - ``` + ```conf + server { + listen 0.0.0.0:443 ssl; + listen [::]:443 ipv6only=on ssl default_server; + listen 0.0.0.0:8443 ssl; + listen [::]:8443 ipv6only=on ssl; + ``` 1. Update the Kerberos section of [gitlab.yml]: - ```yaml - kerberos: - # Dedicated port: Git before 2.4 does not fall back to Basic authentication if Negotiate fails. - # To support both Basic and Negotiate methods with older versions of Git, configure - # nginx to proxy GitLab on an extra port (e.g. 8443) and uncomment the following lines - # to dedicate this port to Kerberos authentication. (default: false) - use_dedicated_port: true - port: 8443 - https: true - ``` + ```yaml + kerberos: + # Dedicated port: Git before 2.4 does not fall back to Basic authentication if Negotiate fails. + # To support both Basic and Negotiate methods with older versions of Git, configure + # nginx to proxy GitLab on an extra port (e.g. 8443) and uncomment the following lines + # to dedicate this port to Kerberos authentication. (default: false) + use_dedicated_port: true + port: 8443 + https: true + ``` 1. [Restart GitLab] and NGINX for the changes to take effect. @@ -178,11 +178,11 @@ keep offering only `basic` authentication. 1. Edit `/etc/gitlab/gitlab.rb`: - ```ruby - gitlab_rails['kerberos_use_dedicated_port'] = true - gitlab_rails['kerberos_port'] = 8443 - gitlab_rails['kerberos_https'] = true - ``` + ```ruby + gitlab_rails['kerberos_use_dedicated_port'] = true + gitlab_rails['kerberos_port'] = 8443 + gitlab_rails['kerberos_https'] = true + ``` 1. [Reconfigure GitLab] for the changes to take effect. @@ -214,12 +214,12 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` / 1. Edit [gitlab.yml] and remove the `- { name: 'kerberos' }` line under omniauth providers: - ```yaml - omniauth: - # ... - providers: - - { name: 'kerberos' } # <-- remove this line - ``` + ```yaml + omniauth: + # ... + providers: + - { name: 'kerberos' } # <-- remove this line + ``` 1. [Restart GitLab] for the changes to take effect. @@ -230,11 +230,11 @@ remove the OmniAuth provider named `kerberos` from your `gitlab.yml` / 1. Edit `/etc/gitlab/gitlab.rb` and remove the `{ "name" => "kerberos" }` line under `gitlab_rails['omniauth_providers']`: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { "name" => "kerberos" } # <-- remove this entry - ] - ``` + ```ruby + gitlab_rails['omniauth_providers'] = [ + { "name" => "kerberos" } # <-- remove this entry + ] + ``` 1. [Reconfigure GitLab] for the changes to take effect. @@ -290,7 +290,7 @@ remote: HTTP Basic: Access denied fatal: Authentication failed for '<KRB5 path>' ``` -If you are using Git v2.11 or newer and see the above error when cloning, you can +If you are using Git v2.11 or newer and see the above error when cloning, you can set the `http.emptyAuth` Git option to `true` to fix this: ``` diff --git a/doc/integration/oauth2_generic.md b/doc/integration/oauth2_generic.md index 3c1a0f2a117..f4119b1d1ce 100644 --- a/doc/integration/oauth2_generic.md +++ b/doc/integration/oauth2_generic.md @@ -24,11 +24,11 @@ This strategy is designed to allow configuration of the simple OmniAuth SSO proc 1. Register your application in the OAuth2 provider you wish to authenticate with. - The redirect URI you provide when registering the application should be: + The redirect URI you provide when registering the application should be: - ``` - http://your-gitlab.host.com/users/auth/oauth2_generic/callback - ``` + ``` + http://your-gitlab.host.com/users/auth/oauth2_generic/callback + ``` 1. You should now be able to get a Client ID and Client Secret. Where this shows up will differ for each provider. @@ -36,18 +36,18 @@ This strategy is designed to allow configuration of the simple OmniAuth SSO proc 1. On your GitLab server, open the configuration file. - For Omnibus package: + For Omnibus package: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For installations from source: + For installations from source: - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` 1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings diff --git a/doc/integration/omniauth.md b/doc/integration/omniauth.md index bf5debc7694..7a92ed994c7 100644 --- a/doc/integration/omniauth.md +++ b/doc/integration/omniauth.md @@ -71,57 +71,57 @@ To change these settings: - **For omnibus package** - Open the configuration file: + Open the configuration file: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - and change: + and change: - ```ruby - # Versions prior to 11.4 require this to be set to true - # gitlab_rails['omniauth_enabled'] = nil + ```ruby + # Versions prior to 11.4 require this to be set to true + # gitlab_rails['omniauth_enabled'] = nil - # CAUTION! - # This allows users to login without having a user account first. Define the allowed providers - # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. - # User accounts will be created automatically when authentication was successful. - gitlab_rails['omniauth_allow_single_sign_on'] = ['saml', 'twitter'] - gitlab_rails['omniauth_auto_link_ldap_user'] = true - gitlab_rails['omniauth_block_auto_created_users'] = true - ``` + # CAUTION! + # This allows users to login without having a user account first. Define the allowed providers + # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. + # User accounts will be created automatically when authentication was successful. + gitlab_rails['omniauth_allow_single_sign_on'] = ['saml', 'twitter'] + gitlab_rails['omniauth_auto_link_ldap_user'] = true + gitlab_rails['omniauth_block_auto_created_users'] = true + ``` - **For installations from source** - Open the configuration file: + Open the configuration file: - ```sh - cd /home/git/gitlab + ```sh + cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` + sudo -u git -H editor config/gitlab.yml + ``` - and change the following section: + and change the following section: - ```yaml - ## OmniAuth settings - omniauth: - # Allow login via Twitter, Google, etc. using OmniAuth providers - # Versions prior to 11.4 require this to be set to true - # enabled: true + ```yaml + ## OmniAuth settings + omniauth: + # Allow login via Twitter, Google, etc. using OmniAuth providers + # Versions prior to 11.4 require this to be set to true + # enabled: true - # CAUTION! - # This allows users to login without having a user account first. Define the allowed providers - # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. - # User accounts will be created automatically when authentication was successful. - allow_single_sign_on: ["saml", "twitter"] + # CAUTION! + # This allows users to login without having a user account first. Define the allowed providers + # using an array, e.g. ["saml", "twitter"], or as true/false to allow all providers or none. + # User accounts will be created automatically when authentication was successful. + allow_single_sign_on: ["saml", "twitter"] - auto_link_ldap_user: true + auto_link_ldap_user: true - # Locks down those users until they have been cleared by the admin (default: true). - block_auto_created_users: true - ``` + # Locks down those users until they have been cleared by the admin (default: true). + block_auto_created_users: true + ``` Now we can choose one or more of the [Supported Providers](#supported-providers) listed above to continue the configuration process. @@ -161,14 +161,14 @@ want their accounts to be upgraded to full internal accounts. **For Omnibus installations** ```ruby - gitlab_rails['omniauth_external_providers'] = ['twitter', 'google_oauth2'] +gitlab_rails['omniauth_external_providers'] = ['twitter', 'google_oauth2'] ``` **For installations from source** ```yaml - omniauth: - external_providers: ['twitter', 'google_oauth2'] +omniauth: + external_providers: ['twitter', 'google_oauth2'] ``` ## Using Custom Omniauth Providers @@ -186,23 +186,31 @@ these cases you can use the Omniauth provider. These steps are fairly general and you will need to figure out the exact details from the Omniauth provider's documentation. -- Stop GitLab: +- Stop GitLab: - sudo service gitlab stop + ```sh + sudo service gitlab stop + ``` -- Add the gem to your [Gemfile](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Gemfile): +- Add the gem to your [Gemfile](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/Gemfile): - gem "omniauth-your-auth-provider" + ```sh + gem "omniauth-your-auth-provider" + ``` -- Install the new Omniauth provider gem by running the following command: +- Install the new Omniauth provider gem by running the following command: - sudo -u git -H bundle install --without development test mysql --path vendor/bundle --no-deployment + ```sh + sudo -u git -H bundle install --without development test mysql --path vendor/bundle --no-deployment + ``` - > These are the same commands you used during initial installation in the [Install Gems section](../install/installation.md#install-gems) with `--path vendor/bundle --no-deployment` instead of `--deployment`. + > These are the same commands you used during initial installation in the [Install Gems section](../install/installation.md#install-gems) with `--path vendor/bundle --no-deployment` instead of `--deployment`. -- Start GitLab: +- Start GitLab: - sudo service gitlab start + ```sh + sudo service gitlab start + ``` ### Examples @@ -247,8 +255,8 @@ gitlab_rails['omniauth_enabled'] = false **For installations from source** ```yaml - omniauth: - enabled: false +omniauth: + enabled: false ``` ## Keep OmniAuth user profiles up to date @@ -258,14 +266,14 @@ You can enable profile syncing from selected OmniAuth providers and for all or f When authenticating using LDAP, the user's name and email are always synced. ```ruby - gitlab_rails['sync_profile_from_provider'] = ['twitter', 'google_oauth2'] - gitlab_rails['sync_profile_attributes'] = ['name', 'email', 'location'] - ``` +gitlab_rails['sync_profile_from_provider'] = ['twitter', 'google_oauth2'] +gitlab_rails['sync_profile_attributes'] = ['name', 'email', 'location'] +``` **For installations from source** ```yaml - omniauth: - sync_profile_from_provider: ['twitter', 'google_oauth2'] - sync_profile_attributes: ['email', 'location'] +omniauth: + sync_profile_from_provider: ['twitter', 'google_oauth2'] + sync_profile_attributes: ['email', 'location'] ``` diff --git a/doc/integration/salesforce.md b/doc/integration/salesforce.md index 1ef43cfcece..176622e8050 100644 --- a/doc/integration/salesforce.md +++ b/doc/integration/salesforce.md @@ -7,73 +7,77 @@ You can integrate your GitLab instance with [Salesforce](https://www.salesforce. To enable Salesforce OmniAuth provider, you must use Salesforce's credentials for your GitLab instance. To get the credentials (a pair of Client ID and Client Secret), you must [create a Connected App](https://help.salesforce.com/articleView?id=connected_app_create.htm&type=5) on Salesforce. -1. Sign in to [Salesforce](https://login.salesforce.com/). +1. Sign in to [Salesforce](https://login.salesforce.com/). -1. In Setup, enter `App Manager` in the Quick Find box, click **App Manager**, then click **New Connected App**. +1. In Setup, enter `App Manager` in the Quick Find box, click **App Manager**, then click **New Connected App**. -1. Fill in the application details into the following fields: - - **Connected App Name** and **API Name**: Set to any value but consider something like `<Organization>'s GitLab`, `<Your Name>'s GitLab`, or something else that is descriptive. - - **Contact Email**: Enter the contact email for Salesforce to use when contacting you or your support team. - - **Description**: Description for the application. +1. Fill in the application details into the following fields: + - **Connected App Name** and **API Name**: Set to any value but consider something like `<Organization>'s GitLab`, `<Your Name>'s GitLab`, or something else that is descriptive. + - **Contact Email**: Enter the contact email for Salesforce to use when contacting you or your support team. + - **Description**: Description for the application. -  -1. Select **API (Enable OAuth Settings)** and click on **Enable OAuth Settings**. -1. Fill in the application details into the following fields: - - **Callback URL**: The callback URL of your GitLab installation. For example, `https://gitlab.example.com/users/auth/salesforce/callback`. - - **Selected OAuth Scopes**: Move **Access your basic information (id, profile, email, address, phone)** and **Allow access to your unique identifier (openid)** to the right column. +  + +1. Select **API (Enable OAuth Settings)** and click on **Enable OAuth Settings**. +1. Fill in the application details into the following fields: + - **Callback URL**: The callback URL of your GitLab installation. For example, `https://gitlab.example.com/users/auth/salesforce/callback`. + - **Selected OAuth Scopes**: Move **Access your basic information (id, profile, email, address, phone)** and **Allow access to your unique identifier (openid)** to the right column. + +  -  1. Click **Save**. -1. On your GitLab server, open the configuration file. +1. On your GitLab server, open the configuration file. + + For omnibus package: + + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` - For omnibus package: + For installations from source: - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` - For installations from source: +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` +1. Add the provider configuration: -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. + For omnibus package: -1. Add the provider configuration: + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "salesforce", + "app_id" => "SALESFORCE_CLIENT_ID", + "app_secret" => "SALESFORCE_CLIENT_SECRET" + } + ] + ``` - For omnibus package: + For installation from source: - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "salesforce", - "app_id" => "SALESFORCE_CLIENT_ID", - "app_secret" => "SALESFORCE_CLIENT_SECRET" - } - ] - ``` + ``` + - { name: 'salesforce', + app_id: 'SALESFORCE_CLIENT_ID', + app_secret: 'SALESFORCE_CLIENT_SECRET' + } + ``` - For installation from source: +1. Change `SALESFORCE_CLIENT_ID` to the Consumer Key from the Salesforce connected application page. +1. Change `SALESFORCE_CLIENT_SECRET` to the Consumer Secret from the Salesforce connected application page. - ``` - - { name: 'salesforce', - app_id: 'SALESFORCE_CLIENT_ID', - app_secret: 'SALESFORCE_CLIENT_SECRET' - } - ``` -1. Change `SALESFORCE_CLIENT_ID` to the Consumer Key from the Salesforce connected application page. -1. Change `SALESFORCE_CLIENT_SECRET` to the Consumer Secret from the Salesforce connected application page. -  +  -1. Save the configuration file. -1. [Reconfigure GitLab]( ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure ) or [restart GitLab]( ../administration/restart_gitlab.md#installations-from-source ) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. +1. Save the configuration file. +1. [Reconfigure GitLab]( ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure ) or [restart GitLab]( ../administration/restart_gitlab.md#installations-from-source ) for the changes to take effect if you installed GitLab via Omnibus or from source respectively. On the sign in page, there should now be a Salesforce icon below the regular sign in form. Click the icon to begin the authentication process. Salesforce will ask the user to sign in and authorize the GitLab application. If everything goes well, the user will be returned to GitLab and will be signed in. NOTE: **Note:** -GitLab requires the email address of each new user. Once the user is logged in using Salesforce, GitLab will redirect the user to the profile page where they will have to provide the email and verify the email.
\ No newline at end of file +GitLab requires the email address of each new user. Once the user is logged in using Salesforce, GitLab will redirect the user to the profile page where they will have to provide the email and verify the email. diff --git a/doc/integration/shibboleth.md b/doc/integration/shibboleth.md index 07c83c1a049..27355d25266 100644 --- a/doc/integration/shibboleth.md +++ b/doc/integration/shibboleth.md @@ -14,35 +14,35 @@ The following changes are needed to enable Shibboleth: 1. Protect omniauth-shibboleth callback URL: - ``` - <Location /users/auth/shibboleth/callback> - AuthType shibboleth - ShibRequestSetting requireSession 1 - ShibUseHeaders On - require valid-user - </Location> - - Alias /shibboleth-sp /usr/share/shibboleth - <Location /shibboleth-sp> - Satisfy any - </Location> - - <Location /Shibboleth.sso> - SetHandler shib - </Location> - ``` + ``` + <Location /users/auth/shibboleth/callback> + AuthType shibboleth + ShibRequestSetting requireSession 1 + ShibUseHeaders On + require valid-user + </Location> + + Alias /shibboleth-sp /usr/share/shibboleth + <Location /shibboleth-sp> + Satisfy any + </Location> + + <Location /Shibboleth.sso> + SetHandler shib + </Location> + ``` 1. Exclude shibboleth URLs from rewriting. Add `RewriteCond %{REQUEST_URI} !/Shibboleth.sso` and `RewriteCond %{REQUEST_URI} !/shibboleth-sp`. Config should look like this: - ``` - # Apache equivalent of Nginx try files - RewriteEngine on - RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f - RewriteCond %{REQUEST_URI} !/Shibboleth.sso - RewriteCond %{REQUEST_URI} !/shibboleth-sp - RewriteRule .* http://127.0.0.1:8080%{REQUEST_URI} [P,QSA] - RequestHeader set X_FORWARDED_PROTO 'https' - ``` + ``` + # Apache equivalent of Nginx try files + RewriteEngine on + RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f + RewriteCond %{REQUEST_URI} !/Shibboleth.sso + RewriteCond %{REQUEST_URI} !/shibboleth-sp + RewriteRule .* http://127.0.0.1:8080%{REQUEST_URI} [P,QSA] + RequestHeader set X_FORWARDED_PROTO 'https' + ``` 1. Edit `/etc/gitlab/gitlab.rb` configuration file to enable OmniAuth and add Shibboleth as an OmniAuth provider. User attributes will be sent from the @@ -60,31 +60,31 @@ The following changes are needed to enable Shibboleth: The file should look like this: - ``` - external_url 'https://gitlab.example.com' - gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - - # disable Nginx - nginx['enable'] = false - - gitlab_rails['omniauth_allow_single_sign_on'] = true - gitlab_rails['omniauth_block_auto_created_users'] = false - gitlab_rails['omniauth_enabled'] = true - gitlab_rails['omniauth_providers'] = [ - { - "name" => "'shibboleth"', - "label" => "Text for Login Button", - "args" => { - "shib_session_id_field" => "HTTP_SHIB_SESSION_ID", - "shib_application_id_field" => "HTTP_SHIB_APPLICATION_ID", - "uid_field" => 'HTTP_EPPN', - "name_field" => 'HTTP_CN', - "info_fields" => { "email" => 'HTTP_MAIL'} - } - } - ] - - ``` + ``` + external_url 'https://gitlab.example.com' + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' + + # disable Nginx + nginx['enable'] = false + + gitlab_rails['omniauth_allow_single_sign_on'] = true + gitlab_rails['omniauth_block_auto_created_users'] = false + gitlab_rails['omniauth_enabled'] = true + gitlab_rails['omniauth_providers'] = [ + { + "name" => "'shibboleth"', + "label" => "Text for Login Button", + "args" => { + "shib_session_id_field" => "HTTP_SHIB_SESSION_ID", + "shib_application_id_field" => "HTTP_SHIB_APPLICATION_ID", + "uid_field" => 'HTTP_EPPN', + "name_field" => 'HTTP_CN', + "info_fields" => { "email" => 'HTTP_MAIL'} + } + } + ] + + ``` 1. [Reconfigure](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart](../administration/restart_gitlab.md#installations-from-source) GitLab for the changes to take effect if you installed GitLab via Omnibus or from source respectively. @@ -97,44 +97,44 @@ The order of the first 2 Location directives is important. If they are reversed, you will not get a shibboleth session! ``` - <Location /> - Require all granted - ProxyPassReverse http://127.0.0.1:8181 - ProxyPassReverse http://YOUR_SERVER_FQDN/ - </Location> - - <Location /users/auth/shibboleth/callback> - AuthType shibboleth - ShibRequestSetting requireSession 1 - ShibUseHeaders On - Require shib-session - </Location> - - Alias /shibboleth-sp /usr/share/shibboleth - - <Location /shibboleth-sp> - Require all granted - </Location> - - <Location /Shibboleth.sso> - SetHandler shib - </Location> - - RewriteEngine on - - #Don't escape encoded characters in api requests - RewriteCond %{REQUEST_URI} ^/api/v4/.* - RewriteCond %{REQUEST_URI} !/Shibboleth.sso - RewriteCond %{REQUEST_URI} !/shibboleth-sp - RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA,NE] - - #Forward all requests to gitlab-workhorse except existing files - RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f [OR] - RewriteCond %{REQUEST_URI} ^/uploads/.* - RewriteCond %{REQUEST_URI} !/Shibboleth.sso - RewriteCond %{REQUEST_URI} !/shibboleth-sp - RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA] - - RequestHeader set X_FORWARDED_PROTO 'https' - RequestHeader set X-Forwarded-Ssl on +<Location /> + Require all granted + ProxyPassReverse http://127.0.0.1:8181 + ProxyPassReverse http://YOUR_SERVER_FQDN/ +</Location> + +<Location /users/auth/shibboleth/callback> + AuthType shibboleth + ShibRequestSetting requireSession 1 + ShibUseHeaders On + Require shib-session +</Location> + +Alias /shibboleth-sp /usr/share/shibboleth + +<Location /shibboleth-sp> + Require all granted +</Location> + +<Location /Shibboleth.sso> + SetHandler shib +</Location> + +RewriteEngine on + +#Don't escape encoded characters in api requests +RewriteCond %{REQUEST_URI} ^/api/v4/.* +RewriteCond %{REQUEST_URI} !/Shibboleth.sso +RewriteCond %{REQUEST_URI} !/shibboleth-sp +RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA,NE] + +#Forward all requests to gitlab-workhorse except existing files +RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f [OR] +RewriteCond %{REQUEST_URI} ^/uploads/.* +RewriteCond %{REQUEST_URI} !/Shibboleth.sso +RewriteCond %{REQUEST_URI} !/shibboleth-sp +RewriteRule .* http://127.0.0.1:8181%{REQUEST_URI} [P,QSA] + +RequestHeader set X_FORWARDED_PROTO 'https' +RequestHeader set X-Forwarded-Ssl on ``` diff --git a/doc/integration/ultra_auth.md b/doc/integration/ultra_auth.md index 69b2a75050d..9ed1bdb4882 100644 --- a/doc/integration/ultra_auth.md +++ b/doc/integration/ultra_auth.md @@ -7,69 +7,78 @@ You can integrate your GitLab instance with [UltraAuth](https://ultraauth.com) t To enable UltraAuth OmniAuth provider, you must use UltraAuth's credentials for your GitLab instance. To get the credentials (a pair of Client ID and Client Secret), you must register an application on UltraAuth. -1. Sign in to [UltraAuth](https://ultraauth.com). -1. Navigate to [Create an App](https://ultraauth.com/select-strategy) and click on "Ruby on Rails". -1. Scroll down the page that is displayed to locate the **Client ID** and **Client Secret**. - Keep this page open as you continue configuration. -  -1. Click on "Edit Callback URL" link. -  -1. The callback URL will be `http(s)://<your_domain>/users/auth/ultraauth/callback` -  -1. Select **Register application**. -1. On your GitLab server, open the configuration file. - - For omnibus package: - - ```sh - sudo editor /etc/gitlab/gitlab.rb - ``` - - For installations from source: - - ```sh - cd /home/git/gitlab - sudo -u git -H editor config/gitlab.yml - ``` -1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. -1. Add the provider configuration: - - For omnibus package: - - ```ruby - gitlab_rails['omniauth_providers'] = [ - { - "name" => "ultraauth", - "app_id" => "OPENID_CLIENT_ID", - "app_secret" => "OPENID_CLIENT_SECRET", - "args" => { - "client_options" => { - "redirect_uri" => "https://example.com/users/auth/ultraauth/callback" - } - } - } - ] - ``` - - For installation from source: - - ``` - - { name: 'ultraauth', - app_id: 'OPENID_CLIENT_ID', - app_secret: 'OPENID_CLIENT_SECRET', - args: { - client_options: { - redirect_uri: 'https://example.com/users/auth/ultraauth/callback' - } - } - } - ``` - __Replace `https://example.com/users/auth/ultraauth/callback` with your application's Callback URL.__ -1. Change `OPENID_CLIENT_ID` to the Client ID from the UltraAuth application page. -1. Change `OPENID_CLIENT_SECRET` to the Client Secret from the UltraAuth application page. -1. Save the configuration file. -1. [Reconfigure GitLab]( ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure ) or [restart GitLab]( ../administration/restart_gitlab.md#installations-from-source ) for the changes to take effect if you - installed GitLab via Omnibus or from source respectively. +1. Sign in to [UltraAuth](https://ultraauth.com). +1. Navigate to [Create an App](https://ultraauth.com/select-strategy) and click on "Ruby on Rails". +1. Scroll down the page that is displayed to locate the **Client ID** and **Client Secret**. + Keep this page open as you continue configuration. + +  + +1. Click on "Edit Callback URL" link. + +  + +1. The callback URL will be `http(s)://<your_domain>/users/auth/ultraauth/callback` + +  + +1. Select **Register application**. +1. On your GitLab server, open the configuration file. + + For omnibus package: + + ```sh + sudo editor /etc/gitlab/gitlab.rb + ``` + + For installations from source: + + ```sh + cd /home/git/gitlab + sudo -u git -H editor config/gitlab.yml + ``` + +1. See [Initial OmniAuth Configuration](omniauth.md#initial-omniauth-configuration) for initial settings. +1. Add the provider configuration: + + For omnibus package: + + ```ruby + gitlab_rails['omniauth_providers'] = [ + { + "name" => "ultraauth", + "app_id" => "OPENID_CLIENT_ID", + "app_secret" => "OPENID_CLIENT_SECRET", + "args" => { + "client_options" => { + "redirect_uri" => "https://example.com/users/auth/ultraauth/callback" + } + } + } + ] + ``` + + For installation from source: + + ``` + - { name: 'ultraauth', + app_id: 'OPENID_CLIENT_ID', + app_secret: 'OPENID_CLIENT_SECRET', + args: { + client_options: { + redirect_uri: 'https://example.com/users/auth/ultraauth/callback' + } + } + } + ``` + + __Replace `https://example.com/users/auth/ultraauth/callback` with your application's Callback URL.__ + +1. Change `OPENID_CLIENT_ID` to the Client ID from the UltraAuth application page. +1. Change `OPENID_CLIENT_SECRET` to the Client Secret from the UltraAuth application page. +1. Save the configuration file. +1. [Reconfigure GitLab](../administration/restart_gitlab.md#omnibus-gitlab-reconfigure) or [restart GitLab](../administration/restart_gitlab.md#installations-from-source) for the changes to take effect if you + installed GitLab via Omnibus or from source respectively. On the sign in page, there should now be an UltraAuth icon below the regular sign in form. Click the icon to begin the authentication process. UltraAuth will ask the user to sign in and authorize the GitLab application. diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index 092b4375208..9b1a4105dc3 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -741,9 +741,10 @@ sudo gitlab-rake gitlab:backup:restore BACKUP=1493107454_2018_04_25_10.6.4-ce Next, restore `/etc/gitlab/gitlab-secrets.json` if necessary as mentioned above. -Restart and check GitLab: +Reconfigure, restart and check GitLab: ```shell +sudo gitlab-ctl reconfigure sudo gitlab-ctl restart sudo gitlab-rake gitlab:check SANITIZE=true ``` diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index b59c06a24ea..8f65fab366e 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -100,7 +100,7 @@ the git repository's config file. This section is formatted as follows: ``` [gitlab] - fullpath = gitlab-org/gitlab-ce + fullpath = gitlab-org/gitlab-ce ``` However, existing repositories were not migrated to include this path. diff --git a/doc/user/admin_area/geo_nodes.md b/doc/user/admin_area/geo_nodes.md index 9e7057f93d4..39753fd885e 100644 --- a/doc/user/admin_area/geo_nodes.md +++ b/doc/user/admin_area/geo_nodes.md @@ -64,8 +64,8 @@ Internal URL defaults to External URL, but you can customize it under CAUTION: **Warning:** We recommend using an HTTPS connection while configuring the Geo nodes. To avoid breaking communication between **primary** and **secondary** nodes when using -HTTPS, customize your Internal URL to point to a Load Balancer with TLS -termination. +HTTPS, customize your Internal URL to point to a load balancer with TLS +terminated at the load balancer. ## Multiple secondary nodes behind a load balancer diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index a06c3d3c662..a8473f76733 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -102,12 +102,15 @@ You must do the following: 1. Ensure GitLab can manage Knative: - For a non-GitLab managed cluster, ensure that the service account for the token provided can manage resources in the `serving.knative.dev` API group. - - For a GitLab managed cluster, - GitLab uses a service account with the `edit` cluster role. This account needs - the ability to manage resources in the `serving.knative.dev` API group. - We suggest you do this with an [aggregated ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles) - adding rules to the default `edit` cluster role: - First, save the following YAML as `knative-serving-only-role.yaml`: + - For a GitLab managed cluster, if you added the cluster in [GitLab 12.1 or later](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/30235), + then GitLab will already have the required access and you can proceed to the next step. + + Otherwise, you need to manually grant GitLab's service account the ability to manage + resources in the `serving.knative.dev` API group. Since every GitLab service account + has the `edit` cluster role, the simplest way to do this is with an + [aggregated ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles) + adding rules to the default `edit` cluster role: First, save the following YAML as + `knative-serving-only-role.yaml`: ```yaml apiVersion: rbac.authorization.k8s.io/v1 @@ -143,6 +146,9 @@ You must do the following: kubectl apply -f knative-serving-only-role.yaml ``` + If you would rather grant permissions on a per service account basis, you can do this + using a `Role` and `RoleBinding` specific to the service account and namespace. + 1. Follow the steps to deploy [functions](#deploying-functions) or [serverless applications](#deploying-serverless-applications) onto your cluster. @@ -376,13 +382,13 @@ cluster. By default, a GitLab serverless deployment will be served over `http`. In order to serve over `https` you must manually obtain and install TLS certificates. -The simplest way to accomplish this is to +The simplest way to accomplish this is to use [Certbot to manually obtain Let's Encrypt certificates](https://knative.dev/docs/serving/using-a-tls-cert/#using-certbot-to-manually-obtain-let-s-encrypt-certificates). Certbot is a free, open source software tool for automatically using Let’s Encrypt certificates on manually-administrated websites to enable HTTPS. NOTE: **Note:** The instructions below relate to installing and running Certbot on a Linux server and may not work on other operating systems. -1. Install Certbot by running the +1. Install Certbot by running the [`certbot-auto` wrapper script](https://certbot.eff.org/docs/install.html#certbot-auto). On the command line of your server, run the following commands: @@ -594,7 +600,7 @@ The instructions below relate to installing and running Certbot on a Linux serve Where `cert.pem` and `cert.pk` are your certificate and private key files. Note that the `istio-ingressgateway-certs` secret name is required. 1. Configure Knative to use the new secret that you created for HTTPS - connections. Run the + connections. Run the following command to open the Knative shared `gateway` in edit mode: ```sh @@ -641,4 +647,4 @@ The instructions below relate to installing and running Certbot on a Linux serve After your changes are running on your Knative cluster, you can begin using the HTTPS protocol for secure access your deployed Knative services. In the event a mistake is made during this process and you need to update the cert, you will need to edit the gateway `knative-ingress-gateway` - to switch back to `PASSTHROUGH` mode. Once corrections are made, edit the file again so the gateway will use the new certificates.
\ No newline at end of file + to switch back to `PASSTHROUGH` mode. Once corrections are made, edit the file again so the gateway will use the new certificates. diff --git a/doc/user/project/code_owners.md b/doc/user/project/code_owners.md index 78ffa11d59b..96c4f16fe04 100644 --- a/doc/user/project/code_owners.md +++ b/doc/user/project/code_owners.md @@ -57,8 +57,8 @@ Example `CODEOWNERS` file: # Files with a `#` can still be accesssed by escaping the pound sign \#file_with_pound.rb @owner-file-with-pound -# Multiple codeowners can be specified, separated by whitespace -CODEOWNERS @multiple @owners @tab-separated +# Multiple codeowners can be specified, separated by spaces or tabs +CODEOWNERS @multiple @code @owners # Both usernames or email addresses can be used to match # users. Everything else will be ignored. For example this will diff --git a/doc/user/project/import/phabricator.md b/doc/user/project/import/phabricator.md index 5c624e3aff6..b8f89caba24 100644 --- a/doc/user/project/import/phabricator.md +++ b/doc/user/project/import/phabricator.md @@ -15,6 +15,12 @@ Currently, only the following basic fields are imported: - Created at - Closed at +## Users + +The assignee and author of a user are deducted from a Task's owner and +author: If a user with the same username has access to the namespace +of the project being imported into, then the user will be linked. + ## Enabling this feature While this feature is incomplete, a feature flag is required to enable it so that diff --git a/doc/user/project/integrations/project_services.md b/doc/user/project/integrations/project_services.md index 62e08a183f7..f63a2c2758b 100644 --- a/doc/user/project/integrations/project_services.md +++ b/doc/user/project/integrations/project_services.md @@ -20,10 +20,10 @@ Below, you will find a list of the currently supported ones accompanied with com Click on the service links to see further configuration instructions and details. -| Service | Description | +| Service | Description | | ------- | ----------- | -| Asana | Asana - Teamwork without email | -| Assembla | Project Management Software (Source Commits Endpoint) | +| Asana | Asana - Teamwork without email | +| Assembla | Project Management Software (Source Commits Endpoint) | | [Atlassian Bamboo CI](bamboo.md) | A continuous integration and build server | | Buildkite | Continuous integration and deployments | | [Bugzilla](bugzilla.md) | Bugzilla issue tracker | diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index 37a0630d0f3..c299a8ecb96 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -274,7 +274,8 @@ branch already exists, the patches will be applied on top of it. > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/26752) in GitLab 11.10. NOTE: **Note:** -Git push options are only available with Git 2.10 or newer. +Git push options are only available with Git 2.10 or newer. With Git older than 2.18 +`git push --push-option=...` should be used instead of `git push -o ...`. GitLab supports using [Git push options](https://git-scm.com/docs/git-push#Documentation/git-push.txt--oltoptiongt) diff --git a/doc/workflow/lfs/lfs_administration.md b/doc/workflow/lfs/lfs_administration.md index 03af8fad759..55183408a10 100644 --- a/doc/workflow/lfs/lfs_administration.md +++ b/doc/workflow/lfs/lfs_administration.md @@ -148,20 +148,20 @@ On Omnibus installations, the settings are prefixed by `lfs_object_store_`: 1. Edit `/etc/gitlab/gitlab.rb` and add the following lines by replacing with the values you want: - ```ruby - gitlab_rails['lfs_object_store_enabled'] = true - gitlab_rails['lfs_object_store_remote_directory'] = "lfs-objects" - gitlab_rails['lfs_object_store_connection'] = { - 'provider' => 'AWS', - 'region' => 'eu-central-1', - 'aws_access_key_id' => '1ABCD2EFGHI34JKLM567N', - 'aws_secret_access_key' => 'abcdefhijklmnopQRSTUVwxyz0123456789ABCDE', - # The below options configure an S3 compatible host instead of AWS - 'host' => 'localhost', - 'endpoint' => 'http://127.0.0.1:9000', - 'path_style' => true - } - ``` + ```ruby + gitlab_rails['lfs_object_store_enabled'] = true + gitlab_rails['lfs_object_store_remote_directory'] = "lfs-objects" + gitlab_rails['lfs_object_store_connection'] = { + 'provider' => 'AWS', + 'region' => 'eu-central-1', + 'aws_access_key_id' => '1ABCD2EFGHI34JKLM567N', + 'aws_secret_access_key' => 'abcdefhijklmnopQRSTUVwxyz0123456789ABCDE', + # The below options configure an S3 compatible host instead of AWS + 'host' => 'localhost', + 'endpoint' => 'http://127.0.0.1:9000', + 'path_style' => true + } + ``` 1. Save the file and [reconfigure GitLab]s for the changes to take effect. 1. Migrate any existing local LFS objects to the object storage: @@ -182,22 +182,22 @@ For source installations the settings are nested under `lfs:` and then 1. Edit `/home/git/gitlab/config/gitlab.yml` and add or amend the following lines: - ```yaml - lfs: - enabled: true - object_store: - enabled: false - remote_directory: lfs-objects # Bucket name - connection: - provider: AWS - aws_access_key_id: 1ABCD2EFGHI34JKLM567N - aws_secret_access_key: abcdefhijklmnopQRSTUVwxyz0123456789ABCDE - region: eu-central-1 - # Use the following options to configure an AWS compatible host such as Minio - host: 'localhost' - endpoint: 'http://127.0.0.1:9000' - path_style: true - ``` + ```yaml + lfs: + enabled: true + object_store: + enabled: false + remote_directory: lfs-objects # Bucket name + connection: + provider: AWS + aws_access_key_id: 1ABCD2EFGHI34JKLM567N + aws_secret_access_key: abcdefhijklmnopQRSTUVwxyz0123456789ABCDE + region: eu-central-1 + # Use the following options to configure an AWS compatible host such as Minio + host: 'localhost' + endpoint: 'http://127.0.0.1:9000' + path_style: true + ``` 1. Save the file and [restart GitLab][] for the changes to take effect. 1. Migrate any existing local LFS objects to the object storage: diff --git a/jest.config.js b/jest.config.js index 986b8465eef..e4ac71a1a17 100644 --- a/jest.config.js +++ b/jest.config.js @@ -19,7 +19,7 @@ let testMatch = ['<rootDir>/spec/frontend/**/*_spec.js', '<rootDir>/ee/spec/fron // workaround for eslint-import-resolver-jest only resolving in test files // see https://github.com/JoinColony/eslint-import-resolver-jest#note -const isESLint = module.parent.path.includes('/eslint-import-resolver-jest/'); +const isESLint = module.parent.filename.includes('/eslint-import-resolver-jest/'); if (isESLint) { testMatch = testMatch.map(path => path.replace('_spec.js', '')); } diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index c7239a5eaa6..81f32ef5bcf 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -55,11 +55,16 @@ module Banzai # Perform multiple render from an Array of Markdown String into an # Array of HTML-safe String of HTML. # - # As the rendered Markdown String can be already cached read all the data - # from the cache using Rails.cache.read_multi operation. If the Markdown String - # is not in the cache or it's not cacheable (no cache_key entry is provided in - # the context) the Markdown String is rendered and stored in the cache so the - # next render call gets the rendered HTML-safe String from the cache. + # The redis cache is completely obviated if we receive a `:rendered` key in the + # context, as it is assumed the item has been pre-rendered somewhere else and there + # is no need to cache it. + # + # If no `:rendered` key is present in the context, as the rendered Markdown String + # can be already cached, read all the data from the cache using + # Rails.cache.read_multi operation. If the Markdown String is not in the cache + # or it's not cacheable (no cache_key entry is provided in the context) the + # Markdown String is rendered and stored in the cache so the next render call + # gets the rendered HTML-safe String from the cache. # # For further explanation see #render method comments. # @@ -76,19 +81,34 @@ module Banzai # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] def self.cache_collection_render(texts_and_contexts) - items_collection = texts_and_contexts.each_with_index do |item, index| + items_collection = texts_and_contexts.each do |item| context = item[:context] - cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) - item[:cache_key] = cache_key if cache_key + if context.key?(:rendered) + item[:rendered] = context.delete(:rendered) + else + # If the attribute didn't come in pre-rendered, let's prepare it for caching it in redis + cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) + item[:cache_key] = cache_key if cache_key + end end - cacheable_items, non_cacheable_items = items_collection.partition { |item| item.key?(:cache_key) } + cacheable_items, non_cacheable_items = items_collection.group_by do |item| + if item.key?(:rendered) + # We're not really doing anything here as these don't need any processing, but leaving it just in case + # as they could have a cache_key and we don't want them to be re-rendered + :rendered + elsif item.key?(:cache_key) + :cacheable + else + :non_cacheable + end + end.values_at(:cacheable, :non_cacheable) items_in_cache = [] items_not_in_cache = [] - unless cacheable_items.empty? + if cacheable_items.present? items_in_cache = Rails.cache.read_multi(*cacheable_items.map { |item| item[:cache_key] }) items_not_in_cache = cacheable_items.reject do |item| item[:rendered] = items_in_cache[item[:cache_key]] @@ -96,7 +116,7 @@ module Banzai end end - (items_not_in_cache + non_cacheable_items).each do |item| + (items_not_in_cache + Array.wrap(non_cacheable_items)).each do |item| item[:rendered] = render(item[:text], item[:context]) Rails.cache.write(item[:cache_key], item[:rendered]) if item[:cache_key] end diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb new file mode 100644 index 00000000000..61011abddf5 --- /dev/null +++ b/lib/gitlab/batch_pop_queueing.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + ## + # This class is a queuing system for processing expensive tasks in an atomic manner + # with batch poping to let you optimize the total processing time. + # + # In usual queuing system, the first item started being processed immediately + # and the following items wait until the next items have been popped from the queue. + # On the other hand, this queueing system, the former part is same, however, + # it pops the enqueued items as batch. This is especially useful when you want to + # drop redandant items from the queue in order to process important items only, + # thus it's more efficient than the traditional queueing system. + # + # Caveats: + # - The order of the items are not guaranteed because of `sadd` (Redis Sets). + # + # Example: + # ``` + # class TheWorker + # def perform + # result = Gitlab::BatchPopQueueing.new('feature', 'queue').safe_execute([item]) do |items_in_queue| + # item = extract_the_most_important_item_from(items_in_queue) + # expensive_process(item) + # end + # + # if result[:status] == :finished && result[:new_items].present? + # item = extract_the_most_important_item_from(items_in_queue) + # TheWorker.perform_async(item.id) + # end + # end + # end + # ``` + # + class BatchPopQueueing + attr_reader :namespace, :queue_id + + EXTRA_QUEUE_EXPIRE_WINDOW = 1.hour + MAX_COUNTS_OF_POP_ALL = 1000 + + # Initialize queue + # + # @param [String] namespace The namespace of the exclusive lock and queue key. Typically, it's a feature name. + # @param [String] queue_id The identifier of the queue. + # @return [Boolean] + def initialize(namespace, queue_id) + raise ArgumentError if namespace.empty? || queue_id.empty? + + @namespace, @queue_id = namespace, queue_id + end + + ## + # Execute the given block in an exclusive lock. + # If there is the other thread has already working on the block, + # it enqueues the items without processing the block. + # + # @param [Array<String>] new_items New items to be added to the queue. + # @param [Time] lock_timeout The timeout of the exclusive lock. Generally, this value should be longer than the maximum prosess timing of the given block. + # @return [Hash] + # - status => One of the `:enqueued` or `:finished`. + # - new_items => Newly enqueued items during the given block had been processed. + # + # NOTE: If an exception is raised in the block, the poppped items will not be recovered. + # We should NOT re-enqueue the items in this case because it could end up in an infinite loop. + def safe_execute(new_items, lock_timeout: 10.minutes, &block) + enqueue(new_items, lock_timeout + EXTRA_QUEUE_EXPIRE_WINDOW) + + lease = Gitlab::ExclusiveLease.new(lock_key, timeout: lock_timeout) + + return { status: :enqueued } unless uuid = lease.try_obtain + + begin + all_args = pop_all + + yield all_args if block_given? + + { status: :finished, new_items: peek_all } + ensure + Gitlab::ExclusiveLease.cancel(lock_key, uuid) + end + end + + private + + def lock_key + @lock_key ||= "batch_pop_queueing:lock:#{namespace}:#{queue_id}" + end + + def queue_key + @queue_key ||= "batch_pop_queueing:queue:#{namespace}:#{queue_id}" + end + + def enqueue(items, expire_time) + Gitlab::Redis::Queues.with do |redis| + redis.sadd(queue_key, items) + redis.expire(queue_key, expire_time.to_i) + end + end + + def pop_all + Gitlab::Redis::Queues.with do |redis| + redis.spop(queue_key, MAX_COUNTS_OF_POP_ALL) + end + end + + def peek_all + Gitlab::Redis::Queues.with do |redis| + redis.smembers(queue_key) + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 8a84744aa2d..005ea4b7a46 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -22,6 +22,7 @@ code_quality: reports: codequality: gl-code-quality-report.json expire_in: 1 week + dependencies: [] only: - branches - tags diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb new file mode 100644 index 00000000000..ac2a78c0f28 --- /dev/null +++ b/lib/gitlab/graphql/docs/helper.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +module Gitlab + module Graphql + module Docs + # Helper with functions to be used by HAML templates + # This includes graphql-docs gem helpers class. + # You can check the included module on: https://github.com/gjtorikian/graphql-docs/blob/v1.6.0/lib/graphql-docs/helpers.rb + module Helper + include GraphQLDocs::Helpers + + def auto_generated_comment + <<-MD.strip_heredoc + <!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check compile_docs task on lib/tasks/gitlab/graphql.rake. + ---> + MD + end + + # Some fields types are arrays of other types and are displayed + # on docs wrapped in square brackets, for example: [String!]. + # This makes GitLab docs renderer thinks they are links so here + # we change them to be rendered as: String! => Array. + def render_field_type(type) + array_type = type[/\[(.+)\]/, 1] + + if array_type + "#{array_type} => Array" + else + type + end + end + + # We are ignoring connections and built in types for now, + # they should be added when queries are generated. + def objects + graphql_object_types.select do |object_type| + !object_type[:name]["Connection"] && + !object_type[:name]["Edge"] && + !object_type[:name]["__"] + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb new file mode 100644 index 00000000000..f47a372aa19 --- /dev/null +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +module Gitlab + module Graphql + module Docs + # Gitlab renderer for graphql-docs. + # Uses HAML templates to parse markdown and generate .md files. + # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs. + # + # Arguments: + # schema - the GraphQL schema defition. For GitLab should be: GitlabSchema.graphql_definition + # output_dir: The folder where the markdown files will be saved + # template: The path of the haml template to be parsed + class Renderer + include Gitlab::Graphql::Docs::Helper + + def initialize(schema, output_dir:, template:) + @output_dir = output_dir + @template = template + @layout = Haml::Engine.new(File.read(template)) + @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse + end + + def render + contents = @layout.render(self) + + write_file(contents) + end + + private + + def write_file(contents) + filename = File.join(@output_dir, 'index.md') + + FileUtils.mkdir_p(@output_dir) + File.write(filename, contents) + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml new file mode 100644 index 00000000000..cc22d43ab4f --- /dev/null +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -0,0 +1,25 @@ +-# haml-lint:disable UnnecessaryStringOutput + += auto_generated_comment + +:plain + # GraphQL API Resources + + This documentation is self-generated based on GitLab current GraphQL schema. + + The API can be explored interactively using the [GraphiQL IDE](../index.md#graphiql). + + ## Objects +\ +- objects.each do |type| + - unless type[:fields].empty? + = "### #{type[:name]}" + \ + ~ "| Name | Type | Description |" + ~ "| --- | ---- | ---------- |" + - type[:fields].each do |field| + = "| `#{field[:name]}` | #{render_field_type(field[:type][:info])} | #{field[:description]} |" + \ + + + diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 87a31a37e3f..14727b03ce9 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -19,8 +19,12 @@ module Gitlab ENV['ID_TO'] end + def self.using_ranges? + !range_from.nil? && !range_to.nil? + end + def self.range_single_item? - !range_from.nil? && range_from == range_to + using_ranges? && range_from == range_to end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index de14df56555..1350924cd76 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -59,6 +59,13 @@ module Gitlab # RBAC methods delegates to the apis/rbac.authorization.k8s.io api # group client + delegate :create_role, + :get_role, + :update_role, + to: :rbac_client + + # RBAC methods delegates to the apis/rbac.authorization.k8s.io api + # group client delegate :create_role_binding, :get_role_binding, :update_role_binding, diff --git a/lib/gitlab/kubernetes/role.rb b/lib/gitlab/kubernetes/role.rb new file mode 100644 index 00000000000..096f60f0372 --- /dev/null +++ b/lib/gitlab/kubernetes/role.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class Role + def initialize(name:, namespace:, rules:) + @name = name + @namespace = namespace + @rules = rules + end + + def generate + ::Kubeclient::Resource.new( + metadata: { name: name, namespace: namespace }, + rules: rules + ) + end + + private + + attr_reader :name, :namespace, :rules + end + end +end diff --git a/lib/gitlab/kubernetes/role_binding.rb b/lib/gitlab/kubernetes/role_binding.rb index cb0cb42d007..0404fb4453c 100644 --- a/lib/gitlab/kubernetes/role_binding.rb +++ b/lib/gitlab/kubernetes/role_binding.rb @@ -3,9 +3,10 @@ module Gitlab module Kubernetes class RoleBinding - def initialize(name:, role_name:, namespace:, service_account_name:) + def initialize(name:, role_name:, role_kind:, namespace:, service_account_name:) @name = name @role_name = role_name + @role_kind = role_kind @namespace = namespace @service_account_name = service_account_name end @@ -20,7 +21,7 @@ module Gitlab private - attr_reader :name, :role_name, :namespace, :service_account_name + attr_reader :name, :role_name, :role_kind, :namespace, :service_account_name def metadata { name: name, namespace: namespace } @@ -29,7 +30,7 @@ module Gitlab def role_ref { apiGroup: 'rbac.authorization.k8s.io', - kind: 'ClusterRole', + kind: role_kind, name: role_name } end diff --git a/lib/gitlab/markdown_cache/active_record/extension.rb b/lib/gitlab/markdown_cache/active_record/extension.rb index f3abe631779..233d3bf1ac7 100644 --- a/lib/gitlab/markdown_cache/active_record/extension.rb +++ b/lib/gitlab/markdown_cache/active_record/extension.rb @@ -26,10 +26,6 @@ module Gitlab attrs end - def changed_markdown_fields - changed_attributes.keys.map(&:to_s) & cached_markdown_fields.markdown_fields.map(&:to_s) - end - def write_markdown_field(field_name, value) write_attribute(field_name, value) end diff --git a/lib/gitlab/markdown_cache/redis/extension.rb b/lib/gitlab/markdown_cache/redis/extension.rb index 97fc23343b4..af3237f4ba6 100644 --- a/lib/gitlab/markdown_cache/redis/extension.rb +++ b/lib/gitlab/markdown_cache/redis/extension.rb @@ -36,8 +36,8 @@ module Gitlab false end - def changed_markdown_fields - [] + def changed_attributes + {} end def cached_markdown diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 5c2f07b95e2..51f48095cb5 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -63,10 +63,6 @@ module Gitlab def self.monotonic_time Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) end - - def self.clk_tck - @clk_tck ||= `getconf CLK_TCK`.to_i - end end end end diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb index fa8b37b20ca..6a2841b6a8e 100644 --- a/lib/gitlab/phabricator_import/cache/map.rb +++ b/lib/gitlab/phabricator_import/cache/map.rb @@ -9,9 +9,15 @@ module Gitlab def get_gitlab_model(phabricator_id) cached_info = get(phabricator_id) - return unless cached_info[:classname] && cached_info[:database_id] - cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + if cached_info[:classname] && cached_info[:database_id] + object = cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) + else + object = yield if block_given? + set_gitlab_model(object, phabricator_id) if object + end + + object end def set_gitlab_model(object, phabricator_id) diff --git a/lib/gitlab/phabricator_import/conduit/user.rb b/lib/gitlab/phabricator_import/conduit/user.rb new file mode 100644 index 00000000000..fc8c3f7cde9 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/user.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true +module Gitlab + module PhabricatorImport + module Conduit + class User + MAX_PAGE_SIZE = 100 + + def initialize(phabricator_url:, api_token:) + @client = Client.new(phabricator_url, api_token) + end + + def users(phids) + phids.each_slice(MAX_PAGE_SIZE).map { |limited_phids| get_page(limited_phids) } + end + + private + + def get_page(phids) + UsersResponse.new(get_users(phids)) + end + + def get_users(phids) + client.get('user.search', + params: { constraints: { phids: phids } }) + end + + attr_reader :client + end + end + end +end diff --git a/lib/gitlab/phabricator_import/conduit/users_response.rb b/lib/gitlab/phabricator_import/conduit/users_response.rb new file mode 100644 index 00000000000..3dfb29a7be5 --- /dev/null +++ b/lib/gitlab/phabricator_import/conduit/users_response.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + module Conduit + class UsersResponse + def initialize(conduit_response) + @conduit_response = conduit_response + end + + def users + @users ||= conduit_response.data.map do |user_json| + Gitlab::PhabricatorImport::Representation::User.new(user_json) + end + end + + private + + attr_reader :conduit_response + end + end + end +end diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb index 40d4392cbc1..77ee11c7cdd 100644 --- a/lib/gitlab/phabricator_import/issues/task_importer.rb +++ b/lib/gitlab/phabricator_import/issues/task_importer.rb @@ -8,9 +8,7 @@ module Gitlab end def execute - # TODO: get the user from the project namespace from the username loaded by Phab-id - # https://gitlab.com/gitlab-org/gitlab-ce/issues/60565 - issue.author = User.ghost + issue.author = user_finder.find(task.author_phid) || User.ghost # TODO: Reformat the description with attachments, escaping accidental # links and add attachments @@ -19,6 +17,10 @@ module Gitlab save! + if owner = user_finder.find(task.owner_phid) + issue.assignees << owner + end + issue end @@ -41,6 +43,10 @@ module Gitlab project.issues.new end + def user_finder + @issue_finder ||= Gitlab::PhabricatorImport::UserFinder.new(project, task.phids) + end + def find_issue_by_phabricator_id(phabricator_id) object_map.get_gitlab_model(phabricator_id) end diff --git a/lib/gitlab/phabricator_import/representation/task.rb b/lib/gitlab/phabricator_import/representation/task.rb index 6aedc71b626..ba93fb37a8e 100644 --- a/lib/gitlab/phabricator_import/representation/task.rb +++ b/lib/gitlab/phabricator_import/representation/task.rb @@ -11,6 +11,18 @@ module Gitlab json['phid'] end + def author_phid + json['fields']['authorPHID'] + end + + def owner_phid + json['fields']['ownerPHID'] + end + + def phids + @phids ||= [author_phid, owner_phid] + end + def issue_attributes @issue_attributes ||= { title: issue_title, diff --git a/lib/gitlab/phabricator_import/representation/user.rb b/lib/gitlab/phabricator_import/representation/user.rb new file mode 100644 index 00000000000..7fd7cecc6ae --- /dev/null +++ b/lib/gitlab/phabricator_import/representation/user.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + module Representation + class User + def initialize(json) + @json = json + end + + def phabricator_id + json['phid'] + end + + def username + json['fields']['username'] + end + + private + + attr_reader :json + end + end + end +end diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb new file mode 100644 index 00000000000..4b50431e0e0 --- /dev/null +++ b/lib/gitlab/phabricator_import/user_finder.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module PhabricatorImport + class UserFinder + def initialize(project, phids) + @project, @phids = project, phids + @loaded_phids = Set.new + end + + def find(phid) + found_user = object_map.get_gitlab_model(phid) do + find_user_for_phid(phid) + end + + loaded_phids << phid + + found_user + end + + private + + attr_reader :project, :phids, :loaded_phids + + def object_map + @object_map ||= Gitlab::PhabricatorImport::Cache::Map.new(project) + end + + def find_user_for_phid(phid) + phabricator_user = phabricator_users.find { |u| u.phabricator_id == phid } + return unless phabricator_user + + project.authorized_users.find_by_username(phabricator_user.username) + end + + def phabricator_users + @user_responses ||= client.users(users_to_request).flat_map(&:users) + end + + def users_to_request + phids - loaded_phids.to_a + end + + def client + @client ||= + Gitlab::PhabricatorImport::Conduit::User + .new(phabricator_url: project.import_data.data['phabricator_url'], + api_token: project.import_data.credentials[:api_token]) + end + end + end +end diff --git a/lib/tasks/gitlab/graphql.rake b/lib/tasks/gitlab/graphql.rake new file mode 100644 index 00000000000..c53d55ceea2 --- /dev/null +++ b/lib/tasks/gitlab/graphql.rake @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +return if Rails.env.production? + +namespace :gitlab do + OUTPUT_DIR = Rails.root.join("doc/api/graphql/reference").freeze + TEMPLATES_DIR = 'lib/gitlab/graphql/docs/templates/'.freeze + + namespace :graphql do + desc 'GitLab | Generate GraphQL docs' + task compile_docs: :environment do + renderer = Gitlab::Graphql::Docs::Renderer.new(GitlabSchema.graphql_definition, render_options) + + renderer.render + + puts "Documentation compiled." + end + end +end + +def render_options + { + output_dir: OUTPUT_DIR, + template: Rails.root.join(TEMPLATES_DIR, 'default.md.haml') + } +end diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index 954f827f716..ccc96b7edfb 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -3,50 +3,44 @@ namespace :gitlab do desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' task migrate_to_hashed: :environment do if Gitlab::Database.read_only? - warn 'This task requires database write access. Exiting.' - - next + abort 'This task requires database write access. Exiting.' end storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper if storage_migrator.rollback_pending? - warn "There is already a rollback operation in progress, " \ + abort "There is already a rollback operation in progress, " \ "running a migration at the same time may have unexpected consequences." - - next end if helper.range_single_item? project = Project.with_unmigrated_storage.find_by(id: helper.range_from) unless project - warn "There are no projects requiring storage migration with ID=#{helper.range_from}" - - next + abort "There are no projects requiring storage migration with ID=#{helper.range_from}" end puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..." storage_migrator.migrate(project) + else + legacy_projects_count = if helper.using_ranges? + Project.with_unmigrated_storage.id_in(helper.range_from..helper.range_to).count + else + Project.with_unmigrated_storage.count + end + + if legacy_projects_count == 0 + abort 'There are no projects requiring storage migration. Nothing to do!' + end - next - end - - legacy_projects_count = Project.with_unmigrated_storage.count - - if legacy_projects_count == 0 - warn 'There are no projects requiring storage migration. Nothing to do!' - - next - end - - print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" + print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches_migration do |start, finish| - storage_migrator.bulk_schedule_migration(start: start, finish: finish) + helper.project_id_batches_migration do |start, finish| + storage_migrator.bulk_schedule_migration(start: start, finish: finish) - print '.' + print '.' + end end puts ' Done!' @@ -55,50 +49,44 @@ namespace :gitlab do desc 'GitLab | Storage | Rollback existing projects to Legacy Storage' task rollback_to_legacy: :environment do if Gitlab::Database.read_only? - warn 'This task requires database write access. Exiting.' - - next + abort 'This task requires database write access. Exiting.' end storage_migrator = Gitlab::HashedStorage::Migrator.new helper = Gitlab::HashedStorage::RakeHelper if storage_migrator.migration_pending? - warn "There is already a migration operation in progress, " \ + abort "There is already a migration operation in progress, " \ "running a rollback at the same time may have unexpected consequences." - - next end if helper.range_single_item? project = Project.with_storage_feature(:repository).find_by(id: helper.range_from) unless project - warn "There are no projects that can be rolledback with ID=#{helper.range_from}" - - next + abort "There are no projects that can be rolledback with ID=#{helper.range_from}" end puts "Enqueueing storage rollback of #{project.full_path} (ID=#{project.id})..." storage_migrator.rollback(project) + else + hashed_projects_count = if helper.using_ranges? + Project.with_storage_feature(:repository).id_in(helper.range_from..helper.range_to).count + else + Project.with_storage_feature(:repository).count + end + + if hashed_projects_count == 0 + abort 'There are no projects that can have storage rolledback. Nothing to do!' + end - next - end - - hashed_projects_count = Project.with_storage_feature(:repository).count - - if hashed_projects_count == 0 - warn 'There are no projects that can have storage rolledback. Nothing to do!' - - next - end - - print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" + print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" - helper.project_id_batches_rollback do |start, finish| - storage_migrator.bulk_schedule_rollback(start: start, finish: finish) + helper.project_id_batches_rollback do |start, finish| + storage_migrator.bulk_schedule_rollback(start: start, finish: finish) - print '.' + print '.' + end end puts ' Done!' diff --git a/package.json b/package.json index 5955790488a..44aa850860e 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "@babel/preset-env": "^7.4.4", "@gitlab/csslab": "^1.9.0", "@gitlab/svgs": "^1.67.0", - "@gitlab/ui": "^5.5.0", + "@gitlab/ui": "^5.6.0", "apollo-cache-inmemory": "^1.5.1", "apollo-client": "^2.5.1", "apollo-link": "^1.2.11", diff --git a/qa/README.md b/qa/README.md index 124a79a36b4..bab19665dac 100644 --- a/qa/README.md +++ b/qa/README.md @@ -75,14 +75,14 @@ You can also supply specific tests to run as another parameter. For example, to run the repository-related specs, you can execute: ``` -bundle exec bin/qa Test::Instance::All http://localhost -- qa/specs/features/browser_ui/3_create/repository +bundle exec bin/qa Test::Instance::All http://localhost:3000 -- qa/specs/features/browser_ui/3_create/repository ``` Since the arguments would be passed to `rspec`, you could use all `rspec` options there. For example, passing `--backtrace` and also line number: ``` -bundle exec bin/qa Test::Instance::All http://localhost -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb:6 --backtrace +bundle exec bin/qa Test::Instance::All http://localhost:3000 -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb:6 --backtrace ``` Note that the separator `--` is required; all subsequent options will be @@ -140,7 +140,7 @@ tests that are expected to fail while a fix is in progress (similar to how can be used). ``` -bundle exec bin/qa Test::Instance::All http://localhost -- --tag quarantine +bundle exec bin/qa Test::Instance::All http://localhost:3000 -- --tag quarantine ``` If `quarantine` is used with other tags, tests will only be run if they have at @@ -159,7 +159,7 @@ option `--enable-feature FEATURE_FLAG`. For example, to enable the feature flag that enforces Gitaly request limits, you would use the command: ``` -bundle exec bin/qa Test::Instance::All http://localhost --enable-feature gitaly_enforce_requests_limits +bundle exec bin/qa Test::Instance::All http://localhost:3000 --enable-feature gitaly_enforce_requests_limits ``` This will instruct the QA framework to enable the `gitaly_enforce_requests_limits` diff --git a/qa/qa/page/component/note.rb b/qa/qa/page/component/note.rb index 07e191f1c9b..fe324574f4d 100644 --- a/qa/qa/page/component/note.rb +++ b/qa/qa/page/component/note.rb @@ -10,6 +10,10 @@ module QA element :discussion_option end + base.view 'app/assets/javascripts/notes/components/note_actions.vue' do + element :note_edit_button + end + base.view 'app/assets/javascripts/notes/components/note_form.vue' do element :reply_input element :reply_comment_button @@ -49,6 +53,12 @@ module QA def expand_replies click_element :expand_replies end + + def edit_comment(text) + click_element :note_edit_button + fill_element :reply_input, text + click_element :reply_comment_button + end end end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb new file mode 100644 index 00000000000..a62a51b11f4 --- /dev/null +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module QA + context 'Plan' do + describe 'Issue comments' do + it 'user comments on an issue and edits the comment' do + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.act { sign_in_using_credentials } + + issue = Resource::Issue.fabricate_via_api! do |issue| + issue.title = 'issue title' + end + issue.visit! + + Page::Project::Issue::Show.perform do |issue_show_page| + first_version_of_comment = 'First version of the comment' + second_version_of_comment = 'Second version of the comment' + + issue_show_page.comment(first_version_of_comment) + + expect(issue_show_page).to have_content(first_version_of_comment) + + issue_show_page.edit_comment(second_version_of_comment) + + expect(issue_show_page).to have_content(second_version_of_comment) + expect(issue_show_page).not_to have_content(first_version_of_comment) + end + end + end + end +end diff --git a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb index 2b3d9b1711d..36cbd1b81f0 100644 --- a/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/repository/add_list_delete_branches_spec.rb @@ -56,7 +56,7 @@ module QA project.visit! end - it 'branches are correctly listed after CRUD operations' do + it 'lists branches correctly after CRUD operations' do Page::Project::Menu.perform(&:go_to_repository_branches) expect(page).to have_content(master_branch) diff --git a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb index b060f15168c..b2c70547421 100644 --- a/qa/qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb +++ b/qa/qa/specs/features/browser_ui/4_verify/ci_variable/add_ci_variable_spec.rb @@ -7,12 +7,12 @@ module QA Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.perform(&:sign_in_using_credentials) - project = Resource::Project.fabricate! do |project| + project = Resource::Project.fabricate_via_api! do |project| project.name = 'project-with-ci-variables' project.description = 'project with CI variables' end - Resource::CiVariable.fabricate! do |resource| + Resource::CiVariable.fabricate_via_api! do |resource| resource.project = project resource.key = 'VARIABLE_KEY' resource.value = 'some_CI_variable' diff --git a/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb b/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb index caa9be341b4..791dc62e32f 100644 --- a/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb +++ b/qa/qa/specs/features/browser_ui/6_release/deploy_token/add_deploy_token_spec.rb @@ -10,7 +10,7 @@ module QA deploy_token_name = 'deploy token name' one_week_from_now = Date.today + 7 - deploy_token = Resource::DeployToken.fabricate! do |resource| + deploy_token = Resource::DeployToken.fabricate_via_browser_ui! do |resource| resource.name = deploy_token_name resource.expires_at = one_week_from_now end diff --git a/spec/controllers/boards/issues_controller_spec.rb b/spec/controllers/boards/issues_controller_spec.rb index c84bb913cad..6cad060d888 100644 --- a/spec/controllers/boards/issues_controller_spec.rb +++ b/spec/controllers/boards/issues_controller_spec.rb @@ -164,6 +164,201 @@ describe Boards::IssuesController do end end + describe 'PUT move_multiple' do + let(:todo) { create(:group_label, group: group, name: 'Todo') } + let(:development) { create(:group_label, group: group, name: 'Development') } + let(:user) { create(:group_member, :maintainer, user: create(:user), group: group ).user } + let(:guest) { create(:group_member, :guest, user: create(:user), group: group ).user } + let(:project) { create(:project, group: group) } + let(:group) { create(:group) } + let(:board) { create(:board, project: project) } + let(:list1) { create(:list, board: board, label: todo, position: 0) } + let(:list2) { create(:list, board: board, label: development, position: 1) } + let(:issue1) { create(:labeled_issue, project: project, labels: [todo], author: user, relative_position: 10) } + let(:issue2) { create(:labeled_issue, project: project, labels: [todo], author: user, relative_position: 20) } + let(:issue3) { create(:labeled_issue, project: project, labels: [todo], author: user, relative_position: 30) } + let(:issue4) { create(:labeled_issue, project: project, labels: [development], author: user, relative_position: 100) } + + let(:move_params) do + { + board_id: board.id, + ids: [issue1.id, issue2.id, issue3.id], + from_list_id: list1.id, + to_list_id: list2.id, + move_before_id: issue4.id, + move_after_id: nil + } + end + + before do + project.add_maintainer(user) + project.add_guest(guest) + end + + shared_examples 'move issues endpoint provider' do + before do + sign_in(signed_in_user) + end + + it 'moves issues as expected' do + put :bulk_move, params: move_issues_params + expect(response).to have_gitlab_http_status(expected_status) + + list_issues user: requesting_user, board: board, list: list2 + expect(response).to have_gitlab_http_status(200) + + expect(response).to match_response_schema('entities/issue_boards') + + responded_issues = json_response['issues'] + expect(responded_issues.length).to eq expected_issue_count + + ids_in_order = responded_issues.pluck('id') + expect(ids_in_order).to eq(expected_issue_ids_in_order) + end + end + + context 'when items are moved to another list' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) { move_params } + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 4 } + let(:expected_issue_ids_in_order) { [issue4.id, issue1.id, issue2.id, issue3.id] } + end + end + + context 'when moving just one issue' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:ids] = [issue2.id] + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 2 } + let(:expected_issue_ids_in_order) { [issue4.id, issue2.id] } + end + end + + context 'when user is not allowed to move issue' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { guest } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:ids] = [issue2.id] + end + end + let(:requesting_user) { user } + let(:expected_status) { 403 } + let(:expected_issue_count) { 1 } + let(:expected_issue_ids_in_order) { [issue4.id] } + end + end + + context 'when issues should be moved visually above existing issue in list' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:move_after_id] = issue4.id + hash[:move_before_id] = nil + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 4 } + let(:expected_issue_ids_in_order) { [issue1.id, issue2.id, issue3.id, issue4.id] } + end + end + + context 'when destination list is empty' do + before do + # Remove issue from list + issue4.labels -= [development] + issue4.save! + end + + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:move_before_id] = nil + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 3 } + let(:expected_issue_ids_in_order) { [issue1.id, issue2.id, issue3.id] } + end + end + + context 'when no position arguments are given' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:move_before_id] = nil + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 4 } + let(:expected_issue_ids_in_order) { [issue1.id, issue2.id, issue3.id, issue4.id] } + end + end + + context 'when move_before_id and move_after_id are given' do + let(:issue5) { create(:labeled_issue, project: project, labels: [development], author: user, relative_position: 90) } + + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:move_before_id] = issue5.id + hash[:move_after_id] = issue4.id + end + end + let(:requesting_user) { user } + let(:expected_status) { 200 } + let(:expected_issue_count) { 5 } + let(:expected_issue_ids_in_order) { [issue5.id, issue1.id, issue2.id, issue3.id, issue4.id] } + end + end + + context 'when request contains too many issues' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:ids] = (0..51).to_a + end + end + let(:requesting_user) { user } + let(:expected_status) { 422 } + let(:expected_issue_count) { 1 } + let(:expected_issue_ids_in_order) { [issue4.id] } + end + end + + context 'when request is malformed' do + it_behaves_like 'move issues endpoint provider' do + let(:signed_in_user) { user } + let(:move_issues_params) do + move_params.dup.tap do |hash| + hash[:ids] = 'foobar' + end + end + let(:requesting_user) { user } + let(:expected_status) { 400 } + let(:expected_issue_count) { 1 } + let(:expected_issue_ids_in_order) { [issue4.id] } + end + end + end + def list_issues(user:, board:, list: nil) sign_in(user) diff --git a/spec/factories/services.rb b/spec/factories/services.rb index ecb481ed84a..cd1d2c33373 100644 --- a/spec/factories/services.rb +++ b/spec/factories/services.rb @@ -100,4 +100,16 @@ FactoryBot.define do type 'HipchatService' token 'test_token' end + + trait :without_properties_callback do + after(:build) do |service| + allow(service).to receive(:handle_properties) + end + + after(:create) do |service| + # we have to remove the stub because the behaviour of + # handle_properties method is tested after the creation + allow(service).to receive(:handle_properties).and_call_original + end + end end diff --git a/spec/fixtures/phabricator_responses/user.search.json b/spec/fixtures/phabricator_responses/user.search.json new file mode 100644 index 00000000000..f3ec653a23e --- /dev/null +++ b/spec/fixtures/phabricator_responses/user.search.json @@ -0,0 +1,62 @@ +{ + "result": { + "data": [ + { + "id": 1, + "type": "USER", + "phid": "PHID-USER-hohoho", + "fields": { + "username": "jane", + "realName": "Jane Doe", + "roles": [ + "admin", + "verified", + "approved", + "activated" + ], + "dateCreated": 1405970599, + "dateModified": 1406705963, + "policy": { + "view": "public", + "edit": "no-one" + } + }, + "attachments": {} + }, + { + "id": 2, + "type": "USER", + "phid": "PHID-USER-hihihi", + "fields": { + "username": "john", + "realName": "John Doe", + "roles": [ + "admin", + "verified", + "approved", + "activated" + ], + "dateCreated": 1403609184, + "dateModified": 1559138722, + "policy": { + "view": "public", + "edit": "no-one" + } + }, + "attachments": {} + } + ], + "maps": {}, + "query": { + "queryKey": null + }, + "cursor": { + "limit": "100", + "after": null, + "before": null, + "order": null + } + }, + "error_code": null, + "error_info": null +} diff --git a/spec/frontend/filterable_list_spec.js b/spec/frontend/filterable_list_spec.js new file mode 100644 index 00000000000..67d18611661 --- /dev/null +++ b/spec/frontend/filterable_list_spec.js @@ -0,0 +1,53 @@ +import FilterableList from '~/filterable_list'; +import { getJSONFixture, setHTMLFixture } from './helpers/fixtures'; + +describe('FilterableList', () => { + let List; + let form; + let filter; + let holder; + + beforeEach(() => { + setHTMLFixture(` + <form id="project-filter-form"> + <input name="name" class="js-projects-list-filter" /> + </div> + <div class="js-projects-list-holder"></div> + `); + getJSONFixture('static/projects.json'); + form = document.querySelector('form#project-filter-form'); + filter = document.querySelector('.js-projects-list-filter'); + holder = document.querySelector('.js-projects-list-holder'); + List = new FilterableList(form, filter, holder); + }); + + it('processes input parameters', () => { + expect(List.filterForm).toEqual(form); + expect(List.listFilterElement).toEqual(filter); + expect(List.listHolderElement).toEqual(holder); + }); + + describe('getPagePath', () => { + it('returns properly constructed base endpoint', () => { + List.filterForm.action = '/foo/bar/'; + List.listFilterElement.value = 'blah'; + + expect(List.getPagePath()).toEqual('/foo/bar/?name=blah'); + }); + + it('properly appends custom parameters to existing URL', () => { + List.filterForm.action = '/foo/bar?alpha=beta'; + List.listFilterElement.value = 'blah'; + + expect(List.getPagePath()).toEqual('/foo/bar?alpha=beta&name=blah'); + }); + }); + + describe('getFilterEndpoint', () => { + it('returns getPagePath by default', () => { + jest.spyOn(List, 'getPagePath').mockReturnValue('blah/blah/foo'); + + expect(List.getFilterEndpoint()).toEqual(List.getPagePath()); + }); + }); +}); diff --git a/spec/frontend/projects/projects_filterable_list_spec.js b/spec/frontend/projects/projects_filterable_list_spec.js new file mode 100644 index 00000000000..e756fb3ab56 --- /dev/null +++ b/spec/frontend/projects/projects_filterable_list_spec.js @@ -0,0 +1,31 @@ +import ProjectsFilterableList from '~/projects/projects_filterable_list'; +import { getJSONFixture, setHTMLFixture } from '../helpers/fixtures'; + +describe('ProjectsFilterableList', () => { + let List; + let form; + let filter; + let holder; + + beforeEach(() => { + setHTMLFixture(` + <form id="project-filter-form"> + <input name="name" class="js-projects-list-filter" /> + </div> + <div class="js-projects-list-holder"></div> + `); + getJSONFixture('static/projects.json'); + form = document.querySelector('form#project-filter-form'); + filter = document.querySelector('.js-projects-list-filter'); + holder = document.querySelector('.js-projects-list-holder'); + List = new ProjectsFilterableList(form, filter, holder); + }); + + describe('getFilterEndpoint', () => { + it('updates converts getPagePath for projects', () => { + jest.spyOn(List, 'getPagePath').mockReturnValue('blah/projects?'); + + expect(List.getFilterEndpoint()).toEqual('blah/projects.json?'); + }); + }); +}); diff --git a/spec/javascripts/helpers/vuex_action_helper.js b/spec/javascripts/helpers/vuex_action_helper.js index 88652202a8e..c5de31a4138 100644 --- a/spec/javascripts/helpers/vuex_action_helper.js +++ b/spec/javascripts/helpers/vuex_action_helper.js @@ -89,9 +89,7 @@ export default ( payload, ); - return new Promise(resolve => { - setImmediate(resolve); - }) + return new Promise(setImmediate) .then(() => result) .catch(error => { validateResults(); diff --git a/spec/javascripts/registry/components/collapsible_container_spec.js b/spec/javascripts/registry/components/collapsible_container_spec.js index 55017b3e26b..2a5d8dd11da 100644 --- a/spec/javascripts/registry/components/collapsible_container_spec.js +++ b/spec/javascripts/registry/components/collapsible_container_spec.js @@ -77,7 +77,7 @@ describe('collapsible registry container', () => { spyOn(vm, 'deleteItem').and.returnValue(Promise.resolve()); Vue.nextTick(() => { - document.querySelector('#confirm-repo-deletion-modal .btn-danger').click(); + document.querySelector(`#${vm.modalId} .btn-danger`).click(); expect(vm.deleteItem).toHaveBeenCalledWith(vm.repo); done(); diff --git a/spec/javascripts/registry/components/table_registry_spec.js b/spec/javascripts/registry/components/table_registry_spec.js index 6a0b16f592e..31ac970378e 100644 --- a/spec/javascripts/registry/components/table_registry_spec.js +++ b/spec/javascripts/registry/components/table_registry_spec.js @@ -51,7 +51,7 @@ describe('table registry', () => { spyOn(vm, 'deleteItem').and.returnValue(Promise.resolve()); Vue.nextTick(() => { - document.querySelector('#confirm-image-deletion-modal .btn-danger').click(); + document.querySelector(`#${vm.modalId} .btn-danger`).click(); expect(vm.deleteItem).toHaveBeenCalledWith(firstImage); expect(vm.itemToBeDeleted).toBeNull(); diff --git a/spec/javascripts/test_bundle.js b/spec/javascripts/test_bundle.js index 50741e249ca..ce453d7c483 100644 --- a/spec/javascripts/test_bundle.js +++ b/spec/javascripts/test_bundle.js @@ -3,6 +3,7 @@ */ import $ from 'jquery'; +import 'core-js/features/set-immediate'; import 'vendor/jasmine-jquery'; import '~/commons'; import Vue from 'vue'; diff --git a/spec/lib/banzai/renderer_spec.rb b/spec/lib/banzai/renderer_spec.rb index aa828e2f0e9..a099f7482c1 100644 --- a/spec/lib/banzai/renderer_spec.rb +++ b/spec/lib/banzai/renderer_spec.rb @@ -19,6 +19,24 @@ describe Banzai::Renderer do object end + describe '#cache_collection_render' do + let(:merge_request) { fake_object(fresh: true) } + let(:context) { { cache_key: [merge_request, 'field'], rendered: merge_request.field_html } } + + context 'when an item has a rendered field' do + before do + allow(merge_request).to receive(:field).and_return('This is the field') + allow(merge_request).to receive(:field_html).and_return('This is the field') + end + + it 'does not touch redis if the field is in the cache' do + expect(Rails).not_to receive(:cache) + + described_class.cache_collection_render([{ text: merge_request.field, context: context }]) + end + end + end + describe '#render_field' do let(:renderer) { described_class } diff --git a/spec/lib/gitlab/batch_pop_queueing_spec.rb b/spec/lib/gitlab/batch_pop_queueing_spec.rb new file mode 100644 index 00000000000..28984d52024 --- /dev/null +++ b/spec/lib/gitlab/batch_pop_queueing_spec.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::BatchPopQueueing do + include ExclusiveLeaseHelpers + using RSpec::Parameterized::TableSyntax + + describe '#initialize' do + where(:namespace, :queue_id, :expect_error, :error_type) do + 'feature' | '1' | false | nil + :feature | '1' | false | nil + nil | '1' | true | NoMethodError + 'feature' | nil | true | NoMethodError + '' | '1' | true | ArgumentError + 'feature' | '' | true | ArgumentError + 'feature' | 1 | true | NoMethodError + end + + with_them do + it do + if expect_error + expect { described_class.new(namespace, queue_id) }.to raise_error(error_type) + else + expect { described_class.new(namespace, queue_id) }.not_to raise_error + end + end + end + end + + describe '#safe_execute', :clean_gitlab_redis_queues do + subject { queue.safe_execute(new_items, lock_timeout: lock_timeout) } + + let(:queue) { described_class.new(namespace, queue_id) } + let(:namespace) { 'feature' } + let(:queue_id) { '1' } + let(:lock_timeout) { 10.minutes } + let(:new_items) { %w[A B] } + let(:lock_key) { queue.send(:lock_key) } + let(:queue_key) { queue.send(:queue_key) } + + it 'enqueues new items always' do + Gitlab::Redis::Queues.with do |redis| + expect(redis).to receive(:sadd).with(queue_key, new_items) + expect(redis).to receive(:expire).with(queue_key, (lock_timeout + described_class::EXTRA_QUEUE_EXPIRE_WINDOW).to_i) + end + + subject + end + + it 'yields the new items with exclusive lease' do + uuid = 'test' + expect_to_obtain_exclusive_lease(lock_key, uuid, timeout: lock_timeout) + expect_to_cancel_exclusive_lease(lock_key, uuid) + + expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } + .to yield_with_args(match_array(new_items)) + end + + it 'returns the result and no items in the queue' do + expect(subject[:status]).to eq(:finished) + expect(subject[:new_items]).to be_empty + + Gitlab::Redis::Queues.with do |redis| + expect(redis.llen(queue_key)).to be(0) + end + end + + context 'when new items are enqueued during the process' do + it 'returns the result with newly added items' do + result = queue.safe_execute(new_items) do + queue.safe_execute(['C']) + end + + expect(result[:status]).to eq(:finished) + expect(result[:new_items]).to eq(['C']) + + Gitlab::Redis::Queues.with do |redis| + expect(redis.scard(queue_key)).to be(1) + end + end + end + + context 'when interger items are enqueued' do + let(:new_items) { [1, 2, 3] } + + it 'yields as String values' do + expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } + .to yield_with_args(%w[1 2 3]) + end + end + + context 'when the queue key does not exist in Redis' do + before do + allow(queue).to receive(:enqueue) { } + end + + it 'yields empty array' do + expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } + .to yield_with_args([]) + end + end + + context 'when the other process has already been working on the queue' do + before do + stub_exclusive_lease_taken(lock_key, timeout: lock_timeout) + end + + it 'does not yield the block' do + expect { |b| queue.safe_execute(new_items, lock_timeout: lock_timeout, &b) } + .not_to yield_control + end + + it 'returns the result' do + expect(subject[:status]).to eq(:enqueued) + end + end + + context 'when a duplicate item is enqueued' do + it 'returns the poped items to the queue and raise an error' do + expect { |b| queue.safe_execute(%w[1 1 2 2], &b) } + .to yield_with_args(match_array(%w[1 2])) + end + end + + context 'when there are two queues' do + it 'enqueues items to each queue' do + queue_1 = described_class.new(namespace, '1') + queue_2 = described_class.new(namespace, '2') + + result_2 = nil + + result_1 = queue_1.safe_execute(['A']) do |_| + result_2 = queue_2.safe_execute(['B']) do |_| + queue_1.safe_execute(['C']) + queue_2.safe_execute(['D']) + end + end + + expect(result_1[:status]).to eq(:finished) + expect(result_1[:new_items]).to eq(['C']) + expect(result_2[:status]).to eq(:finished) + expect(result_2[:new_items]).to eq(['D']) + end + end + end +end diff --git a/spec/lib/gitlab/kubernetes/kube_client_spec.rb b/spec/lib/gitlab/kubernetes/kube_client_spec.rb index 978e64c4407..97ebb5f1554 100644 --- a/spec/lib/gitlab/kubernetes/kube_client_spec.rb +++ b/spec/lib/gitlab/kubernetes/kube_client_spec.rb @@ -176,6 +176,9 @@ describe Gitlab::Kubernetes::KubeClient do let(:rbac_client) { client.rbac_client } [ + :create_role, + :get_role, + :update_role, :create_cluster_role_binding, :get_cluster_role_binding, :update_cluster_role_binding diff --git a/spec/lib/gitlab/kubernetes/role_binding_spec.rb b/spec/lib/gitlab/kubernetes/role_binding_spec.rb index 50acee254cb..4c200eb545f 100644 --- a/spec/lib/gitlab/kubernetes/role_binding_spec.rb +++ b/spec/lib/gitlab/kubernetes/role_binding_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' describe Gitlab::Kubernetes::RoleBinding, '#generate' do let(:role_name) { 'edit' } + let(:role_kind) { 'ClusterRole' } let(:namespace) { 'my-namespace' } let(:service_account_name) { 'my-service-account' } @@ -20,7 +21,7 @@ describe Gitlab::Kubernetes::RoleBinding, '#generate' do let(:role_ref) do { apiGroup: 'rbac.authorization.k8s.io', - kind: 'ClusterRole', + kind: role_kind, name: role_name } end @@ -37,6 +38,7 @@ describe Gitlab::Kubernetes::RoleBinding, '#generate' do described_class.new( name: "gitlab-#{namespace}", role_name: role_name, + role_kind: role_kind, namespace: namespace, service_account_name: service_account_name ).generate diff --git a/spec/lib/gitlab/kubernetes/role_spec.rb b/spec/lib/gitlab/kubernetes/role_spec.rb new file mode 100644 index 00000000000..3a5cd3b6704 --- /dev/null +++ b/spec/lib/gitlab/kubernetes/role_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Kubernetes::Role do + let(:role) { described_class.new(name: name, namespace: namespace, rules: rules) } + let(:name) { 'example-name' } + let(:namespace) { 'example-namespace' } + + let(:rules) do + [{ + apiGroups: %w(hello.world), + resources: %w(oil diamonds coffee), + verbs: %w(say do walk run) + }] + end + + describe '#generate' do + subject { role.generate } + + let(:resource) do + ::Kubeclient::Resource.new( + metadata: { name: name, namespace: namespace }, + rules: rules + ) + end + + it { is_expected.to eq(resource) } + end +end diff --git a/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb b/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb index 18052b1991c..c5fc74afea5 100644 --- a/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb +++ b/spec/lib/gitlab/markdown_cache/active_record/extension_spec.rb @@ -9,12 +9,13 @@ describe Gitlab::MarkdownCache::ActiveRecord::Extension do cache_markdown_field :title, whitelisted: true cache_markdown_field :description, pipeline: :single_line - attr_accessor :author, :project + attribute :author + attribute :project end end let(:cache_version) { Gitlab::MarkdownCache::CACHE_COMMONMARK_VERSION << 16 } - let(:thing) { klass.new(title: markdown, title_html: html, cached_markdown_version: cache_version) } + let(:thing) { klass.create(title: markdown, title_html: html, cached_markdown_version: cache_version) } let(:markdown) { '`Foo`' } let(:html) { '<p data-sourcepos="1:1-1:5" dir="auto"><code>Foo</code></p>' } @@ -37,7 +38,7 @@ describe Gitlab::MarkdownCache::ActiveRecord::Extension do end context 'a changed markdown field' do - let(:thing) { klass.new(title: markdown, title_html: html, cached_markdown_version: cache_version) } + let(:thing) { klass.create(title: markdown, title_html: html, cached_markdown_version: cache_version) } before do thing.title = updated_markdown diff --git a/spec/lib/gitlab/phabricator_import/cache/map_spec.rb b/spec/lib/gitlab/phabricator_import/cache/map_spec.rb index 52c7a02219f..b6629fad453 100644 --- a/spec/lib/gitlab/phabricator_import/cache/map_spec.rb +++ b/spec/lib/gitlab/phabricator_import/cache/map_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' describe Gitlab::PhabricatorImport::Cache::Map, :clean_gitlab_redis_cache do @@ -28,6 +30,21 @@ describe Gitlab::PhabricatorImport::Cache::Map, :clean_gitlab_redis_cache do expect(ttl).to be > 10.seconds end + + it 'sets the object in redis once if a block was given and nothing was cached' do + issue = create(:issue, project: project) + + expect(map.get_gitlab_model('does not exist') { issue }).to eq(issue) + + expect { |b| map.get_gitlab_model('does not exist', &b) } + .not_to yield_control + end + + it 'does not cache `nil` objects' do + expect(map).not_to receive(:set_gitlab_model) + + map.get_gitlab_model('does not exist') { nil } + end end describe '#set_gitlab_model' do diff --git a/spec/lib/gitlab/phabricator_import/conduit/user_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/user_spec.rb new file mode 100644 index 00000000000..e88eec2c393 --- /dev/null +++ b/spec/lib/gitlab/phabricator_import/conduit/user_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Gitlab::PhabricatorImport::Conduit::User do + let(:user_client) do + described_class.new(phabricator_url: 'https://see-ya-later.phabricator', api_token: 'api-token') + end + + describe '#users' do + let(:fake_client) { double('Phabricator client') } + + before do + allow(user_client).to receive(:client).and_return(fake_client) + end + + it 'calls the api with the correct params' do + expected_params = { + constraints: { phids: ['phid-1', 'phid-2'] } + } + + expect(fake_client).to receive(:get).with('user.search', + params: expected_params) + + user_client.users(['phid-1', 'phid-2']) + end + + it 'returns an array of parsed responses' do + response = Gitlab::PhabricatorImport::Conduit::Response + .new(fixture_file('phabricator_responses/user.search.json')) + + allow(fake_client).to receive(:get).and_return(response) + + expect(user_client.users(%w[some phids])).to match_array([an_instance_of(Gitlab::PhabricatorImport::Conduit::UsersResponse)]) + end + + it 'performs multiple requests if more phids than the maximum page size are passed' do + stub_const('Gitlab::PhabricatorImport::Conduit::User::MAX_PAGE_SIZE', 1) + first_params = { constraints: { phids: ['phid-1'] } } + second_params = { constraints: { phids: ['phid-2'] } } + + expect(fake_client).to receive(:get).with('user.search', + params: first_params).once + expect(fake_client).to receive(:get).with('user.search', + params: second_params).once + + user_client.users(['phid-1', 'phid-2']) + end + end +end diff --git a/spec/lib/gitlab/phabricator_import/conduit/users_response_spec.rb b/spec/lib/gitlab/phabricator_import/conduit/users_response_spec.rb new file mode 100644 index 00000000000..00778ad90fd --- /dev/null +++ b/spec/lib/gitlab/phabricator_import/conduit/users_response_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Gitlab::PhabricatorImport::Conduit::UsersResponse do + let(:conduit_response) do + Gitlab::PhabricatorImport::Conduit::Response + .new(JSON.parse(fixture_file('phabricator_responses/user.search.json'))) + end + + subject(:response) { described_class.new(conduit_response) } + + describe '#users' do + it 'builds the correct users representation' do + tasks = response.users + + usernames = tasks.map(&:username) + + expect(usernames).to contain_exactly('jane', 'john') + end + end +end diff --git a/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb b/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb index 2412cf76f79..667321409da 100644 --- a/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb +++ b/spec/lib/gitlab/phabricator_import/issues/importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe Gitlab::PhabricatorImport::Issues::Importer do - set(:project) { create(:project) } + let(:project) { create(:project) } let(:response) do Gitlab::PhabricatorImport::Conduit::TasksResponse.new( @@ -15,7 +15,6 @@ describe Gitlab::PhabricatorImport::Issues::Importer do before do client = instance_double(Gitlab::PhabricatorImport::Conduit::Maniphest) - allow(client).to receive(:tasks).and_return(response) allow(importer).to receive(:client).and_return(client) end @@ -34,20 +33,29 @@ describe Gitlab::PhabricatorImport::Issues::Importer do importer.execute end - it 'schedules the next batch if there is one' do - expect(Gitlab::PhabricatorImport::ImportTasksWorker) - .to receive(:schedule).with(project.id, response.pagination.next_page) + context 'stubbed task import' do + before do + # Stub out the actual importing so we don't perform aditional requests + expect_next_instance_of(Gitlab::PhabricatorImport::Issues::TaskImporter) do |task_importer| + allow(task_importer).to receive(:execute) + end.at_least(1) + end - importer.execute - end + it 'schedules the next batch if there is one' do + expect(Gitlab::PhabricatorImport::ImportTasksWorker) + .to receive(:schedule).with(project.id, response.pagination.next_page) - it 'does not reschedule when there is no next page' do - allow(response.pagination).to receive(:has_next_page?).and_return(false) + importer.execute + end - expect(Gitlab::PhabricatorImport::ImportTasksWorker) - .not_to receive(:schedule) + it 'does not reschedule when there is no next page' do + allow(response.pagination).to receive(:has_next_page?).and_return(false) - importer.execute + expect(Gitlab::PhabricatorImport::ImportTasksWorker) + .not_to receive(:schedule) + + importer.execute + end end end end diff --git a/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb b/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb index 1625604e754..06ed264e781 100644 --- a/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb +++ b/spec/lib/gitlab/phabricator_import/issues/task_importer_spec.rb @@ -12,6 +12,8 @@ describe Gitlab::PhabricatorImport::Issues::TaskImporter do 'description' => { 'raw' => '# This is markdown\n it can contain more text.' }, + 'authorPHID' => 'PHID-USER-456', + 'ownerPHID' => 'PHID-USER-123', 'dateCreated' => '1518688921', 'dateClosed' => '1518789995' } @@ -19,9 +21,18 @@ describe Gitlab::PhabricatorImport::Issues::TaskImporter do ) end + subject(:importer) { described_class.new(project, task) } + describe '#execute' do + let(:fake_user_finder) { instance_double(Gitlab::PhabricatorImport::UserFinder) } + + before do + allow(fake_user_finder).to receive(:find) + allow(importer).to receive(:user_finder).and_return(fake_user_finder) + end + it 'creates the issue with the expected attributes' do - issue = described_class.new(project, task).execute + issue = importer.execute expect(issue.project).to eq(project) expect(issue).to be_persisted @@ -34,21 +45,38 @@ describe Gitlab::PhabricatorImport::Issues::TaskImporter do end it 'does not recreate the issue when called multiple times' do - expect { described_class.new(project, task).execute } + expect { importer.execute } .to change { project.issues.reload.size }.from(0).to(1) - expect { described_class.new(project, task).execute } + expect { importer.execute } .not_to change { project.issues.reload.size } end it 'does not trigger a save when the object did not change' do existing_issue = create(:issue, task.issue_attributes.merge(author: User.ghost)) - importer = described_class.new(project, task) allow(importer).to receive(:issue).and_return(existing_issue) expect(existing_issue).not_to receive(:save!) importer.execute end + + it 'links the author if the author can be found' do + author = create(:user) + expect(fake_user_finder).to receive(:find).with('PHID-USER-456').and_return(author) + + issue = importer.execute + + expect(issue.author).to eq(author) + end + + it 'links an assignee if the user can be found' do + assignee = create(:user) + expect(fake_user_finder).to receive(:find).with('PHID-USER-123').and_return(assignee) + + issue = importer.execute + + expect(issue.assignees).to include(assignee) + end end end diff --git a/spec/lib/gitlab/phabricator_import/representation/task_spec.rb b/spec/lib/gitlab/phabricator_import/representation/task_spec.rb index dfbd8c546eb..5603a6961d6 100644 --- a/spec/lib/gitlab/phabricator_import/representation/task_spec.rb +++ b/spec/lib/gitlab/phabricator_import/representation/task_spec.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require 'spec_helper' describe Gitlab::PhabricatorImport::Representation::Task do @@ -7,6 +9,8 @@ describe Gitlab::PhabricatorImport::Representation::Task do 'phid' => 'the-phid', 'fields' => { 'name' => 'Title'.ljust(257, '.'), # A string padded to 257 chars + 'authorPHID' => 'a phid', + 'ownerPHID' => 'another user phid', 'description' => { 'raw' => '# This is markdown\n it can contain more text.' }, @@ -30,4 +34,16 @@ describe Gitlab::PhabricatorImport::Representation::Task do expect(task.issue_attributes).to eq(expected_attributes) end end + + describe '#author_phid' do + it 'returns the correct field' do + expect(task.author_phid).to eq('a phid') + end + end + + describe '#owner_phid' do + it 'returns the correct field' do + expect(task.owner_phid).to eq('another user phid') + end + end end diff --git a/spec/lib/gitlab/phabricator_import/representation/user_spec.rb b/spec/lib/gitlab/phabricator_import/representation/user_spec.rb new file mode 100644 index 00000000000..f52467a0cf1 --- /dev/null +++ b/spec/lib/gitlab/phabricator_import/representation/user_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::PhabricatorImport::Representation::User do + subject(:user) do + described_class.new( + { + 'phid' => 'the-phid', + 'fields' => { + 'username' => 'the-username' + } + } + ) + end + + describe '#phabricator_id' do + it 'returns the phabricator id' do + expect(user.phabricator_id).to eq('the-phid') + end + end + + describe '#username' do + it 'returns the username' do + expect(user.username).to eq('the-username') + end + end +end diff --git a/spec/lib/gitlab/phabricator_import/user_finder_spec.rb b/spec/lib/gitlab/phabricator_import/user_finder_spec.rb new file mode 100644 index 00000000000..096321cda5f --- /dev/null +++ b/spec/lib/gitlab/phabricator_import/user_finder_spec.rb @@ -0,0 +1,89 @@ +require 'spec_helper' + +describe Gitlab::PhabricatorImport::UserFinder, :clean_gitlab_redis_cache do + let(:project) { create(:project, namespace: create(:group)) } + subject(:finder) { described_class.new(project, ['first-phid', 'second-phid']) } + + before do + project.namespace.add_developer(existing_user) + end + + describe '#find' do + let!(:existing_user) { create(:user, username: 'existing-user') } + let(:cache) { Gitlab::PhabricatorImport::Cache::Map.new(project) } + + before do + allow(finder).to receive(:object_map).and_return(cache) + end + + context 'for a cached phid' do + before do + cache.set_gitlab_model(existing_user, 'first-phid') + end + + it 'returns the existing user' do + expect(finder.find('first-phid')).to eq(existing_user) + end + + it 'does not perform a find using the API' do + expect(finder).not_to receive(:find_user_for_phid) + + finder.find('first-phid') + end + + it 'excludes the phid from the request if one needs to be made' do + client = instance_double(Gitlab::PhabricatorImport::Conduit::User) + allow(finder).to receive(:client).and_return(client) + + expect(client).to receive(:users).with(['second-phid']).and_return([]) + + finder.find('first-phid') + finder.find('second-phid') + end + end + + context 'when the phid is not cached' do + let(:response) do + [ + instance_double( + Gitlab::PhabricatorImport::Conduit::UsersResponse, + users: [instance_double(Gitlab::PhabricatorImport::Representation::User, phabricator_id: 'second-phid', username: 'existing-user')] + ), + instance_double( + Gitlab::PhabricatorImport::Conduit::UsersResponse, + users: [instance_double(Gitlab::PhabricatorImport::Representation::User, phabricator_id: 'first-phid', username: 'other-user')] + ) + ] + end + let(:client) do + client = instance_double(Gitlab::PhabricatorImport::Conduit::User) + allow(client).to receive(:users).and_return(response) + + client + end + + before do + allow(finder).to receive(:client).and_return(client) + end + + it 'loads the users from the API once' do + expect(client).to receive(:users).and_return(response).once + + expect(finder.find('second-phid')).to eq(existing_user) + expect(finder.find('first-phid')).to be_nil + end + + it 'adds found users to the cache' do + expect { finder.find('second-phid') } + .to change { cache.get_gitlab_model('second-phid') } + .from(nil).to(existing_user) + end + + it 'only returns users that are members of the project' do + create(:user, username: 'other-user') + + expect(finder.find('first-phid')).to eq(nil) + end + end + end +end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index d98db024f73..78862de0657 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -2013,6 +2013,7 @@ describe Ci::Build do { key: 'CI', value: 'true', public: true, masked: false }, { key: 'GITLAB_CI', value: 'true', public: true, masked: false }, { key: 'GITLAB_FEATURES', value: project.licensed_features.join(','), public: true, masked: false }, + { key: 'CI_SERVER_HOST', value: Gitlab.config.gitlab.host, public: true, masked: false }, { key: 'CI_SERVER_NAME', value: 'GitLab', public: true, masked: false }, { key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true, masked: false }, { key: 'CI_SERVER_VERSION_MAJOR', value: Gitlab.version_info.major.to_s, public: true, masked: false }, diff --git a/spec/models/commit_range_spec.rb b/spec/models/commit_range_spec.rb index b96ca89c893..4a524b585e1 100644 --- a/spec/models/commit_range_spec.rb +++ b/spec/models/commit_range_spec.rb @@ -139,8 +139,8 @@ describe CommitRange do end describe '#has_been_reverted?' do - let(:issue) { create(:issue) } - let(:user) { issue.author } + let(:user) { create(:user) } + let(:issue) { create(:issue, author: user, project: project) } it 'returns true if the commit has been reverted' do create(:note_on_issue, @@ -149,9 +149,11 @@ describe CommitRange do note: commit1.revert_description(user), project: issue.project) - expect_any_instance_of(Commit).to receive(:reverts_commit?) - .with(commit1, user) - .and_return(true) + expect_next_instance_of(Commit) do |commit| + expect(commit).to receive(:reverts_commit?) + .with(commit1, user) + .and_return(true) + end expect(commit1.has_been_reverted?(user, issue.notes_with_associations)).to eq(true) end diff --git a/spec/models/concerns/cache_markdown_field_spec.rb b/spec/models/concerns/cache_markdown_field_spec.rb index 0e5fb2b5153..9a12c3d6965 100644 --- a/spec/models/concerns/cache_markdown_field_spec.rb +++ b/spec/models/concerns/cache_markdown_field_spec.rb @@ -198,6 +198,36 @@ describe CacheMarkdownField, :clean_gitlab_redis_cache do end end end + + describe '#updated_cached_html_for' do + let(:thing) { klass.new(description: markdown, description_html: html, cached_markdown_version: cache_version) } + + context 'when the markdown cache is outdated' do + before do + thing.cached_markdown_version += 1 + end + + it 'calls #refresh_markdown_cache' do + expect(thing).to receive(:refresh_markdown_cache) + + expect(thing.updated_cached_html_for(:description)).to eq(html) + end + end + + context 'when the markdown field does not exist' do + it 'returns nil' do + expect(thing.updated_cached_html_for(:something)).to eq(nil) + end + end + + context 'when the markdown cache is up to date' do + it 'does not call #refresh_markdown_cache' do + expect(thing).not_to receive(:refresh_markdown_cache) + + expect(thing.updated_cached_html_for(:description)).to eq(html) + end + end + end end context 'for Active record classes' do diff --git a/spec/models/note_spec.rb b/spec/models/note_spec.rb index 7a1ab20186a..03003e3dd7d 100644 --- a/spec/models/note_spec.rb +++ b/spec/models/note_spec.rb @@ -177,6 +177,7 @@ describe Note do pipeline: :note, cache_key: [note1, "note"], project: note1.project, + rendered: note1.note_html, author: note1.author } }]).and_call_original @@ -189,6 +190,7 @@ describe Note do pipeline: :note, cache_key: [note2, "note"], project: note2.project, + rendered: note2.note_html, author: note2.author } }]).and_call_original diff --git a/spec/models/project_services/bugzilla_service_spec.rb b/spec/models/project_services/bugzilla_service_spec.rb index d5b0f94f461..74c85a13c88 100644 --- a/spec/models/project_services/bugzilla_service_spec.rb +++ b/spec/models/project_services/bugzilla_service_spec.rb @@ -44,7 +44,9 @@ describe BugzillaService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:bugzilla_service, properties: properties) } + let(:service) do + create(:bugzilla_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -60,7 +62,7 @@ describe BugzillaService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:bugzilla_service, title: title, description: description, properties: properties) + create(:bugzilla_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/custom_issue_tracker_service_spec.rb b/spec/models/project_services/custom_issue_tracker_service_spec.rb index 56b0bda6626..5259357a254 100644 --- a/spec/models/project_services/custom_issue_tracker_service_spec.rb +++ b/spec/models/project_services/custom_issue_tracker_service_spec.rb @@ -58,7 +58,9 @@ describe CustomIssueTrackerService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:custom_issue_tracker_service, properties: properties) } + let(:service) do + create(:custom_issue_tracker_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -74,7 +76,7 @@ describe CustomIssueTrackerService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:custom_issue_tracker_service, title: title, description: description, properties: properties) + create(:custom_issue_tracker_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/gitlab_issue_tracker_service_spec.rb b/spec/models/project_services/gitlab_issue_tracker_service_spec.rb index a3726f09dc5..0c4fc290a13 100644 --- a/spec/models/project_services/gitlab_issue_tracker_service_spec.rb +++ b/spec/models/project_services/gitlab_issue_tracker_service_spec.rb @@ -61,7 +61,9 @@ describe GitlabIssueTrackerService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:gitlab_issue_tracker_service, properties: properties) } + let(:service) do + create(:gitlab_issue_tracker_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -77,7 +79,7 @@ describe GitlabIssueTrackerService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:gitlab_issue_tracker_service, title: title, description: description, properties: properties) + create(:gitlab_issue_tracker_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/jira_service_spec.rb b/spec/models/project_services/jira_service_spec.rb index 9b122d85293..235cf314af5 100644 --- a/spec/models/project_services/jira_service_spec.rb +++ b/spec/models/project_services/jira_service_spec.rb @@ -145,7 +145,9 @@ describe JiraService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:jira_service, properties: properties) } + let(:service) do + create(:jira_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -161,7 +163,7 @@ describe JiraService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:jira_service, title: title, description: description, properties: properties) + create(:jira_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/redmine_service_spec.rb b/spec/models/project_services/redmine_service_spec.rb index 806e3695962..c1ee6546b12 100644 --- a/spec/models/project_services/redmine_service_spec.rb +++ b/spec/models/project_services/redmine_service_spec.rb @@ -50,7 +50,9 @@ describe RedmineService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:redmine_service, properties: properties) } + let(:service) do + create(:redmine_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -66,7 +68,7 @@ describe RedmineService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:redmine_service, title: title, description: description, properties: properties) + create(:redmine_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/models/project_services/youtrack_service_spec.rb b/spec/models/project_services/youtrack_service_spec.rb index b47ef6702b4..c48bf487af0 100644 --- a/spec/models/project_services/youtrack_service_spec.rb +++ b/spec/models/project_services/youtrack_service_spec.rb @@ -47,7 +47,9 @@ describe YoutrackService do # this will be removed as part of https://gitlab.com/gitlab-org/gitlab-ce/issues/63084 context 'when data are stored in properties' do let(:properties) { access_params.merge(title: title, description: description) } - let(:service) { create(:youtrack_service, properties: properties) } + let(:service) do + create(:youtrack_service, :without_properties_callback, properties: properties) + end include_examples 'issue tracker fields' end @@ -63,7 +65,7 @@ describe YoutrackService do context 'when data are stored in both properties and separated fields' do let(:properties) { access_params.merge(title: 'wrong title', description: 'wrong description') } let(:service) do - create(:youtrack_service, title: title, description: description, properties: properties) + create(:youtrack_service, :without_properties_callback, title: title, description: description, properties: properties) end include_examples 'issue tracker fields' diff --git a/spec/services/boards/issues/move_service_spec.rb b/spec/services/boards/issues/move_service_spec.rb index 16e2a2fba6b..1bfb5602df2 100644 --- a/spec/services/boards/issues/move_service_spec.rb +++ b/spec/services/boards/issues/move_service_spec.rb @@ -52,5 +52,91 @@ describe Boards::Issues::MoveService do it_behaves_like 'issues move service', true end + + describe '#execute_multiple' do + set(:group) { create(:group) } + set(:user) { create(:user) } + set(:project) { create(:project, namespace: group) } + set(:board1) { create(:board, group: group) } + set(:development) { create(:group_label, group: group, name: 'Development') } + set(:testing) { create(:group_label, group: group, name: 'Testing') } + set(:list1) { create(:list, board: board1, label: development, position: 0) } + set(:list2) { create(:list, board: board1, label: testing, position: 1) } + let(:params) { { board_id: board1.id, from_list_id: list1.id, to_list_id: list2.id } } + + before do + project.add_developer(user) + end + + it 'returns false if list of issues is empty' do + expect(described_class.new(group, user, params).execute_multiple([])).to eq(false) + end + + context 'moving multiple issues' do + let(:issue1) { create(:labeled_issue, project: project, labels: [development]) } + let(:issue2) { create(:labeled_issue, project: project, labels: [development]) } + + it 'moves multiple issues from one list to another' do + expect(described_class.new(group, user, params).execute_multiple([issue1, issue2])).to be_truthy + + expect(issue1.labels).to eq([testing]) + expect(issue2.labels).to eq([testing]) + end + end + + context 'moving a single issue' do + let(:issue1) { create(:labeled_issue, project: project, labels: [development]) } + + it 'moves one issue' do + expect(described_class.new(group, user, params).execute_multiple([issue1])).to be_truthy + + expect(issue1.labels).to eq([testing]) + end + end + + context 'moving issues visually after an existing issue' do + let(:existing_issue) { create(:labeled_issue, project: project, labels: [testing], relative_position: 10) } + let(:issue1) { create(:labeled_issue, project: project, labels: [development]) } + let(:issue2) { create(:labeled_issue, project: project, labels: [development]) } + + let(:move_params) do + params.dup.tap do |hash| + hash[:move_before_id] = existing_issue.id + end + end + + it 'moves one issue' do + expect(described_class.new(group, user, move_params).execute_multiple([issue1, issue2])).to be_truthy + + expect(issue1.labels).to eq([testing]) + expect(issue2.labels).to eq([testing]) + + expect(issue1.relative_position > existing_issue.relative_position).to eq(true) + expect(issue2.relative_position > issue1.relative_position).to eq(true) + end + end + + context 'moving issues visually before an existing issue' do + let(:existing_issue) { create(:labeled_issue, project: project, labels: [testing], relative_position: 10) } + let(:issue1) { create(:labeled_issue, project: project, labels: [development]) } + let(:issue2) { create(:labeled_issue, project: project, labels: [development]) } + + let(:move_params) do + params.dup.tap do |hash| + hash[:move_after_id] = existing_issue.id + end + end + + it 'moves one issue' do + expect(described_class.new(group, user, move_params).execute_multiple([issue1, issue2])).to be_truthy + + expect(issue1.labels).to eq([testing]) + expect(issue2.labels).to eq([testing]) + + expect(issue2.relative_position < existing_issue.relative_position).to eq(true) + expect(issue1.relative_position < issue2.relative_position).to eq(true) + end + end + end end end diff --git a/spec/services/clusters/gcp/kubernetes/create_or_update_namespace_service_spec.rb b/spec/services/clusters/gcp/kubernetes/create_or_update_namespace_service_spec.rb index be052a07da7..44407ae2793 100644 --- a/spec/services/clusters/gcp/kubernetes/create_or_update_namespace_service_spec.rb +++ b/spec/services/clusters/gcp/kubernetes/create_or_update_namespace_service_spec.rb @@ -34,6 +34,8 @@ describe Clusters::Gcp::Kubernetes::CreateOrUpdateNamespaceService, '#execute' d stub_kubeclient_create_service_account(api_url, namespace: namespace) stub_kubeclient_create_secret(api_url, namespace: namespace) stub_kubeclient_put_secret(api_url, "#{namespace}-token", namespace: namespace) + stub_kubeclient_put_role(api_url, Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, namespace: namespace) + stub_kubeclient_put_role_binding(api_url, Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME, namespace: namespace) stub_kubeclient_get_secret( api_url, diff --git a/spec/services/clusters/gcp/kubernetes/create_or_update_service_account_service_spec.rb b/spec/services/clusters/gcp/kubernetes/create_or_update_service_account_service_spec.rb index 382b9043566..8b874989758 100644 --- a/spec/services/clusters/gcp/kubernetes/create_or_update_service_account_service_spec.rb +++ b/spec/services/clusters/gcp/kubernetes/create_or_update_service_account_service_spec.rb @@ -143,6 +143,8 @@ describe Clusters::Gcp::Kubernetes::CreateOrUpdateServiceAccountService do stub_kubeclient_get_role_binding_error(api_url, role_binding_name, namespace: namespace) stub_kubeclient_create_role_binding(api_url, namespace: namespace) + stub_kubeclient_put_role(api_url, Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, namespace: namespace) + stub_kubeclient_put_role_binding(api_url, Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_BINDING_NAME, namespace: namespace) end it_behaves_like 'creates service account and token' @@ -169,6 +171,24 @@ describe Clusters::Gcp::Kubernetes::CreateOrUpdateServiceAccountService do ) ) end + + it 'creates a role and role binding granting knative serving permissions to the service account' do + subject + + expect(WebMock).to have_requested(:put, api_url + "/apis/rbac.authorization.k8s.io/v1/namespaces/#{namespace}/roles/#{Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME}").with( + body: hash_including( + metadata: { + name: Clusters::Gcp::Kubernetes::GITLAB_KNATIVE_SERVING_ROLE_NAME, + namespace: namespace + }, + rules: [{ + apiGroups: %w(serving.knative.dev), + resources: %w(configurations configurationgenerations routes revisions revisionuids autoscalers services), + verbs: %w(get list create update delete patch watch) + }] + ) + ) + end end end end diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index f25e2fe5e2b..1d7bf91fda1 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -215,13 +215,14 @@ describe NotificationService, :mailer do let(:project) { create(:project, :private) } let(:issue) { create(:issue, project: project, assignees: [assignee]) } let(:mentioned_issue) { create(:issue, assignees: issue.assignees) } - let(:note) { create(:note_on_issue, noteable: issue, project_id: issue.project_id, note: '@mention referenced, @unsubscribed_mentioned and @outsider also') } + let(:author) { create(:user) } + let(:note) { create(:note_on_issue, author: author, noteable: issue, project_id: issue.project_id, note: '@mention referenced, @unsubscribed_mentioned and @outsider also') } before do - build_team(note.project) + build_team(project) project.add_maintainer(issue.author) project.add_maintainer(assignee) - project.add_maintainer(note.author) + project.add_maintainer(author) @u_custom_off = create_user_with_notification(:custom, 'custom_off') project.add_guest(@u_custom_off) @@ -240,7 +241,8 @@ describe NotificationService, :mailer do describe '#new_note' do it do - add_users_with_subscription(note.project, issue) + add_users(project) + add_user_subscriptions(issue) reset_delivered_emails! expect(SentNotification).to receive(:record).with(issue, any_args).exactly(10).times @@ -268,7 +270,8 @@ describe NotificationService, :mailer do end it "emails the note author if they've opted into notifications about their activity" do - add_users_with_subscription(note.project, issue) + add_users(project) + add_user_subscriptions(issue) reset_delivered_emails! note.author.notified_of_own_activity = true @@ -415,13 +418,15 @@ describe NotificationService, :mailer do let(:project) { create(:project, :public) } let(:issue) { create(:issue, project: project, assignees: [assignee]) } let(:mentioned_issue) { create(:issue, assignees: issue.assignees) } - let(:note) { create(:note_on_issue, noteable: issue, project_id: issue.project_id, note: '@all mentioned') } + let(:author) { create(:user) } + let(:note) { create(:note_on_issue, author: author, noteable: issue, project_id: issue.project_id, note: '@all mentioned') } before do - build_team(note.project) - build_group(note.project) - note.project.add_maintainer(note.author) - add_users_with_subscription(note.project, issue) + build_team(project) + build_group(project) + add_users(project) + add_user_subscriptions(issue) + project.add_maintainer(author) reset_delivered_emails! end @@ -473,17 +478,18 @@ describe NotificationService, :mailer do context 'project snippet note' do let!(:project) { create(:project, :public) } let(:snippet) { create(:project_snippet, project: project, author: create(:user)) } - let(:note) { create(:note_on_project_snippet, noteable: snippet, project_id: project.id, note: '@all mentioned') } + let(:author) { create(:user) } + let(:note) { create(:note_on_project_snippet, author: author, noteable: snippet, project_id: project.id, note: '@all mentioned') } before do build_team(project) build_group(project) + project.add_maintainer(author) # make sure these users can read the project snippet! project.add_guest(@u_guest_watcher) project.add_guest(@u_guest_custom) add_member_for_parent_group(@pg_watcher, project) - note.project.add_maintainer(note.author) reset_delivered_emails! end @@ -708,10 +714,11 @@ describe NotificationService, :mailer do let(:issue) { create :issue, project: project, assignees: [assignee], description: 'cc @participant @unsubscribed_mentioned' } before do - build_team(issue.project) - build_group(issue.project) + build_team(project) + build_group(project) - add_users_with_subscription(issue.project, issue) + add_users(project) + add_user_subscriptions(issue) reset_delivered_emails! update_custom_notification(:new_issue, @u_guest_custom, resource: project) update_custom_notification(:new_issue, @u_custom_global) @@ -1281,13 +1288,16 @@ describe NotificationService, :mailer do let(:project) { create(:project, :public, :repository, namespace: group) } let(:another_project) { create(:project, :public, namespace: group) } let(:assignee) { create(:user) } - let(:merge_request) { create :merge_request, source_project: project, assignees: [assignee], description: 'cc @participant' } + let(:assignees) { Array.wrap(assignee) } + let(:author) { create(:user) } + let(:merge_request) { create :merge_request, author: author, source_project: project, assignees: assignees, description: 'cc @participant' } before do - project.add_maintainer(merge_request.author) - merge_request.assignees.each { |assignee| project.add_maintainer(assignee) } - build_team(merge_request.target_project) - add_users_with_subscription(merge_request.target_project, merge_request) + project.add_maintainer(author) + assignees.each { |assignee| project.add_maintainer(assignee) } + build_team(project) + add_users(project) + add_user_subscriptions(merge_request) update_custom_notification(:new_merge_request, @u_guest_custom, resource: project) update_custom_notification(:new_merge_request, @u_custom_global) reset_delivered_emails! @@ -2417,7 +2427,7 @@ describe NotificationService, :mailer do should_not_email(user, recipients: email_recipients) end - def add_users_with_subscription(project, issuable) + def add_users(project) @subscriber = create :user @unsubscriber = create :user @unsubscribed_mentioned = create :user, username: 'unsubscribed_mentioned' @@ -2429,7 +2439,9 @@ describe NotificationService, :mailer do project.add_maintainer(@unsubscriber) project.add_maintainer(@watcher_and_subscriber) project.add_maintainer(@unsubscribed_mentioned) + end + def add_user_subscriptions(issuable) issuable.subscriptions.create(user: @unsubscribed_mentioned, project: project, subscribed: false) issuable.subscriptions.create(user: @subscriber, project: project, subscribed: true) issuable.subscriptions.create(user: @subscribed_participant, project: project, subscribed: true) diff --git a/spec/support/features/rss_shared_examples.rb b/spec/support/features/rss_shared_examples.rb index 0de92aedba5..02d310a9afa 100644 --- a/spec/support/features/rss_shared_examples.rb +++ b/spec/support/features/rss_shared_examples.rb @@ -6,7 +6,7 @@ end shared_examples "it has an RSS button with current_user's feed token" do it "shows the RSS button with current_user's feed token" do - expect(page).to have_css("a:has(.fa-rss)[href*='feed_token=#{user.feed_token}']") + expect(page).to have_css("a:has(.fa-rss)[href*='feed_token=#{user.feed_token}'], .js-rss-button[href*='feed_token=#{user.feed_token}']") end end @@ -18,6 +18,6 @@ end shared_examples "it has an RSS button without a feed token" do it "shows the RSS button without a feed token" do - expect(page).to have_css("a:has(.fa-rss):not([href*='feed_token'])") + expect(page).to have_css("a:has(.fa-rss):not([href*='feed_token']), .js-rss-button:not([href*='feed_token'])") end end diff --git a/spec/support/helpers/kubernetes_helpers.rb b/spec/support/helpers/kubernetes_helpers.rb index 3c7bcba2b42..278264f3df5 100644 --- a/spec/support/helpers/kubernetes_helpers.rb +++ b/spec/support/helpers/kubernetes_helpers.rb @@ -199,6 +199,11 @@ module KubernetesHelpers .to_return(kube_response({})) end + def stub_kubeclient_put_role(api_url, name, namespace: 'default') + WebMock.stub_request(:put, api_url + "/apis/rbac.authorization.k8s.io/v1/namespaces/#{namespace}/roles/#{name}") + .to_return(kube_response({})) + end + def kube_v1_secret_body(**options) { "kind" => "SecretList", diff --git a/spec/support/matchers/abort_matcher.rb b/spec/support/matchers/abort_matcher.rb new file mode 100644 index 00000000000..ce1dd140210 --- /dev/null +++ b/spec/support/matchers/abort_matcher.rb @@ -0,0 +1,46 @@ +RSpec::Matchers.define :abort_execution do + match do |code_block| + @captured_stderr = StringIO.new + original_stderr = $stderr + $stderr = @captured_stderr + + code_block.call + + false + rescue SystemExit => e + captured = @captured_stderr.string.chomp + @actual_exit_code = e.status + break false unless e.status == 1 + + if @message + if @message.is_a? String + @message == captured + elsif @message.is_a? Regexp + @message.match?(captured) + else + raise ArgumentError, 'with_message must be either a String or a Regular Expression' + end + end + + ensure + $stderr = original_stderr + end + + chain :with_message do |message| + @message = message + end + + failure_message do |block| + unless @actual_exit_code + break "expected #{block} to abort with '#{@message}' but didnt call abort." + end + + if @actual_exit_code != 1 + break "expected #{block} to abort with: '#{@message}' but exited with success instead." + end + + "expected #{block} to abort with: '#{@message}' \n but received: '#{@captured_stderr.string.chomp}' instead." + end + + supports_block_expectations +end diff --git a/spec/support/shared_examples/mentionable_shared_examples.rb b/spec/support/shared_examples/mentionable_shared_examples.rb index 1226841f24c..fea52c2eeb2 100644 --- a/spec/support/shared_examples/mentionable_shared_examples.rb +++ b/spec/support/shared_examples/mentionable_shared_examples.rb @@ -76,6 +76,30 @@ shared_examples 'a mentionable' do expect(refs).to include(ext_commit) end + context 'when there are cached markdown fields' do + before do + if subject.is_a?(CacheMarkdownField) + subject.refresh_markdown_cache + end + end + + it 'sends in cached markdown fields when appropriate' do + if subject.is_a?(CacheMarkdownField) + expect_next_instance_of(Gitlab::ReferenceExtractor) do |ext| + attrs = subject.class.mentionable_attrs.collect(&:first) & subject.cached_markdown_fields.markdown_fields + attrs.each do |field| + expect(ext).to receive(:analyze).with(subject.send(field), hash_including(rendered: anything)) + end + end + + expect(subject).not_to receive(:refresh_markdown_cache) + expect(subject).to receive(:cached_markdown_fields).at_least(:once).and_call_original + + subject.all_references(author) + end + end + end + it 'creates cross-reference notes' do mentioned_objects = [mentioned_issue, mentioned_mr, mentioned_commit, ext_issue, ext_mr, ext_commit] @@ -98,6 +122,33 @@ shared_examples 'an editable mentionable' do [create(:issue, project: project), create(:issue, project: ext_proj)] end + context 'when there are cached markdown fields' do + before do + if subject.is_a?(CacheMarkdownField) + subject.refresh_markdown_cache + end + end + + it 'refreshes markdown cache if necessary' do + subject.save! + + set_mentionable_text.call('This is a text') + + if subject.is_a?(CacheMarkdownField) + expect_next_instance_of(Gitlab::ReferenceExtractor) do |ext| + subject.cached_markdown_fields.markdown_fields.each do |field| + expect(ext).to receive(:analyze).with(subject.send(field), hash_including(rendered: anything)) + end + end + + expect(subject).to receive(:refresh_markdown_cache) + expect(subject).to receive(:cached_markdown_fields).at_least(:once).and_call_original + + subject.all_references(author) + end + end + end + it 'creates new cross-reference notes when the mentionable text is edited' do subject.save subject.create_cross_references! diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb index 4b04d9cec39..0e47408fc72 100644 --- a/spec/tasks/gitlab/storage_rake_spec.rb +++ b/spec/tasks/gitlab/storage_rake_spec.rb @@ -50,7 +50,7 @@ describe 'rake gitlab:storage:*', :sidekiq do expect(Project).not_to receive(:with_unmigrated_storage) - expect { run_rake_task(task) }.to output(/This task requires database write access. Exiting./).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/This task requires database write access. Exiting./) end end end @@ -96,7 +96,7 @@ describe 'rake gitlab:storage:*', :sidekiq do expect(Project).not_to receive(:with_unmigrated_storage) - expect { run_rake_task(task) }.to output(/There is already a rollback operation in progress/).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/There is already a rollback operation in progress/) end end end @@ -105,14 +105,23 @@ describe 'rake gitlab:storage:*', :sidekiq do it 'does nothing' do expect(::HashedStorage::MigratorWorker).not_to receive(:perform_async) - run_rake_task(task) + expect { run_rake_task(task) }.to abort_execution.with_message('There are no projects requiring storage migration. Nothing to do!') end end context 'with 3 legacy projects' do let(:projects) { create_list(:project, 3, :legacy_storage) } - it_behaves_like "handles custom BATCH env var", ::HashedStorage::MigratorWorker + it 'enqueues migrations and count projects correctly' do + projects.map(&:id).sort.tap do |ids| + stub_env('ID_FROM', ids[0]) + stub_env('ID_TO', ids[1]) + end + + expect { run_rake_task(task) }.to output(/Enqueuing migration of 2 projects in batches/).to_stdout + end + + it_behaves_like 'handles custom BATCH env var', ::HashedStorage::MigratorWorker end context 'with same id in range' do @@ -120,7 +129,7 @@ describe 'rake gitlab:storage:*', :sidekiq do stub_env('ID_FROM', 99999) stub_env('ID_TO', 99999) - expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=99999/).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/There are no projects requiring storage migration with ID=99999/) end it 'displays a message when project exists but its already migrated' do @@ -128,7 +137,7 @@ describe 'rake gitlab:storage:*', :sidekiq do stub_env('ID_FROM', project.id) stub_env('ID_TO', project.id) - expect { run_rake_task(task) }.to output(/There are no projects requiring storage migration with ID=#{project.id}/).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/There are no projects requiring storage migration with ID=#{project.id}/) end it 'enqueues migration when project can be found' do @@ -153,7 +162,7 @@ describe 'rake gitlab:storage:*', :sidekiq do expect(Project).not_to receive(:with_unmigrated_storage) - expect { run_rake_task(task) }.to output(/There is already a migration operation in progress/).to_stderr + expect { run_rake_task(task) }.to abort_execution.with_message(/There is already a migration operation in progress/) end end end @@ -162,13 +171,22 @@ describe 'rake gitlab:storage:*', :sidekiq do it 'does nothing' do expect(::HashedStorage::RollbackerWorker).not_to receive(:perform_async) - run_rake_task(task) + expect { run_rake_task(task) }.to abort_execution.with_message('There are no projects that can have storage rolledback. Nothing to do!') end end context 'with 3 hashed projects' do let(:projects) { create_list(:project, 3) } + it 'enqueues migrations and count projects correctly' do + projects.map(&:id).sort.tap do |ids| + stub_env('ID_FROM', ids[0]) + stub_env('ID_TO', ids[1]) + end + + expect { run_rake_task(task) }.to output(/Enqueuing rollback of 2 projects in batches/).to_stdout + end + it_behaves_like "handles custom BATCH env var", ::HashedStorage::RollbackerWorker end end diff --git a/yarn.lock b/yarn.lock index dc5e0662396..e8c11b1a033 100644 --- a/yarn.lock +++ b/yarn.lock @@ -705,10 +705,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.67.0.tgz#c7b94eca13b99fd3aaa737fb6dcc0abc41d3c579" integrity sha512-hJOmWEs6RkjzyKkb1vc9wwKGZIBIP0coHkxu/KgOoxhBVudpGk4CH7xJ6UuB2TKpb0SEh5CC1CzRZfBYaFhsaA== -"@gitlab/ui@^5.5.0": - version "5.5.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.5.0.tgz#2000b2ed0c3825dd8c4430191023f4d03c923ecb" - integrity sha512-6e/AFFLDk/gm4wKnHM9rcpTRqCsWkPKW/Vjsnd6h4wyfif2/TusHeIn/jedQxUaORbO/XZKzg4V5COhXXbCx4w== +"@gitlab/ui@^5.6.0": + version "5.6.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.6.0.tgz#6b5408050229e2135359f3fce5d0de7718326a0d" + integrity sha512-EohpACc5OCK8pOWgwB7/QZUcg3tA9k863ku6Ik9NxaRSKt/JIpQ8RI4wCr4UmqhejZLQMD9VZHLUmc9Sf3Mk9w== dependencies: "@babel/standalone" "^7.0.0" "@gitlab/vue-toasted" "^1.2.1" |