diff options
311 files changed, 3757 insertions, 1317 deletions
diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index b865b212ac0..13c8b4a8458 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -9,8 +9,12 @@ app/assets/ @ClemMakesApps @fatihacet @filipa @iamphill @mikegreiling @timzallmann @kushalpandya @pslaughter *.scss @annabeldunstone @ClemMakesApps @fatihacet @filipa @iamphill @mikegreiling @timzallmann @kushalpandya @pslaughter -# Someone from the database team should review changes in `db/` +# Maintainers from the Database team should review changes in `db/` db/ @abrandl @NikolayS +lib/gitlab/background_migration/ @abrandl @NikolayS +lib/gitlab/database/ @abrandl @NikolayS +lib/gitlab/sql/ @abrandl @NikolayS +/ee/db/ @abrandl @NikolayS # Feature specific owners /ee/lib/gitlab/code_owners/ @reprazent diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml index 45a6a177943..b00d2f46088 100644 --- a/.gitlab/ci/frontend.gitlab-ci.yml +++ b/.gitlab/ci/frontend.gitlab-ci.yml @@ -113,37 +113,6 @@ compile-assets pull-cache: - master@gitlab-org/gitlab-ee - /(^docs[\/-].*|.*-docs$)/ -gitlab:ui:visual: - extends: .dedicated-runner - before_script: [] - allow_failure: true - dependencies: - - compile-assets - - compile-assets pull-cache - script: - # Remove node modules from GitLab that may conflict with gitlab-ui - - rm -r node_modules - - git clone https://gitlab.com/gitlab-org/gitlab-ui.git - - cp public/assets/application-*.css gitlab-ui/styles/application.css - - cd gitlab-ui - - yarn install - - CSS_URL=./application.css yarn test - only: - changes: - - app/assets/stylesheets/*.scss - - app/assets/stylesheets/**/*.scss - - app/assets/stylesheets/**/**/*.scss - except: - refs: - - /(^docs[\/-].*|.*-docs$)/ - - master - variables: - - $CI_COMMIT_MESSAGE =~ /\[skip visual\]/i - artifacts: - paths: - - gitlab-ui/tests/__image_snapshots__/ - when: always - karma: extends: .dedicated-no-docs-pull-cache-job <<: *use-pg @@ -168,6 +137,8 @@ karma: paths: - chrome_debug.log - coverage-javascript/ + - tmp/tests/frontend/ +# see https://gitlab.com/gitlab-org/gitlab-ce/issues/64756 # reports: # junit: junit_karma.xml @@ -181,7 +152,7 @@ jest: script: - scripts/gitaly-test-spawn - date - - bundle exec rake karma:fixtures + - bundle exec rake frontend:fixtures - date - yarn jest --ci --coverage artifacts: @@ -191,6 +162,8 @@ jest: paths: - coverage-frontend/ - junit_jest.xml + - tmp/tests/frontend/ +# see https://gitlab.com/gitlab-org/gitlab-ce/issues/64756 # reports: # junit: junit_jest.xml cache: diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index c7a51beeee5..d0b1f1ab98f 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -80,6 +80,7 @@ - rspec_profiling/ - tmp/capybara/ - tmp/memory_test/ +# see https://gitlab.com/gitlab-org/gitlab-ce/issues/64756 # reports: # junit: junit_rspec.xml diff --git a/.gitlab/merge_request_templates/Database changes.md b/.gitlab/merge_request_templates/Database changes.md index 3f457174492..2077997a0cb 100644 --- a/.gitlab/merge_request_templates/Database changes.md +++ b/.gitlab/merge_request_templates/Database changes.md @@ -39,20 +39,11 @@ When adding tables: - [ ] Ordered columns based on the [Ordering Table Columns](https://docs.gitlab.com/ee/development/ordering_table_columns.html#ordering-table-columns) guidelines - [ ] Added foreign keys to any columns pointing to data in other tables -- [ ] Added indexes for fields that are used in statements such as WHERE, ORDER BY, GROUP BY, and JOINs +- [ ] Added indexes for fields that are used in statements such as `WHERE`, `ORDER BY`, `GROUP BY`, and `JOIN`s When removing columns, tables, indexes or other structures: - [ ] Removed these in a post-deployment migration - [ ] Made sure the application no longer uses (or ignores) these structures -## General checklist - -- [ ] [Changelog entry](https://docs.gitlab.com/ee/development/changelog.html) added, if necessary -- [ ] [Documentation created/updated](https://docs.gitlab.com/ee/development/documentation/) -- [ ] [Tests added for this feature/bug](https://docs.gitlab.com/ee/development/testing_guide/index.html) -- [ ] Conforms to the [code review guidelines](https://docs.gitlab.com/ee/development/code_review.html) -- [ ] Conforms to the [merge request performance guidelines](https://docs.gitlab.com/ee/development/merge_request_performance_guidelines.html) -- [ ] Conforms to the [style guides](https://docs.gitlab.com/ee/development/contributing/style_guides.html) - -/label ~database +/label ~database ~"database::review pending" diff --git a/.haml-lint.yml b/.haml-lint.yml index 0412b24a48c..399fa9656a0 100644 --- a/.haml-lint.yml +++ b/.haml-lint.yml @@ -118,7 +118,6 @@ linters: - Lint/ParenthesesAsGroupedExpression - Lint/RedundantWithIndex - Lint/SafeNavigationConsistency - - Lint/Syntax - Metrics/BlockNesting - Naming/VariableName - Performance/RedundantMatch @@ -155,6 +154,9 @@ linters: enabled: true style: space + Syntax: + enabled: true + Indentation: enabled: true character: space # or tab diff --git a/.rubocop.yml b/.rubocop.yml index e5fe527e611..79e06439ac2 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -60,8 +60,8 @@ Style/FrozenStringLiteralComment: RSpec/FilePath: Exclude: - 'qa/**/*' - - 'spec/javascripts/fixtures/*' - - 'ee/spec/javascripts/fixtures/*' + - 'spec/frontend/fixtures/*' + - 'ee/spec/frontend/fixtures/*' - 'spec/requests/api/v3/*' Naming/FileName: @@ -299,7 +299,6 @@ gem 'peek-gc', '~> 0.0.2' gem 'peek-mysql2', '~> 1.2.0', group: :mysql gem 'peek-pg', '~> 1.3.0', group: :postgres gem 'peek-rblineprof', '~> 0.2.0' -gem 'peek-redis', '~> 1.2.0' # Memory benchmarks gem 'derailed_benchmarks', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 60939ae918c..2bcc3527de4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -76,7 +76,6 @@ GEM asciidoctor-plantuml (0.0.9) asciidoctor (>= 1.5.6, < 3.0.0) ast (2.4.0) - atomic (1.1.99) attr_encrypted (3.1.0) encryptor (~> 3.0.0) attr_required (1.0.1) @@ -658,10 +657,6 @@ GEM peek-rblineprof (0.2.0) peek rblineprof - peek-redis (1.2.0) - atomic (>= 1.0.0) - peek - redis pg (1.1.4) po_to_json (1.0.1) json (>= 1.6.0) @@ -1199,7 +1194,6 @@ DEPENDENCIES peek-mysql2 (~> 1.2.0) peek-pg (~> 1.3.0) peek-rblineprof (~> 0.2.0) - peek-redis (~> 1.2.0) pg (~> 1.1) premailer-rails (~> 1.9.7) prometheus-client-mmap (~> 0.9.8) diff --git a/app/assets/javascripts/boards/components/board_blank_state.vue b/app/assets/javascripts/boards/components/board_blank_state.vue index d8b0b60c183..9f26337d153 100644 --- a/app/assets/javascripts/boards/components/board_blank_state.vue +++ b/app/assets/javascripts/boards/components/board_blank_state.vue @@ -1,6 +1,6 @@ <script> import { __ } from '~/locale'; -/* global ListLabel */ +import ListLabel from '~/boards/models/label'; import Cookies from 'js-cookie'; import boardsStore from '../stores/boards_store'; @@ -30,13 +30,17 @@ export default { }); // Save the labels - gl.boardService + boardsStore .generateDefaultLists() .then(res => res.data) .then(data => { data.forEach(listObj => { const list = boardsStore.findList('title', listObj.title); + if (!list) { + return; + } + list.id = listObj.id; list.label.id = listObj.label.id; list.getIssues().catch(() => { @@ -69,8 +73,7 @@ export default { <span :style="{ backgroundColor: label.color }" class="label-color position-relative d-inline-block rounded" - > - </span> + ></span> {{ label.title }} </li> </ul> diff --git a/app/assets/javascripts/boards/components/project_select.vue b/app/assets/javascripts/boards/components/project_select.vue index a1cf1866faf..e8d25e84be1 100644 --- a/app/assets/javascripts/boards/components/project_select.vue +++ b/app/assets/javascripts/boards/components/project_select.vue @@ -68,13 +68,15 @@ export default { <li> <a href='#' class='dropdown-menu-link' data-project-id="${ project.id - }" data-project-name="${project.name}"> - ${_.escape(project.name)} + }" data-project-name="${project.name}" data-project-name-with-namespace="${ + project.name_with_namespace + }"> + ${_.escape(project.name_with_namespace)} </a> </li> `; }, - text: project => project.name, + text: project => project.name_with_namespace, }); }, }; diff --git a/app/assets/javascripts/boards/ee_functions.js b/app/assets/javascripts/boards/ee_functions.js new file mode 100644 index 00000000000..583270fcae5 --- /dev/null +++ b/app/assets/javascripts/boards/ee_functions.js @@ -0,0 +1,7 @@ +export const setPromotionState = () => {}; + +export const setWeigthFetchingState = () => {}; +export const setEpicFetchingState = () => {}; + +export const getMilestoneTitle = () => ({}); +export const getBoardsModalData = () => ({}); diff --git a/app/assets/javascripts/boards/index.js b/app/assets/javascripts/boards/index.js index f5a617a85ad..3bded4a3258 100644 --- a/app/assets/javascripts/boards/index.js +++ b/app/assets/javascripts/boards/index.js @@ -30,6 +30,13 @@ import { } from '~/lib/utils/common_utils'; import boardConfigToggle from 'ee_else_ce/boards/config_toggle'; import toggleFocusMode from 'ee_else_ce/boards/toggle_focus'; +import { + setPromotionState, + setWeigthFetchingState, + setEpicFetchingState, + getMilestoneTitle, + getBoardsModalData, +} from 'ee_else_ce/boards/ee_functions'; import mountMultipleBoardsSwitcher from './mount_multiple_boards_switcher'; let issueBoardsApp; @@ -129,6 +136,7 @@ export default () => { }); boardsStore.addBlankState(); + setPromotionState(boardsStore); this.loading = false; }) .catch(() => { @@ -143,6 +151,8 @@ export default () => { const { sidebarInfoEndpoint } = newIssue; if (sidebarInfoEndpoint && newIssue.subscribed === undefined) { newIssue.setFetchingState('subscriptions', true); + setWeigthFetchingState(newIssue, true); + setEpicFetchingState(newIssue, true); BoardService.getIssueInfo(sidebarInfoEndpoint) .then(res => res.data) .then(data => { @@ -157,6 +167,8 @@ export default () => { } = convertObjectPropsToCamelCase(data); newIssue.setFetchingState('subscriptions', false); + setWeigthFetchingState(newIssue, false); + setEpicFetchingState(newIssue, false); newIssue.updateData({ humanTimeSpent: humanTotalTimeSpent, timeSpent: totalTimeSpent, @@ -169,6 +181,7 @@ export default () => { }) .catch(() => { newIssue.setFetchingState('subscriptions', false); + setWeigthFetchingState(newIssue, false); Flash(__('An error occurred while fetching sidebar data')); }); } @@ -203,6 +216,7 @@ export default () => { el: document.getElementById('js-add-list'), data: { filters: boardsStore.state.filters, + ...getMilestoneTitle($boardApp), }, mounted() { initNewListDropdown(); @@ -222,6 +236,7 @@ export default () => { return { modal: ModalStore.store, store: boardsStore.state, + ...getBoardsModalData($boardApp), canAdminList: this.$options.el.hasAttribute('data-can-admin-list'), }; }, @@ -285,6 +300,6 @@ export default () => { }); } - toggleFocusMode(ModalStore, boardsStore); + toggleFocusMode(ModalStore, boardsStore, $boardApp); mountMultipleBoardsSwitcher(); }; diff --git a/app/assets/javascripts/commons/polyfills.js b/app/assets/javascripts/commons/polyfills.js index a4394ab7e92..7a6ad3dc771 100644 --- a/app/assets/javascripts/commons/polyfills.js +++ b/app/assets/javascripts/commons/polyfills.js @@ -13,6 +13,7 @@ import 'core-js/es/string/code-point-at'; import 'core-js/es/string/from-code-point'; import 'core-js/es/string/includes'; import 'core-js/es/string/starts-with'; +import 'core-js/es/string/ends-with'; import 'core-js/es/symbol'; import 'core-js/es/map'; import 'core-js/es/weak-map'; diff --git a/app/assets/javascripts/ide/services/index.js b/app/assets/javascripts/ide/services/index.js index 840761f68db..ba33b6826d6 100644 --- a/app/assets/javascripts/ide/services/index.js +++ b/app/assets/javascripts/ide/services/index.js @@ -56,13 +56,7 @@ export default { return Api.branchSingle(projectId, currentBranchId); }, commit(projectId, payload) { - // Currently the `commit` endpoint does not support `start_sha` so we - // have to make the request in the FE. This is not ideal and will be - // resolved soon. https://gitlab.com/gitlab-org/gitlab-ce/issues/59023 - const { branch, start_sha: ref } = payload; - const branchPromise = ref ? Api.createBranch(projectId, { ref, branch }) : Promise.resolve(); - - return branchPromise.then(() => Api.commitMultiple(projectId, payload)); + return Api.commitMultiple(projectId, payload); }, getFiles(projectUrl, branchId) { const url = `${projectUrl}/files/${branchId}`; diff --git a/app/assets/javascripts/ide/stores/mutations/file.js b/app/assets/javascripts/ide/stores/mutations/file.js index a52f1e235ed..1442ea7dbfa 100644 --- a/app/assets/javascripts/ide/stores/mutations/file.js +++ b/app/assets/javascripts/ide/stores/mutations/file.js @@ -43,10 +43,14 @@ export default { [stateEntry, stagedFile, openFile, changedFile].forEach(f => { if (f) { - Object.assign(f, convertObjectPropsToCamelCase(data, { dropKeys: ['raw', 'baseRaw'] }), { - raw: (stateEntry && stateEntry.raw) || null, - baseRaw: null, - }); + Object.assign( + f, + convertObjectPropsToCamelCase(data, { dropKeys: ['path', 'name', 'raw', 'baseRaw'] }), + { + raw: (stateEntry && stateEntry.raw) || null, + baseRaw: null, + }, + ); } }); }, diff --git a/app/assets/javascripts/ide/stores/utils.js b/app/assets/javascripts/ide/stores/utils.js index 01f78a29cf6..366314536c6 100644 --- a/app/assets/javascripts/ide/stores/utils.js +++ b/app/assets/javascripts/ide/stores/utils.js @@ -155,11 +155,11 @@ export const createCommitPayload = ({ last_commit_id: newBranch || f.deleted || f.prevPath || f.replaces ? undefined : f.lastCommitSha, })), - start_sha: newBranch ? rootGetters.lastCommit.short_id : undefined, + start_sha: newBranch ? rootGetters.lastCommit.id : undefined, }); export const createNewMergeRequestUrl = (projectUrl, source, target) => - `${projectUrl}/merge_requests/new?merge_request[source_branch]=${source}&merge_request[target_branch]=${target}`; + `${projectUrl}/merge_requests/new?merge_request[source_branch]=${source}&merge_request[target_branch]=${target}&nav_source=webide`; const sortTreesByTypeAndName = (a, b) => { if (a.type === 'tree' && b.type === 'blob') { diff --git a/app/assets/javascripts/notes/components/discussion_actions.vue b/app/assets/javascripts/notes/components/discussion_actions.vue index f4570c1292c..7aa8580d794 100644 --- a/app/assets/javascripts/notes/components/discussion_actions.vue +++ b/app/assets/javascripts/notes/components/discussion_actions.vue @@ -39,30 +39,27 @@ export default { </script> <template> - <div class="discussion-with-resolve-btn"> + <div class="discussion-with-resolve-btn clearfix"> <reply-placeholder :button-text="s__('MergeRequests|Reply...')" class="qa-discussion-reply" @onClick="$emit('showReplyForm')" /> - <resolve-discussion-button - v-if="discussion.resolvable" - :is-resolving="isResolving" - :button-title="resolveButtonTitle" - @onClick="$emit('resolve')" - /> - <div v-if="discussion.resolvable" class="btn-group discussion-actions ml-sm-2" role="group"> - <resolve-with-issue-button v-if="resolveWithIssuePath" :url="resolveWithIssuePath" /> - <jump-to-next-discussion-button - v-if="shouldShowJumpToNextDiscussion" - @onClick="$emit('jumpToNextDiscussion')" - /> + + <div class="btn-group discussion-actions" role="group"> + <div class="btn-group"> + <resolve-discussion-button + v-if="discussion.resolvable" + :is-resolving="isResolving" + :button-title="resolveButtonTitle" + @onClick="$emit('resolve')" + /> + </div> <resolve-with-issue-button v-if="discussion.resolvable && resolveWithIssuePath" :url="resolveWithIssuePath" /> </div> - <div v-if="discussion.resolvable && shouldShowJumpToNextDiscussion" class="btn-group discussion-actions ml-sm-2" diff --git a/app/assets/javascripts/reports/components/report_item.vue b/app/assets/javascripts/reports/components/report_item.vue index d477fafd3f5..f3f7d2648a8 100644 --- a/app/assets/javascripts/reports/components/report_item.vue +++ b/app/assets/javascripts/reports/components/report_item.vue @@ -45,7 +45,7 @@ export default { <template> <li :class="{ 'is-dismissed': issue.isDismissed }" - class="report-block-list-issue justify-content-center align-items-center" + class="report-block-list-issue align-items-center" > <issue-status-icon v-if="showReportSectionStatusIcon" diff --git a/app/assets/javascripts/reports/components/summary_row.vue b/app/assets/javascripts/reports/components/summary_row.vue index 1caf52431e2..aba798e63d0 100644 --- a/app/assets/javascripts/reports/components/summary_row.vue +++ b/app/assets/javascripts/reports/components/summary_row.vue @@ -44,9 +44,7 @@ export default { }; </script> <template> - <div - class="report-block-list-issue report-block-list-issue-parent justify-content-center align-items-center" - > + <div class="report-block-list-issue report-block-list-issue-parent align-items-center"> <div class="report-block-list-icon append-right-default"> <gl-loading-icon v-if="statusIcon === 'loading'" diff --git a/app/assets/stylesheets/pages/pipelines.scss b/app/assets/stylesheets/pages/pipelines.scss index ff4fa8aacdc..5f4db37c317 100644 --- a/app/assets/stylesheets/pages/pipelines.scss +++ b/app/assets/stylesheets/pages/pipelines.scss @@ -695,6 +695,10 @@ top: -1px; } + .spinner { + top: 2px; + } + &.play { svg { left: 2px; @@ -861,6 +865,7 @@ button.mini-pipeline-graph-dropdown-toggle { } } + .spinner, svg { width: $ci-action-dropdown-svg-size; height: $ci-action-dropdown-svg-size; diff --git a/app/controllers/admin/requests_profiles_controller.rb b/app/controllers/admin/requests_profiles_controller.rb index 89d4c4f18d9..1d14812186a 100644 --- a/app/controllers/admin/requests_profiles_controller.rb +++ b/app/controllers/admin/requests_profiles_controller.rb @@ -10,10 +10,10 @@ class Admin::RequestsProfilesController < Admin::ApplicationController clean_name = Rack::Utils.clean_path_info(params[:name]) profile = Gitlab::RequestProfiler::Profile.find(clean_name) - if profile - render html: profile.content.html_safe - else - redirect_to admin_requests_profiles_path, alert: 'Profile not found' + unless profile && profile.content_type + return redirect_to admin_requests_profiles_path, alert: 'Profile not found' end + + send_file profile.file_path, type: "#{profile.content_type}; charset=utf-8", disposition: 'inline' end end diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb index 2985da35d83..ac008165c16 100644 --- a/app/controllers/chaos_controller.rb +++ b/app/controllers/chaos_controller.rb @@ -1,57 +1,38 @@ # frozen_string_literal: true class ChaosController < ActionController::Base - before_action :validate_chaos_secret, unless: :development? - before_action :request_start_time + before_action :validate_chaos_secret, unless: :development_or_test? def leakmem - retainer = [] - # Add `n` 1mb chunks of memory to the retainer array - memory_mb.times { retainer << "x" * 1.megabyte } - - Kernel.sleep(duration_left) - - render plain: "OK" + do_chaos :leak_mem, Chaos::LeakMemWorker, memory_mb, duration_s end def cpu_spin - rand while Time.now < expected_end_time - - render plain: "OK" + do_chaos :cpu_spin, Chaos::CpuSpinWorker, duration_s end def db_spin - while Time.now < expected_end_time - ActiveRecord::Base.connection.execute("SELECT 1") - - end_interval_time = Time.now + [duration_s, interval_s].min - rand while Time.now < end_interval_time - end + do_chaos :db_spin, Chaos::DbSpinWorker, duration_s, interval_s end def sleep - Kernel.sleep(duration_left) - - render plain: "OK" + do_chaos :sleep, Chaos::SleepWorker, duration_s end def kill - Process.kill("KILL", Process.pid) + do_chaos :kill, Chaos::KillWorker end private - def request_start_time - @start_time ||= Time.now - end - - def expected_end_time - request_start_time + duration_s - end + def do_chaos(method, worker, *args) + if async + worker.perform_async(*args) + else + Gitlab::Chaos.public_send(method, *args) # rubocop: disable GitlabSecurity/PublicSend + end - def duration_left - # returns 0 if over time - [expected_end_time - Time.now, 0].max + render plain: "OK" end def validate_chaos_secret @@ -91,7 +72,12 @@ class ChaosController < ActionController::Base memory_mb.to_i end - def development? - Rails.env.development? + def async + async = params[:async] || false + Gitlab::Utils.to_boolean(async) + end + + def development_or_test? + Rails.env.development? || Rails.env.test? end end diff --git a/app/controllers/ide_controller.rb b/app/controllers/ide_controller.rb index eeeebe430a7..af1e6cc703b 100644 --- a/app/controllers/ide_controller.rb +++ b/app/controllers/ide_controller.rb @@ -4,5 +4,6 @@ class IdeController < ApplicationController layout 'fullscreen' def index + Gitlab::UsageDataCounters::WebIdeCounter.increment_views_count end end diff --git a/app/controllers/projects/merge_requests/creations_controller.rb b/app/controllers/projects/merge_requests/creations_controller.rb index 32cefe54613..6ac5bb90706 100644 --- a/app/controllers/projects/merge_requests/creations_controller.rb +++ b/app/controllers/projects/merge_requests/creations_controller.rb @@ -23,6 +23,8 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap @merge_request = ::MergeRequests::CreateService.new(project, current_user, merge_request_params).execute if @merge_request.valid? + incr_count_webide_merge_request + redirect_to(merge_request_path(@merge_request)) else @source_project = @merge_request.source_project @@ -135,4 +137,10 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap def whitelist_query_limiting Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42384') end + + def incr_count_webide_merge_request + return if params[:nav_source] != 'webide' + + Gitlab::UsageDataCounters::WebIdeCounter.increment_merge_requests_count + end end diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 952de92cae1..e60b6497cb7 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -427,4 +427,11 @@ module Issuable def wipless_title_changed(old_title) old_title != title end + + ## + # Overridden on EE module + # + def supports_milestone? + respond_to?(:milestone_id) + end end diff --git a/app/models/concerns/project_api_compatibility.rb b/app/models/concerns/project_api_compatibility.rb index cb00efb06df..631b2a11e9a 100644 --- a/app/models/concerns/project_api_compatibility.rb +++ b/app/models/concerns/project_api_compatibility.rb @@ -9,12 +9,10 @@ module ProjectAPICompatibility end def auto_devops_enabled=(value) - self.build_auto_devops if self.auto_devops&.enabled.nil? - self.auto_devops.update! enabled: value + (auto_devops || build_auto_devops).enabled = value end def auto_devops_deploy_strategy=(value) - self.build_auto_devops if self.auto_devops&.enabled.nil? - self.auto_devops.update! deploy_strategy: value + (auto_devops || build_auto_devops).deploy_strategy = value end end diff --git a/app/models/dashboard_group_milestone.rb b/app/models/dashboard_group_milestone.rb index 74aa04ab7d0..ec52f1ed370 100644 --- a/app/models/dashboard_group_milestone.rb +++ b/app/models/dashboard_group_milestone.rb @@ -15,8 +15,7 @@ class DashboardGroupMilestone < GlobalMilestone milestones = Milestone.of_groups(groups.select(:id)) .reorder_by_due_date_asc .order_by_name_asc - .active milestones = milestones.search_title(params[:search_title]) if params[:search_title].present? - milestones.map { |m| new(m) } + Milestone.filter_by_state(milestones, params[:state]).map { |m| new(m) } end end diff --git a/app/models/environment.rb b/app/models/environment.rb index b8ee54c1696..392481ea0cc 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -4,11 +4,6 @@ class Environment < ApplicationRecord include Gitlab::Utils::StrongMemoize include ReactiveCaching - # Used to generate random suffixes for the slug - LETTERS = ('a'..'z').freeze - NUMBERS = ('0'..'9').freeze - SUFFIX_CHARS = LETTERS.to_a + NUMBERS.to_a - belongs_to :project, required: true has_many :deployments, -> { success }, dependent: :destroy # rubocop:disable Cop/ActiveRecordDependent @@ -203,40 +198,6 @@ class Environment < ApplicationRecord super.presence || generate_slug end - # An environment name is not necessarily suitable for use in URLs, DNS - # or other third-party contexts, so provide a slugified version. A slug has - # the following properties: - # * contains only lowercase letters (a-z), numbers (0-9), and '-' - # * begins with a letter - # * has a maximum length of 24 bytes (OpenShift limitation) - # * cannot end with `-` - def generate_slug - # Lowercase letters and numbers only - slugified = +name.to_s.downcase.gsub(/[^a-z0-9]/, '-') - - # Must start with a letter - slugified = 'env-' + slugified unless LETTERS.cover?(slugified[0]) - - # Repeated dashes are invalid (OpenShift limitation) - slugified.gsub!(/\-+/, '-') - - # Maximum length: 24 characters (OpenShift limitation) - slugified = slugified[0..23] - - # Cannot end with a dash (Kubernetes label limitation) - slugified.chop! if slugified.end_with?('-') - - # Add a random suffix, shortening the current string if necessary, if it - # has been slugified. This ensures uniqueness. - if slugified != name - slugified = slugified[0..16] - slugified << '-' unless slugified.end_with?('-') - slugified << random_suffix - end - - self.slug = slugified - end - def external_url_for(path, commit_sha) return unless self.external_url @@ -274,11 +235,7 @@ class Environment < ApplicationRecord private - # Slugifying a name may remove the uniqueness guarantee afforded by it being - # based on name (which must be unique). To compensate, we add a random - # 6-byte suffix in those circumstances. This is not *guaranteed* uniqueness, - # but the chance of collisions is vanishingly small - def random_suffix - (0..5).map { SUFFIX_CHARS.sample }.join + def generate_slug + self.slug = Gitlab::Slug::Environment.new(name).generate end end diff --git a/app/models/project.rb b/app/models/project.rb index 2906aca75fc..8dfe2212282 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -277,7 +277,7 @@ class Project < ApplicationRecord has_many :project_deploy_tokens has_many :deploy_tokens, through: :project_deploy_tokens - has_one :auto_devops, class_name: 'ProjectAutoDevops' + has_one :auto_devops, class_name: 'ProjectAutoDevops', inverse_of: :project, autosave: true has_many :custom_attributes, class_name: 'ProjectCustomAttribute' has_many :project_badges, class_name: 'ProjectBadge' diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb index 67c12363a3c..f39f54f0434 100644 --- a/app/models/project_auto_devops.rb +++ b/app/models/project_auto_devops.rb @@ -5,7 +5,7 @@ class ProjectAutoDevops < ApplicationRecord ignore_column :domain - belongs_to :project + belongs_to :project, inverse_of: :auto_devops enum deploy_strategy: { continuous: 0, diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index e571700fd02..7ab79242cc3 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -54,7 +54,7 @@ class JiraService < IssueTrackerService username: self.username, password: self.password, site: URI.join(url, '/').to_s, # Intended to find the root - context_path: url.path.chomp('/'), + context_path: url.path, auth_type: :basic, read_timeout: 120, use_cookies: true, @@ -103,6 +103,12 @@ class JiraService < IssueTrackerService "#{url}/secure/CreateIssue.jspa" end + alias_method :original_url, :url + + def url + original_url&.chomp('/') + end + def execute(push) # This method is a no-op, because currently JiraService does not # support any events. diff --git a/app/services/commits/create_service.rb b/app/services/commits/create_service.rb index bb34a3d3352..f3be68f9602 100644 --- a/app/services/commits/create_service.rb +++ b/app/services/commits/create_service.rb @@ -10,6 +10,7 @@ module Commits @start_project = params[:start_project] || @project @start_branch = params[:start_branch] + @start_sha = params[:start_sha] @branch_name = params[:branch_name] @force = params[:force] || false end @@ -40,7 +41,7 @@ module Commits end def different_branch? - @start_branch != @branch_name || @start_project != @project + @start_project != @project || @start_branch != @branch_name || @start_sha.present? end def force? @@ -49,6 +50,7 @@ module Commits def validate! validate_permissions! + validate_start_sha! validate_on_branch! validate_branch_existence! @@ -63,7 +65,21 @@ module Commits end end + def validate_start_sha! + return unless @start_sha + + if @start_branch + raise_error("You can't pass both start_branch and start_sha") + elsif !Gitlab::Git.commit_id?(@start_sha) + raise_error("Invalid start_sha '#{@start_sha}'") + elsif !@start_project.repository.commit(@start_sha) + raise_error("Cannot find start_sha '#{@start_sha}'") + end + end + def validate_on_branch! + return unless @start_branch + if !@start_project.empty_repo? && !@start_project.repository.branch_exists?(@start_branch) raise_error('You can only create or edit files when you are on a branch') end diff --git a/app/services/files/multi_service.rb b/app/services/files/multi_service.rb index d8c4e5bc5e8..65af4dd5a28 100644 --- a/app/services/files/multi_service.rb +++ b/app/services/files/multi_service.rb @@ -47,6 +47,7 @@ module Files author_name: @author_name, start_project: @start_project, start_branch_name: @start_branch, + start_sha: @start_sha, force: force? ) rescue ArgumentError => e diff --git a/app/services/issuable/clone/attributes_rewriter.rb b/app/services/issuable/clone/attributes_rewriter.rb index 0300cc0d8d3..3c061d35558 100644 --- a/app/services/issuable/clone/attributes_rewriter.rb +++ b/app/services/issuable/clone/attributes_rewriter.rb @@ -17,6 +17,8 @@ module Issuable private def cloneable_milestone + return unless new_entity.supports_milestone? + title = original_entity.milestone&.title return unless title diff --git a/app/services/wiki_pages/base_service.rb b/app/services/wiki_pages/base_service.rb index e259f5bd1bc..b9df690c2b7 100644 --- a/app/services/wiki_pages/base_service.rb +++ b/app/services/wiki_pages/base_service.rb @@ -8,6 +8,12 @@ module WikiPages page_data = Gitlab::DataBuilder::WikiPage.build(page, current_user, action) @project.execute_hooks(page_data, :wiki_page_hooks) @project.execute_services(page_data, :wiki_page_hooks) + increment_usage(action) + end + + # This method throws an error if the action is an unanticipated value. + def increment_usage(action) + Gitlab::UsageDataCounters::WikiPageCounter.count(action) end end end diff --git a/app/views/admin/requests_profiles/index.html.haml b/app/views/admin/requests_profiles/index.html.haml index adfc67d66d0..86bfeef580c 100644 --- a/app/views/admin/requests_profiles/index.html.haml +++ b/app/views/admin/requests_profiles/index.html.haml @@ -19,7 +19,8 @@ %ul.content-list - profiles.each do |profile| %li - = link_to profile.time.to_s(:long), admin_requests_profile_path(profile) + = link_to profile.time.to_s(:long) + ' ' + profile.profile_mode.capitalize, + admin_requests_profile_path(profile) - else %p No profiles found diff --git a/app/views/admin/users/index.html.haml b/app/views/admin/users/index.html.haml index c3d5ce0fe70..6fc7ec1bb6f 100644 --- a/app/views/admin/users/index.html.haml +++ b/app/views/admin/users/index.html.haml @@ -52,7 +52,7 @@ = icon("search", class: "search-icon") = button_tag s_('AdminUsers|Search users') if Rails.env.test? .dropdown.user-sort-dropdown - - toggle_text = if @sort.present? then users_sort_options_hash[@sort] else sort_title_name end + - toggle_text = @sort.present? ? users_sort_options_hash[@sort] : sort_title_name = dropdown_toggle(toggle_text, { toggle: 'dropdown' }) %ul.dropdown-menu.dropdown-menu-right %li.dropdown-header diff --git a/app/views/devise/passwords/edit.html.haml b/app/views/devise/passwords/edit.html.haml index 09ea7716a47..fee87c6324c 100644 --- a/app/views/devise/passwords/edit.html.haml +++ b/app/views/devise/passwords/edit.html.haml @@ -7,12 +7,12 @@ = f.hidden_field :reset_password_token .form-group = f.label 'New password', for: "user_password" - = f.password_field :password, class: "form-control top qa-password-field", required: true, title: 'This field is required' + = f.password_field :password, class: "form-control top", required: true, title: 'This field is required', data: { qa_selector: 'password_field'} .form-group = f.label 'Confirm new password', for: "user_password_confirmation" - = f.password_field :password_confirmation, class: "form-control bottom qa-password-confirmation", title: 'This field is required', required: true + = f.password_field :password_confirmation, class: "form-control bottom", title: 'This field is required', data: { qa_selector: 'password_confirmation_field' }, required: true .clearfix - = f.submit "Change your password", class: "btn btn-primary qa-change-password-button" + = f.submit "Change your password", class: "btn btn-primary", data: { qa_selector: 'change_password_button' } .clearfix.prepend-top-20 %p diff --git a/app/views/devise/sessions/_new_ldap.html.haml b/app/views/devise/sessions/_new_ldap.html.haml index f856773526d..31c4bb0e33e 100644 --- a/app/views/devise/sessions/_new_ldap.html.haml +++ b/app/views/devise/sessions/_new_ldap.html.haml @@ -3,13 +3,13 @@ = form_tag(omniauth_callback_path(:user, server['provider_name']), id: 'new_ldap_user', class: "gl-show-field-errors") do .form-group = label_tag :username, "#{server['label']} Username" - = text_field_tag :username, nil, { class: "form-control top qa-username-field", title: "This field is required.", autofocus: "autofocus", required: true } + = text_field_tag :username, nil, { class: "form-control top", title: "This field is required.", autofocus: "autofocus", data: { qa_selector: 'username_field' }, required: true } .form-group = label_tag :password - = password_field_tag :password, nil, { class: "form-control bottom qa-password-field", title: "This field is required.", required: true } + = password_field_tag :password, nil, { class: "form-control bottom", title: "This field is required.", data: { qa_selector: 'password_field' }, required: true } - if devise_mapping.rememberable? .remember-me %label{ for: "remember_me" } = check_box_tag :remember_me, '1', false, id: 'remember_me' %span Remember me - = submit_tag "Sign in", class: "btn-success btn qa-sign-in-button" + = submit_tag "Sign in", class: "btn-success btn", data: { qa_selector: 'sign_in_button' } diff --git a/app/views/devise/shared/_tabs_ldap.html.haml b/app/views/devise/shared/_tabs_ldap.html.haml index b1a9470cf1c..db54c166a53 100644 --- a/app/views/devise/shared/_tabs_ldap.html.haml +++ b/app/views/devise/shared/_tabs_ldap.html.haml @@ -5,13 +5,13 @@ = render_if_exists "devise/shared/kerberos_tab" - @ldap_servers.each_with_index do |server, i| %li.nav-item - = link_to server['label'], "##{server['provider_name']}", class: "nav-link #{active_when(i.zero? && form_based_auth_provider_has_active_class?(:ldapmain))} qa-ldap-tab", 'data-toggle' => 'tab' + = link_to server['label'], "##{server['provider_name']}", class: "nav-link #{active_when(i.zero? && form_based_auth_provider_has_active_class?(:ldapmain))}", data: { toggle: 'tab', qa_selector: 'ldap_tab' } = render_if_exists 'devise/shared/tab_smartcard' - if password_authentication_enabled_for_web? %li.nav-item - = link_to 'Standard', '#login-pane', class: 'nav-link qa-standard-tab', 'data-toggle' => 'tab' + = link_to 'Standard', '#login-pane', class: 'nav-link', data: { toggle: 'tab', qa_selector: 'standard_tab' } - if allow_signup? %li.nav-item - = link_to 'Register', '#register-pane', class: 'nav-link qa-register-tab', 'data-toggle' => 'tab' + = link_to 'Register', '#register-pane', class: 'nav-link', data: { toggle: 'tab', qa_selector: 'register_tab' } diff --git a/app/views/devise/shared/_tabs_normal.html.haml b/app/views/devise/shared/_tabs_normal.html.haml index ab8c22532fd..b6a1b8805ee 100644 --- a/app/views/devise/shared/_tabs_normal.html.haml +++ b/app/views/devise/shared/_tabs_normal.html.haml @@ -1,6 +1,6 @@ %ul.nav-links.new-session-tabs.nav-tabs.nav{ role: 'tablist' } %li.nav-item{ role: 'presentation' } - %a.nav-link.qa-sign-in-tab.active{ href: '#login-pane', data: { toggle: 'tab' }, role: 'tab' } Sign in + %a.nav-link.active{ href: '#login-pane', data: { toggle: 'tab', qa_selector: 'sign_in_tab' }, role: 'tab' } Sign in - if allow_signup? %li.nav-item{ role: 'presentation' } - %a.nav-link.qa-register-tab{ href: '#register-pane', data: { track_label: 'sign_in_register', track_property: '', track_event: 'click_button', track_value: '', toggle: 'tab' }, role: 'tab' } Register + %a.nav-link{ href: '#register-pane', data: { track_label: 'sign_in_register', track_property: '', track_event: 'click_button', track_value: '', toggle: 'tab', qa_selector: 'register_tab' }, role: 'tab' } Register diff --git a/app/views/layouts/header/_current_user_dropdown.html.haml b/app/views/layouts/header/_current_user_dropdown.html.haml index 45fc39dbbdb..808290afcad 100644 --- a/app/views/layouts/header/_current_user_dropdown.html.haml +++ b/app/views/layouts/header/_current_user_dropdown.html.haml @@ -20,8 +20,8 @@ = link_to s_("CurrentUser|Profile"), current_user, class: 'profile-link', data: { user: current_user.username } - if current_user_menu?(:settings) %li - = link_to s_("CurrentUser|Settings"), profile_path + = link_to s_("CurrentUser|Settings"), profile_path, data: { qa_selector: 'settings_link' } - if current_user_menu?(:sign_out) %li.divider %li - = link_to _("Sign out"), destroy_user_session_path, class: "sign-out-link qa-sign-out-link" + = link_to _("Sign out"), destroy_user_session_path, class: "sign-out-link", data: { qa_selector: 'sign_out_link' } diff --git a/app/views/projects/merge_requests/creations/_new_compare.html.haml b/app/views/projects/merge_requests/creations/_new_compare.html.haml index 11272a67f93..be01905dd35 100644 --- a/app/views/projects/merge_requests/creations/_new_compare.html.haml +++ b/app/views/projects/merge_requests/creations/_new_compare.html.haml @@ -2,6 +2,8 @@ New Merge Request = form_for [@project.namespace.becomes(Namespace), @project, @merge_request], url: project_new_merge_request_path(@project), method: :get, html: { class: "merge-request-form js-requires-input" } do |f| + - if params[:nav_source].present? + = hidden_field_tag(:nav_source, params[:nav_source]) .hide.alert.alert-danger.mr-compare-errors .js-merge-request-new-compare.row{ 'data-source-branch-url': project_new_merge_request_branch_from_path(@source_project), 'data-target-branch-url': project_new_merge_request_branch_to_path(@source_project) } .col-lg-6 diff --git a/app/views/projects/merge_requests/creations/_new_submit.html.haml b/app/views/projects/merge_requests/creations/_new_submit.html.haml index 464f8fa65e9..543441b9479 100644 --- a/app/views/projects/merge_requests/creations/_new_submit.html.haml +++ b/app/views/projects/merge_requests/creations/_new_submit.html.haml @@ -17,6 +17,9 @@ = f.hidden_field :target_project_id = f.hidden_field :target_branch, id: '' + - if params[:nav_source].present? + = hidden_field_tag(:nav_source, params[:nav_source]) + .mr-compare.merge-request.js-merge-request-new-submit{ 'data-mr-submit-action': "#{j params[:tab].presence || 'new'}" } - if @commits.empty? .commits-empty diff --git a/app/views/shared/_issuable_meta_data.html.haml b/app/views/shared/_issuable_meta_data.html.haml index 71b13a5d741..7807371285c 100644 --- a/app/views/shared/_issuable_meta_data.html.haml +++ b/app/views/shared/_issuable_meta_data.html.haml @@ -1,7 +1,7 @@ - note_count = @issuable_meta_data[issuable.id].user_notes_count - issue_votes = @issuable_meta_data[issuable.id] - upvotes, downvotes = issue_votes.upvotes, issue_votes.downvotes -- issuable_url = @collection_type == "Issue" ? issue_path(issuable, anchor: 'notes') : merge_request_path(issuable, anchor: 'notes') +- issuable_path = issuable_path(issuable, anchor: 'notes') - issuable_mr = @issuable_meta_data[issuable.id].merge_requests_count(current_user) - if issuable_mr > 0 @@ -20,6 +20,6 @@ = downvotes %li.issuable-comments.d-none.d-sm-block - = link_to issuable_url, class: ['has-tooltip', ('no-comments' if note_count.zero?)], title: _('Comments') do + = link_to issuable_path, class: ['has-tooltip', ('no-comments' if note_count.zero?)], title: _('Comments') do = icon('comments') = note_count diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 3d34bfc05c7..991a177018e 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -3,6 +3,12 @@ - auto_merge:auto_merge_process +- chaos:chaos_cpu_spin +- chaos:chaos_db_spin +- chaos:chaos_kill +- chaos:chaos_leak_mem +- chaos:chaos_sleep + - cronjob:admin_email - cronjob:expire_build_artifacts - cronjob:gitlab_usage_ping diff --git a/app/workers/chaos/cpu_spin_worker.rb b/app/workers/chaos/cpu_spin_worker.rb new file mode 100644 index 00000000000..43a32c3274f --- /dev/null +++ b/app/workers/chaos/cpu_spin_worker.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Chaos + class CpuSpinWorker + include ApplicationWorker + include ChaosQueue + + def perform(duration_s) + Gitlab::Chaos.cpu_spin(duration_s) + end + end +end diff --git a/app/workers/chaos/db_spin_worker.rb b/app/workers/chaos/db_spin_worker.rb new file mode 100644 index 00000000000..217ddabbcb6 --- /dev/null +++ b/app/workers/chaos/db_spin_worker.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Chaos + class DbSpinWorker + include ApplicationWorker + include ChaosQueue + + def perform(duration_s, interval_s) + Gitlab::Chaos.db_spin(duration_s, interval_s) + end + end +end diff --git a/app/workers/chaos/kill_worker.rb b/app/workers/chaos/kill_worker.rb new file mode 100644 index 00000000000..bbad53c9b86 --- /dev/null +++ b/app/workers/chaos/kill_worker.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Chaos + class KillWorker + include ApplicationWorker + include ChaosQueue + + def perform + Gitlab::Chaos.kill + end + end +end diff --git a/app/workers/chaos/leak_mem_worker.rb b/app/workers/chaos/leak_mem_worker.rb new file mode 100644 index 00000000000..0caa99e0de9 --- /dev/null +++ b/app/workers/chaos/leak_mem_worker.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Chaos + class LeakMemWorker + include ApplicationWorker + include ChaosQueue + + def perform(memory_mb, duration_s) + Gitlab::Chaos.leak_mem(memory_mb, duration_s) + end + end +end diff --git a/app/workers/chaos/sleep_worker.rb b/app/workers/chaos/sleep_worker.rb new file mode 100644 index 00000000000..7c724c4cb4e --- /dev/null +++ b/app/workers/chaos/sleep_worker.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Chaos + class SleepWorker + include ApplicationWorker + include ChaosQueue + + def perform(duration_s) + Gitlab::Chaos.sleep(duration_s) + end + end +end diff --git a/app/workers/concerns/chaos_queue.rb b/app/workers/concerns/chaos_queue.rb new file mode 100644 index 00000000000..e406509d12d --- /dev/null +++ b/app/workers/concerns/chaos_queue.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true +# +module ChaosQueue + extend ActiveSupport::Concern + + included do + queue_namespace :chaos + end +end diff --git a/app/workers/gitlab_usage_ping_worker.rb b/app/workers/gitlab_usage_ping_worker.rb index b75e724ca98..a5e22f88a3b 100644 --- a/app/workers/gitlab_usage_ping_worker.rb +++ b/app/workers/gitlab_usage_ping_worker.rb @@ -6,10 +6,16 @@ class GitlabUsagePingWorker include ApplicationWorker include CronjobQueue + # Retry for up to approximately three hours then give up. + sidekiq_options retry: 10, dead: false + def perform # Multiple Sidekiq workers could run this. We should only do this at most once a day. return unless try_obtain_lease + # Splay the request over a minute to avoid thundering herd problems. + sleep(rand(0.0..60.0).round(3)) + SubmitUsagePingService.new.execute end diff --git a/changelogs/unreleased/63833-fix-jira-issues-url.yml b/changelogs/unreleased/63833-fix-jira-issues-url.yml new file mode 100644 index 00000000000..24d6bca3842 --- /dev/null +++ b/changelogs/unreleased/63833-fix-jira-issues-url.yml @@ -0,0 +1,5 @@ +--- +title: Handle trailing slashes when generating Jira issue URLs +merge_request: 30911 +author: +type: fixed diff --git a/changelogs/unreleased/64160-fix-duplicate-buttons.yml b/changelogs/unreleased/64160-fix-duplicate-buttons.yml new file mode 100644 index 00000000000..12416a611ed --- /dev/null +++ b/changelogs/unreleased/64160-fix-duplicate-buttons.yml @@ -0,0 +1,5 @@ +--- +title: Remove duplicate buttons in diff discussion +merge_request: 30757 +author: +type: fixed diff --git a/changelogs/unreleased/64265-center-loading-icon.yml b/changelogs/unreleased/64265-center-loading-icon.yml new file mode 100644 index 00000000000..cd4253b63c6 --- /dev/null +++ b/changelogs/unreleased/64265-center-loading-icon.yml @@ -0,0 +1,5 @@ +--- +title: Center loading icon in CI action component +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/64295-predictable-environment-slugs.yml b/changelogs/unreleased/64295-predictable-environment-slugs.yml new file mode 100644 index 00000000000..de581b2b8e1 --- /dev/null +++ b/changelogs/unreleased/64295-predictable-environment-slugs.yml @@ -0,0 +1,5 @@ +--- +title: Use predictable environment slugs +merge_request: 30551 +author: +type: added diff --git a/changelogs/unreleased/64731-fix-project-auto-devops-api.yml b/changelogs/unreleased/64731-fix-project-auto-devops-api.yml new file mode 100644 index 00000000000..7fe2c036773 --- /dev/null +++ b/changelogs/unreleased/64731-fix-project-auto-devops-api.yml @@ -0,0 +1,5 @@ +--- +title: Fix the project auto devops API +merge_request: 30946 +author: +type: fixed diff --git a/changelogs/unreleased/add-support-for-start-sha-to-commits-api.yml b/changelogs/unreleased/add-support-for-start-sha-to-commits-api.yml new file mode 100644 index 00000000000..f810c2c5ada --- /dev/null +++ b/changelogs/unreleased/add-support-for-start-sha-to-commits-api.yml @@ -0,0 +1,5 @@ +--- +title: Add support for start_sha to commits API +merge_request: 29598 +author: +type: changed diff --git a/changelogs/unreleased/an-sidekiq-chaos.yml b/changelogs/unreleased/an-sidekiq-chaos.yml new file mode 100644 index 00000000000..cede35c95cc --- /dev/null +++ b/changelogs/unreleased/an-sidekiq-chaos.yml @@ -0,0 +1,5 @@ +--- +title: Adds chaos endpoints to Sidekiq +merge_request: 30814 +author: +type: other diff --git a/changelogs/unreleased/bjk-64064_cache_metrics.yml b/changelogs/unreleased/bjk-64064_cache_metrics.yml new file mode 100644 index 00000000000..c9baff7cb7b --- /dev/null +++ b/changelogs/unreleased/bjk-64064_cache_metrics.yml @@ -0,0 +1,5 @@ +--- +title: Adjust redis cache metrics +merge_request: 30572 +author: +type: changed diff --git a/changelogs/unreleased/bjk-usage_ping.yml b/changelogs/unreleased/bjk-usage_ping.yml new file mode 100644 index 00000000000..dee6c1ad291 --- /dev/null +++ b/changelogs/unreleased/bjk-usage_ping.yml @@ -0,0 +1,5 @@ +--- +title: Update usage ping cron behavior +merge_request: 30842 +author: +type: performance diff --git a/changelogs/unreleased/ci_default_git_depth_only.yml b/changelogs/unreleased/ci_default_git_depth_only.yml new file mode 100644 index 00000000000..928e2fe3ec4 --- /dev/null +++ b/changelogs/unreleased/ci_default_git_depth_only.yml @@ -0,0 +1,5 @@ +--- +title: 'API: Allow changing only ci_default_git_depth' +merge_request: 30888 +author: Mathieu Parent +type: fixed diff --git a/changelogs/unreleased/fe-delete-old-boardservice.yml b/changelogs/unreleased/fe-delete-old-boardservice.yml new file mode 100644 index 00000000000..bb06bfe80d5 --- /dev/null +++ b/changelogs/unreleased/fe-delete-old-boardservice.yml @@ -0,0 +1,6 @@ +--- +title: Change BoardService in favor of boardsStore on board blank state of the component + board +merge_request: 30546 +author: eduarmreyes +type: other diff --git a/changelogs/unreleased/feature-gb-serverless-app-deployment-template.yml b/changelogs/unreleased/feature-gb-serverless-app-deployment-template.yml new file mode 100644 index 00000000000..bd9001bd671 --- /dev/null +++ b/changelogs/unreleased/feature-gb-serverless-app-deployment-template.yml @@ -0,0 +1,5 @@ +--- +title: Deploy serverless apps with gitlabktl +merge_request: 30740 +author: +type: added diff --git a/changelogs/unreleased/fix-alignment-on-security-reports.yml b/changelogs/unreleased/fix-alignment-on-security-reports.yml new file mode 100644 index 00000000000..5339b6d764d --- /dev/null +++ b/changelogs/unreleased/fix-alignment-on-security-reports.yml @@ -0,0 +1,5 @@ +--- +title: Fixes alignment issues with reports +merge_request: 30839 +author: +type: fixed diff --git a/changelogs/unreleased/fj-avoid-incresaing-usage-ping-when-not-enabled.yml b/changelogs/unreleased/fj-avoid-incresaing-usage-ping-when-not-enabled.yml new file mode 100644 index 00000000000..f1077f2d56d --- /dev/null +++ b/changelogs/unreleased/fj-avoid-incresaing-usage-ping-when-not-enabled.yml @@ -0,0 +1,5 @@ +--- +title: Avoid increasing redis counters when usage_ping is disabled +merge_request: 30949 +author: +type: changed diff --git a/changelogs/unreleased/fj-count-web-ide-merge-requests.yml b/changelogs/unreleased/fj-count-web-ide-merge-requests.yml new file mode 100644 index 00000000000..adcd0af37e8 --- /dev/null +++ b/changelogs/unreleased/fj-count-web-ide-merge-requests.yml @@ -0,0 +1,5 @@ +--- +title: Add Web IDE Usage Ping for Create SMAU +merge_request: 30800 +author: +type: changed diff --git a/changelogs/unreleased/georgekoltsov-64311-set-visibility-private-if-internal-restricted.yml b/changelogs/unreleased/georgekoltsov-64311-set-visibility-private-if-internal-restricted.yml new file mode 100644 index 00000000000..18af16e5216 --- /dev/null +++ b/changelogs/unreleased/georgekoltsov-64311-set-visibility-private-if-internal-restricted.yml @@ -0,0 +1,5 @@ +--- +title: Set visibility level 'Private' for restricted 'Internal' imported projects when 'Internal' visibility setting is restricted in admin settings +merge_request: 30522 +author: +type: other diff --git a/changelogs/unreleased/group-milestones-dashboard-blunceford.yml b/changelogs/unreleased/group-milestones-dashboard-blunceford.yml new file mode 100644 index 00000000000..a6ded27cb8c --- /dev/null +++ b/changelogs/unreleased/group-milestones-dashboard-blunceford.yml @@ -0,0 +1,5 @@ +--- +title: Fix bug in dashboard display of closed milestones +merge_request: 30820 +author: +type: fixed diff --git a/changelogs/unreleased/sh-add-rugged-logs.yml b/changelogs/unreleased/sh-add-rugged-logs.yml new file mode 100644 index 00000000000..1f464dd92ff --- /dev/null +++ b/changelogs/unreleased/sh-add-rugged-logs.yml @@ -0,0 +1,5 @@ +--- +title: Add Rugged calls and duration to API and Rails logs +merge_request: 30871 +author: +type: other diff --git a/changelogs/unreleased/sh-fix-gitaly-server-info-cache.yml b/changelogs/unreleased/sh-fix-gitaly-server-info-cache.yml new file mode 100644 index 00000000000..ef8f6e25856 --- /dev/null +++ b/changelogs/unreleased/sh-fix-gitaly-server-info-cache.yml @@ -0,0 +1,5 @@ +--- +title: Fix Gitaly auto-detection caching +merge_request: 30954 +author: +type: performance diff --git a/changelogs/unreleased/wiki-usage-pings.yml b/changelogs/unreleased/wiki-usage-pings.yml new file mode 100644 index 00000000000..c3d084228c3 --- /dev/null +++ b/changelogs/unreleased/wiki-usage-pings.yml @@ -0,0 +1,5 @@ +--- +title: Count wiki creation, update and delete events +merge_request: 30864 +author: +type: added diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb index 53c3eac3c74..3f2691dde95 100644 --- a/config/initializers/7_prometheus_metrics.rb +++ b/config/initializers/7_prometheus_metrics.rb @@ -1,5 +1,4 @@ require 'prometheus/client' -require 'prometheus/client/support/unicorn' # Keep separate directories for separate processes def prometheus_default_multiproc_dir @@ -23,7 +22,7 @@ Prometheus::Client.configure do |config| config.multiprocess_files_dir = ENV['prometheus_multiproc_dir'] || prometheus_default_multiproc_dir - config.pid_provider = Prometheus::Client::Support::Unicorn.method(:worker_pid_provider) + config.pid_provider = Prometheus::PidProvider.method(:worker_id) end Gitlab::Application.configure do |config| diff --git a/config/initializers/lograge.rb b/config/initializers/lograge.rb index fbec28186eb..3d84b4e44ce 100644 --- a/config/initializers/lograge.rb +++ b/config/initializers/lograge.rb @@ -34,6 +34,13 @@ unless Sidekiq.server? payload[:gitaly_duration] = Gitlab::GitalyClient.query_time_ms end + rugged_calls = Gitlab::RuggedInstrumentation.query_count + + if rugged_calls > 0 + payload[:rugged_calls] = rugged_calls + payload[:rugged_duration_ms] = Gitlab::RuggedInstrumentation.query_time_ms + end + payload[:response] = event.payload[:response] if event.payload[:response] payload[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb index d492b60705d..2aec32e878c 100644 --- a/config/initializers/peek.rb +++ b/config/initializers/peek.rb @@ -29,7 +29,7 @@ end Peek.into PEEK_DB_VIEW Peek.into Peek::Views::Gitaly Peek.into Peek::Views::Rblineprof -Peek.into Peek::Views::Redis +Peek.into Peek::Views::RedisDetailed Peek.into Peek::Views::GC Peek.into Peek::Views::Tracing if Labkit::Tracing.tracing_url_enabled? diff --git a/config/karma.config.js b/config/karma.config.js index 2a5bf3581e0..97794225a3f 100644 --- a/config/karma.config.js +++ b/config/karma.config.js @@ -107,7 +107,8 @@ if (specFilters.length) { module.exports = function(config) { process.env.TZ = 'Etc/UTC'; - const fixturesPath = `${IS_EE ? 'ee/' : ''}spec/javascripts/fixtures`; + const fixturesPath = `tmp/tests/frontend/fixtures${IS_EE ? '-ee' : ''}`; + const staticFixturesPath = 'spec/frontend/fixtures/static'; const karmaConfig = { basePath: ROOT_PATH, @@ -131,8 +132,13 @@ module.exports = function(config) { frameworks: ['jasmine'], files: [ { pattern: 'spec/javascripts/test_bundle.js', watched: false }, - { pattern: `${fixturesPath}/**/*@(.json|.html|.png|.bmpr|.pdf)`, included: false }, + { pattern: `${fixturesPath}/**/*`, included: false }, + { pattern: `${staticFixturesPath}/**/*`, included: false }, ], + proxies: { + '/fixtures/': `/base/${fixturesPath}/`, + '/fixtures/static/': `/base/${staticFixturesPath}/`, + }, preprocessors: { 'spec/javascripts/**/*.js': ['webpack', 'sourcemap'], 'ee/spec/javascripts/**/*.js': ['webpack', 'sourcemap'], diff --git a/config/routes.rb b/config/routes.rb index 641807203bf..459f2b22bf0 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -116,7 +116,7 @@ Rails.application.routes.draw do end end - if ENV['GITLAB_CHAOS_SECRET'] || Rails.env.development? + if ENV['GITLAB_CHAOS_SECRET'] || Rails.env.development? || Rails.env.test? resource :chaos, only: [] do get :leakmem get :cpu_spin diff --git a/config/routes/admin.rb b/config/routes/admin.rb index f609739d9fd..6f9a5552564 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -73,7 +73,7 @@ namespace :admin do resource :background_jobs, controller: 'background_jobs', only: [:show] resource :system_info, controller: 'system_info', only: [:show] - resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ } + resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.(html|txt)/ } resources :projects, only: [:index] diff --git a/config/settings.rb b/config/settings.rb index da459afcce2..8756c120645 100644 --- a/config/settings.rb +++ b/config/settings.rb @@ -1,4 +1,5 @@ require 'settingslogic' +require 'digest/md5' # We can not use `Rails.root` here, as this file might be loaded without the # full Rails environment being loaded. We can not use `require_relative` either, @@ -170,14 +171,17 @@ class Settings < Settingslogic URI.parse(url_without_path).host end - # Runs every minute in a random ten-minute period on Sundays, to balance the - # load on the server receiving these pings. The usage ping is safe to run - # multiple times because of a 24 hour exclusive lock. + # Runs at a random time of day on a consistent day of the week based on + # the instance UUID. This is to balance the load on the service receiving + # these pings. The sidekiq job handles temporary http failures. def cron_for_usage_ping hour = rand(24) - minute = rand(6) + minute = rand(60) + # Set a default UUID for the case when the UUID hasn't been initialized. + uuid = Gitlab::CurrentSettings.uuid || 'uuid-not-set' + day_of_week = Digest::MD5.hexdigest(uuid).to_i(16) % 7 - "#{minute}0-#{minute}9 #{hour} * * 0" + "#{minute} #{hour} * * #{day_of_week}" end end end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 80791795390..c7586aa1e38 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -95,6 +95,7 @@ - [update_project_statistics, 1] - [phabricator_import_import_tasks, 1] - [update_namespace_statistics, 1] + - [chaos, 2] # EE-specific queues - [ldap_group_sync, 2] diff --git a/danger/database/Dangerfile b/danger/database/Dangerfile index 083e95b8da7..3550cb7eabf 100644 --- a/danger/database/Dangerfile +++ b/danger/database/Dangerfile @@ -8,6 +8,21 @@ updated too (unless the migration isn't changing the DB schema and isn't the most recent one). MSG +DB_MESSAGE = <<~MSG +This merge request requires a database review. To make sure these +changes are reviewed, take the following steps: + +1. Ensure the merge request has ~database and ~"database::review pending" labels. + If the merge request modifies database files, Danger will do this for you. +1. Use the [Database changes checklist](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.gitlab/merge_request_templates/Database%20changes.md) + template or add the appropriate items to the MR description. +1. Assign and mention the database reviewer suggested by Reviewer Roulette. +MSG + +DB_FILES_MESSAGE = <<~MSG +The following files require a review from the Database team: +MSG + non_geo_db_schema_updated = !git.modified_files.grep(%r{\Adb/schema\.rb}).empty? geo_db_schema_updated = !git.modified_files.grep(%r{\Aee/db/geo/schema\.rb}).empty? @@ -24,27 +39,18 @@ end db_paths_to_review = helper.changes_by_category[:database] -unless db_paths_to_review.empty? +if gitlab.mr_labels.include?('database') || db_paths_to_review.any? message 'This merge request adds or changes files that require a ' \ - 'review from the Database team.' - - markdown(<<~MARKDOWN.strip) -## Database Review - -The following files require a review from the Database team: - -* #{db_paths_to_review.map { |path| "`#{path}`" }.join("\n* ")} + 'review from the [Database team](https://gitlab.com/groups/gl-database/-/group_members).' -To make sure these changes are reviewed, take the following steps: + markdown(DB_MESSAGE) + markdown(DB_FILES_MESSAGE + helper.markdown_list(db_paths_to_review)) if db_paths_to_review.any? -1. Edit your merge request, and add `gl-database` to the list of Group - approvers. -1. Mention `@gl-database` in a separate comment, and explain what needs to be - reviewed by the team. Please don't mention the team until your changes are - ready for review. - MARKDOWN + database_labels = helper.missing_database_labels(gitlab.mr_labels) - unless gitlab.mr_labels.include?('database') - warn 'This merge request is missing the ~database label.' + if database_labels.any? + gitlab.api.update_merge_request(gitlab.mr_json['project_id'], + gitlab.mr_json['iid'], + labels: (gitlab.mr_labels + database_labels).join(',')) end end diff --git a/danger/roulette/Dangerfile b/danger/roulette/Dangerfile index 6718e218233..19a5076778c 100644 --- a/danger/roulette/Dangerfile +++ b/danger/roulette/Dangerfile @@ -55,22 +55,15 @@ def spin_for_category(team, project, category, branch_name) "| #{helper.label_for_category(category)} | #{reviewer&.markdown_name || NO_REVIEWER} | #{maintainer&.markdown_name || NO_MAINTAINER} |" end -def build_list(items) - list = items.map { |filename| "* `#{filename}`" }.join("\n") - - if items.size > 10 - "\n<details>\n\n#{list}\n\n</details>" - else - list - end -end - changes = helper.changes_by_category # Ignore any files that are known but uncategorized. Prompt for any unknown files changes.delete(:none) categories = changes.keys - [:unknown] +# Ensure to spin for database reviewer/maintainer when ~database is applied (e.g. to review SQL queries) +categories << :database if gitlab.mr_labels.include?('database') && !categories.include?(:database) + # Single codebase MRs are reviewed using a slightly different process, so we # disable the review roulette for such MRs. # CSS Clean up MRs are reviewed using a slightly different process, so we @@ -95,5 +88,5 @@ if changes.any? && !gitlab.mr_labels.include?('single codebase') && !gitlab.mr_l markdown(MESSAGE) markdown(CATEGORY_TABLE_HEADER + rows.join("\n")) unless rows.empty? - markdown(UNKNOWN_FILES_MESSAGE + build_list(unknown)) unless unknown.empty? + markdown(UNKNOWN_FILES_MESSAGE + helper.markdown_list(unknown)) unless unknown.empty? end diff --git a/doc/administration/auth/ldap-ee.md b/doc/administration/auth/ldap-ee.md index 2f2ee8a27d3..1b7af2d945a 100644 --- a/doc/administration/auth/ldap-ee.md +++ b/doc/administration/auth/ldap-ee.md @@ -193,7 +193,7 @@ group, as opposed to the full DN. to lock down user abilities to invite new members to a group. When enabled following happens: 1. Only administrator can manage memberships of any group including access levels. -2. Users are not allowed to share project with other groups or invite members to a project created in a group. +1. Users are not allowed to share project with other groups or invite members to a project created in a group. ## Adjusting LDAP user sync schedule diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 4407facfca9..0f547ef03bf 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -2,45 +2,35 @@ [Gitaly](https://gitlab.com/gitlab-org/gitaly) is the service that provides high-level RPC access to Git repositories. Without it, no other -components can read or write Git data. +components can read or write Git data. GitLab components that access Git +repositories (gitlab-rails, gitlab-shell, gitlab-workhorse, etc.) act as clients +to Gitaly. End users do not have direct access to Gitaly. -GitLab components that access Git repositories (gitlab-rails, -gitlab-shell, gitlab-workhorse) act as clients to Gitaly. End users do -not have direct access to Gitaly. +In the rest of this page, Gitaly server is referred to the standalone node that +only runs Gitaly, and Gitaly client to the GitLab Rails node that runs all other +processes except Gitaly. ## Configuring Gitaly The Gitaly service itself is configured via a TOML configuration file. -This file is documented [in the gitaly +This file is documented [in the Gitaly repository](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configuration/README.md). -To change a Gitaly setting in Omnibus you can use -`gitaly['my_setting']` in `/etc/gitlab/gitlab.rb`. Changes will be applied -when you run `gitlab-ctl reconfigure`. +In case you want to change some of its settings: -```ruby -gitaly['prometheus_listen_addr'] = 'localhost:9236' -``` - -To change a Gitaly setting in installations from source you can edit -`/home/git/gitaly/config.toml`. Changes will be applied when you run -`service gitlab restart`. +**For Omnibus GitLab** -```toml -prometheus_listen_addr = "localhost:9236" -``` +1. Edit `/etc/gitlab/gitlab.rb` and add or change the [Gitaly settings](https://gitlab.com/gitlab-org/omnibus-gitlab/blob/1dd07197c7e5ae23626aad5a4a070a800b670380/files/gitlab-config-template/gitlab.rb.template#L1622-1676). +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -## Client-side GRPC logs +**For installations from source** -Gitaly uses the [gRPC](https://grpc.io/) RPC framework. The Ruby gRPC -client has its own log file which may contain useful information when -you are seeing Gitaly errors. You can control the log level of the -gRPC client with the `GRPC_LOG_LEVEL` environment variable. The -default level is `WARN`. +1. Edit `/home/git/gitaly/config.toml` and add or change the [Gitaly settings](https://gitlab.com/gitlab-org/gitaly/blob/master/config.toml.example). +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). ## Running Gitaly on its own server -> This is an optional way to deploy Gitaly which can benefit GitLab +This is an optional way to deploy Gitaly which can benefit GitLab installations that are larger than a single machine. Most installations will be better served with the default configuration used by Omnibus and the GitLab source installation guide. @@ -53,76 +43,78 @@ But since 11.8 the indexer uses Gitaly for data access as well. NFS can still be leveraged for redudancy on block level of the Git data. But only has to be mounted on the Gitaly server. -NOTE: **Note:** While Gitaly can be used as a replacement for NFS, we do not recommend -using EFS as it may impact GitLab's performance. Please review the [relevant documentation](../high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) +NOTE: **Note:** While Gitaly can be used as a replacement for NFS, it's not recommended +to use EFS as it may impact GitLab's performance. Review the [relevant documentation](../high_availability/nfs.md#avoid-using-awss-elastic-file-system-efs) for more details. ### Network architecture -- gitlab-rails shards repositories into "repository storages" -- `gitlab-rails/config/gitlab.yml` contains a map from storage names to - (Gitaly address, Gitaly token) pairs -- the `storage name` -\> `(Gitaly address, Gitaly token)` map in - `gitlab.yml` is the single source of truth for the Gitaly network - topology -- a (Gitaly address, Gitaly token) corresponds to a Gitaly server -- a Gitaly server hosts one or more storages -- Gitaly addresses must be specified in such a way that they resolve - correctly for ALL Gitaly clients -- Gitaly clients are: unicorn, sidekiq, gitlab-workhorse, - gitlab-shell, Elasticsearch Indexer, and Gitaly itself -- special case: a Gitaly server must be able to make RPC calls **to - itself** via its own (Gitaly address, Gitaly token) pair as - specified in `gitlab-rails/config/gitlab.yml` -- Gitaly servers must not be exposed to the public internet - -Gitaly network traffic is unencrypted by default, but supports -[TLS](#tls-support). Authentication is done through a static token. - -NOTE: **Note:** Gitaly network traffic is unencrypted so we recommend a firewall to -restrict access to your Gitaly server. +The following list depicts what the network architecture of Gitaly is: + +- GitLab Rails shards repositories into [repository storages](../repository_storage_paths.md). +- `/config/gitlab.yml` contains a map from storage names to + `(Gitaly address, Gitaly token)` pairs. +- the `storage name` -\> `(Gitaly address, Gitaly token)` map in + `/config/gitlab.yml` is the single source of truth for the Gitaly network + topology. +- A `(Gitaly address, Gitaly token)` corresponds to a Gitaly server. +- A Gitaly server hosts one or more storages. +- Gitaly addresses must be specified in such a way that they resolve + correctly for ALL Gitaly clients. +- Gitaly clients are: Unicorn, Sidekiq, gitlab-workhorse, + gitlab-shell, Elasticsearch Indexer, and Gitaly itself. +- A Gitaly server must be able to make RPC calls **to itself** via its own + `(Gitaly address, Gitaly token)` pair as specified in `/config/gitlab.yml`. +- Gitaly servers must not be exposed to the public internet as Gitaly's network + traffic is unencrypted by default. The use of firewall is highly recommended + to restrict access to the Gitaly server. Another option is to + [use TLS](#tls-support). +- Authentication is done through a static token which is shared among the Gitaly + and GitLab Rails nodes. Below we describe how to configure a Gitaly server at address `gitaly.internal:8075` with secret token `abc123secret`. We assume your GitLab installation has two repository storages, `default` and `storage1`. -### Installation +### 1. Installation -First install Gitaly using either Omnibus or from source. +First install Gitaly using either Omnibus GitLab or install it from source: -Omnibus: [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab -package you want using **steps 1 and 2** from the GitLab downloads page but -**_do not_** provide the `EXTERNAL_URL=` value. +- For Omnibus GitLab: [Download/install](https://about.gitlab.com/install/) the Omnibus GitLab + package you want using **steps 1 and 2** from the GitLab downloads page but + **_do not_** provide the `EXTERNAL_URL=` value. +- From source: [Install Gitaly](../../install/installation.md#install-gitaly). -Source: [Install Gitaly](../../install/installation.md#install-gitaly) +### 2. Client side token configuration -### Client side token configuration +Configure a token on the instance that runs the GitLab Rails application. -Configure a token on the client side. +**For Omnibus GitLab** -Omnibus installations: +1. On the client node(s), edit `/etc/gitlab/gitlab.rb`: -```ruby -# /etc/gitlab/gitlab.rb -gitlab_rails['gitaly_token'] = 'abc123secret' -``` + ```ruby + gitlab_rails['gitaly_token'] = 'abc123secret' + ``` -Source installations: +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -```yaml -# /home/git/gitlab/config/gitlab.yml -gitlab: - gitaly: - token: 'abc123secret' -``` +**For installations from source** + +1. On the client node(s), edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + gitlab: + gitaly: + token: 'abc123secret' + ``` -You need to reconfigure (Omnibus) or restart (source) for these -changes to be picked up. +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -### Gitaly server configuration +### 3. Gitaly server configuration -Next, on the Gitaly server, we need to configure storage paths, enable +Next, on the Gitaly server, you need to configure storage paths, enable the network listener and configure the token. NOTE: **Note:** if you want to reduce the risk of downtime when you enable @@ -137,88 +129,99 @@ the Gitaly server. The easiest way to accomplish this is to copy `/etc/gitlab/gi from an existing GitLab server to the Gitaly server. Without this shared secret, Git operations in GitLab will result in an API error. -NOTE: **Note:** In most or all cases the storage paths below end in `/repositories` which is -different than `path` in `git_data_dirs` of Omnibus installations. Check the -directory layout on your Gitaly server to be sure. - -Omnibus installations: - -<!-- -updates to following example must also be made at -https://gitlab.com/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab ---> - -```ruby -# /etc/gitlab/gitlab.rb - -# Avoid running unnecessary services on the Gitaly server -postgresql['enable'] = false -redis['enable'] = false -nginx['enable'] = false -prometheus['enable'] = false -unicorn['enable'] = false -sidekiq['enable'] = false -gitlab_workhorse['enable'] = false - -# Prevent database connections during 'gitlab-ctl reconfigure' -gitlab_rails['rake_cache_clear'] = false -gitlab_rails['auto_migrate'] = false - -# Configure the gitlab-shell API callback URL. Without this, `git push` will -# fail. This can be your 'front door' GitLab URL or an internal load -# balancer. -# Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. -gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' - -# Make Gitaly accept connections on all network interfaces. You must use -# firewalls to restrict access to this address/port. -gitaly['listen_addr'] = "0.0.0.0:8075" -gitaly['auth_token'] = 'abc123secret' - -gitaly['storage'] = [ - { 'name' => 'default', 'path' => '/mnt/gitlab/default/repositories' }, - { 'name' => 'storage1', 'path' => '/mnt/gitlab/storage1/repositories' }, -] - -# To use TLS for Gitaly you need to add -gitaly['tls_listen_addr'] = "0.0.0.0:9999" -gitaly['certificate_path'] = "path/to/cert.pem" -gitaly['key_path'] = "path/to/key.pem" -``` +NOTE: **Note:** +In most or all cases, the storage paths below end in `/repositories` which is +not that case with `path` in `git_data_dirs` of Omnibus GitLab installations. +Check the directory layout on your Gitaly server to be sure. -Source installations: +**For Omnibus GitLab** -```toml -# /home/git/gitaly/config.toml -listen_addr = '0.0.0.0:8075' -tls_listen_addr = '0.0.0.0:9999' +1. Edit `/etc/gitlab/gitlab.rb`: -[tls] -certificate_path = /path/to/cert.pem -key_path = /path/to/key.pem + <!-- + updates to following example must also be made at + https://gitlab.com/charts/gitlab/blob/master/doc/advanced/external-gitaly/external-omnibus-gitaly.md#configure-omnibus-gitlab + --> -[auth] -token = 'abc123secret' + ```ruby + # /etc/gitlab/gitlab.rb -[[storage]] -name = 'default' -path = '/mnt/gitlab/default/repositories' + # Avoid running unnecessary services on the Gitaly server + postgresql['enable'] = false + redis['enable'] = false + nginx['enable'] = false + prometheus['enable'] = false + unicorn['enable'] = false + sidekiq['enable'] = false + gitlab_workhorse['enable'] = false -[[storage]] -name = 'storage1' -path = '/mnt/gitlab/storage1/repositories' -``` + # Prevent database connections during 'gitlab-ctl reconfigure' + gitlab_rails['rake_cache_clear'] = false + gitlab_rails['auto_migrate'] = false + + # Configure the gitlab-shell API callback URL. Without this, `git push` will + # fail. This can be your 'front door' GitLab URL or an internal load + # balancer. + # Don't forget to copy `/etc/gitlab/gitlab-secrets.json` from web server to Gitaly server. + gitlab_rails['internal_api_url'] = 'https://gitlab.example.com' -Again, reconfigure (Omnibus) or restart (source). + # Make Gitaly accept connections on all network interfaces. You must use + # firewalls to restrict access to this address/port. + gitaly['listen_addr'] = "0.0.0.0:8075" + gitaly['auth_token'] = 'abc123secret' -### Converting clients to use the Gitaly server + gitaly['storage'] = [ + { 'name' => 'default' }, + { 'name' => 'storage1' }, + ] -Now as the final step update the client machines to switch from using -their local Gitaly service to the new Gitaly server you just -configured. This is a risky step because if there is any sort of -network, firewall, or name resolution problem preventing your GitLab -server from reaching the Gitaly server then all Gitaly requests will -fail. + # To use TLS for Gitaly you need to add + gitaly['tls_listen_addr'] = "0.0.0.0:9999" + gitaly['certificate_path'] = "path/to/cert.pem" + gitaly['key_path'] = "path/to/key.pem" + ``` + + NOTE: **Note:** + In some cases, you'll have to set `path` for `gitaly['storage']` in the + format `'path' => '/mnt/gitlab/<storage name>/repositories'`. + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +**For installations from source** + +1. On the client node(s), edit `/home/git/gitaly/config.toml`: + + ```toml + listen_addr = '0.0.0.0:8075' + tls_listen_addr = '0.0.0.0:9999' + + [tls] + certificate_path = /path/to/cert.pem + key_path = /path/to/key.pem + + [auth] + token = 'abc123secret' + + [[storage]] + name = 'default' + + [[storage]] + name = 'storage1' + ``` + + NOTE: **Note:** + In some cases, you'll have to set `path` for each `[[storage]]` in the + format `path = '/mnt/gitlab/<storage name>/repositories'`. + +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). + +### 4. Converting clients to use the Gitaly server + +As the final step, you need to update the client machines to switch from using +their local Gitaly service to the new Gitaly server you just configured. This +is a risky step because if there is any sort of network, firewall, or name +resolution problem preventing your GitLab server from reaching the Gitaly server, +then all Gitaly requests will fail. Additionally, you need to [disable Rugged if previously manually enabled](../high_availability/nfs.md#improving-nfs-performance-with-gitlab). @@ -227,41 +230,93 @@ We assume that your Gitaly server can be reached at `gitaly.internal:8075` from your GitLab server, and that Gitaly can read and write to `/mnt/gitlab/default` and `/mnt/gitlab/storage1` respectively. -Omnibus installations: +**For Omnibus GitLab** -```ruby -# /etc/gitlab/gitlab.rb -git_data_dirs({ - 'default' => { 'gitaly_address' => 'tcp://gitaly.internal:8075' }, - 'storage1' => { 'gitaly_address' => 'tcp://gitaly.internal:8075' }, -}) +1. Edit `/etc/gitlab/gitlab.rb`: -gitlab_rails['gitaly_token'] = 'abc123secret' -``` + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tcp://gitaly.internal:8075' }, + 'storage1' => { 'gitaly_address' => 'tcp://gitaly.internal:8075' }, + }) -Source installations: - -```yaml -# /home/git/gitlab/config/gitlab.yml -gitlab: - repositories: - storages: - default: - path: /mnt/gitlab/default/repositories - gitaly_address: tcp://gitaly.internal:8075 - storage1: - path: /mnt/gitlab/storage1/repositories - gitaly_address: tcp://gitaly.internal:8075 - - gitaly: - token: 'abc123secret' -``` + gitlab_rails['gitaly_token'] = 'abc123secret' + ``` -Now reconfigure (Omnibus) or restart (source). When you tail the -Gitaly logs on your Gitaly server (`sudo gitlab-ctl tail gitaly` or -`tail -f /home/git/gitlab/log/gitaly.log`) you should see requests -coming in. One sure way to trigger a Gitaly request is to clone a -repository from your GitLab server over HTTP. + NOTE: **Note:** + In some cases, you'll have to set `path` for each `git_data_dirs` in the + format `'path' => '/mnt/gitlab/<storage name>'`. + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. Tail the logs to see the requests: + + ```sh + sudo gitlab-ctl tail gitaly + ``` + +**For installations from source** + +1. Edit `/home/git/gitlab/config/gitlab.yml`: + + ```yaml + gitlab: + repositories: + storages: + default: + gitaly_address: tcp://gitaly.internal:8075 + storage1: + gitaly_address: tcp://gitaly.internal:8075 + + gitaly: + token: 'abc123secret' + ``` + + NOTE: **Note:** + In some cases, you'll have to set `path` for each of the `storages` in the + format `path: /mnt/gitlab/<storage name>/repositories`. + +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +1. Tail the logs to see the requests: + + ```sh + tail -f /home/git/gitlab/log/gitaly.log + ``` + +When you tail the Gitaly logs on your Gitaly server you should see requests +coming in. One sure way to trigger a Gitaly request is to clone a repository +from your GitLab server over HTTP. + +### Disabling the Gitaly service in a cluster environment + +If you are running Gitaly [as a remote +service](#running-gitaly-on-its-own-server) you may want to disable +the local Gitaly service that runs on your GitLab server by default. +Disabling Gitaly only makes sense when you run GitLab in a custom +cluster configuration, where different services run on different +machines. Disabling Gitaly on all machines in the cluster is not a +valid configuration. + +To disable Gitaly on a client node: + +**For Omnibus GitLab** + +1. Edit `/etc/gitlab/gitlab.rb`: + + ```ruby + gitaly['enable'] = false + ``` + +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). + +**For installations from source** + +1. Edit `/etc/default/gitlab`: + + ```shell + gitaly_enabled=false + ``` + +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). ## TLS support @@ -271,168 +326,128 @@ Gitaly supports TLS encryption. To be able to communicate with a Gitaly instance that listens for secure connections you will need to use `tls://` url scheme in the `gitaly_address` of the corresponding storage entry in the GitLab configuration. -The admin needs to bring their own certificate as we do not provide that automatically. -The certificate to be used needs to be installed on all Gitaly nodes and on all client nodes that communicate with it following procedures described in [GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). +You will need to bring your own certificates as this isn't provided automatically. +The certificate to be used needs to be installed on all Gitaly nodes and on all +client nodes that communicate with it following the procedure described in +[GitLab custom certificate configuration](https://docs.gitlab.com/omnibus/settings/ssl.html#install-custom-public-certificates). -Note that it is possible to configure Gitaly servers with both an +NOTE: **Note:** +It is possible to configure Gitaly servers with both an unencrypted listening address `listen_addr` and an encrypted listening address `tls_listen_addr` at the same time. This allows you to do a gradual transition from unencrypted to encrypted traffic, if necessary. -To observe what type of connections are actually being used in a -production environment you can use the following Prometheus query: - -``` -sum(rate(gitaly_connections_total[5m])) by (type) -``` +To configure Gitaly with TLS: -### Example TLS configuration +**For Omnibus GitLab** -### Omnibus installations: +1. On the client nodes, edit `/etc/gitlab/gitlab.rb`: -#### On client nodes: + ```ruby + git_data_dirs({ + 'default' => { 'gitaly_address' => 'tls://gitaly.internal:9999' }, + 'storage1' => { 'gitaly_address' => 'tls://gitaly.internal:9999' }, + }) -```ruby -# /etc/gitlab/gitlab.rb -git_data_dirs({ - 'default' => { 'gitaly_address' => 'tls://gitaly.internal:9999' }, - 'storage1' => { 'gitaly_address' => 'tls://gitaly.internal:9999' }, -}) + gitlab_rails['gitaly_token'] = 'abc123secret' + ``` -gitlab_rails['gitaly_token'] = 'abc123secret' -``` +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). +1. On the Gitaly server nodes, edit `/etc/gitlab/gitlab.rb`: -#### On Gitaly server nodes: + ```ruby + gitaly['tls_listen_addr'] = "0.0.0.0:9999" + gitaly['certificate_path'] = "path/to/cert.pem" + gitaly['key_path'] = "path/to/key.pem" + ``` -```ruby -gitaly['tls_listen_addr'] = "0.0.0.0:9999" -gitaly['certificate_path'] = "path/to/cert.pem" -gitaly['key_path'] = "path/to/key.pem" -``` +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -### Source installations: +**For installations from source** -#### On client nodes: +1. On the client nodes, edit `/home/git/gitlab/config/gitlab.yml`: -```yaml -# /home/git/gitlab/config/gitlab.yml -gitlab: - repositories: - storages: - default: - path: /mnt/gitlab/default/repositories - gitaly_address: tls://gitaly.internal:9999 - storage1: - path: /mnt/gitlab/storage1/repositories - gitaly_address: tls://gitaly.internal:9999 + ```yaml + gitlab: + repositories: + storages: + default: + gitaly_address: tls://gitaly.internal:9999 + storage1: + gitaly_address: tls://gitaly.internal:9999 - gitaly: - token: 'abc123secret' -``` + gitaly: + token: 'abc123secret' + ``` -#### On Gitaly server nodes: + NOTE: **Note:** + In some cases, you'll have to set `path` for each of the `storages` in the + format `path: /mnt/gitlab/<storage name>/repositories`. -```toml -# /home/git/gitaly/config.toml -tls_listen_addr = '0.0.0.0:9999' +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). +1. On the Gitaly server nodes, edit `/home/git/gitaly/config.toml`: -[tls] -certificate_path = '/path/to/cert.pem' -key_path = '/path/to/key.pem' -``` + ```toml + tls_listen_addr = '0.0.0.0:9999' -## Gitaly-ruby + [tls] + certificate_path = '/path/to/cert.pem' + key_path = '/path/to/key.pem' + ``` -Gitaly was developed to replace Ruby application code in gitlab-ce/ee. -In order to save time and/or avoid the risk of rewriting existing -application logic, in some cases we chose to copy some application code -from gitlab-ce into Gitaly almost as-is. To be able to run that code, we -made gitaly-ruby, which is a sidecar process for the main Gitaly Go -process. Some examples of things that are implemented in gitaly-ruby are -RPC's that deal with wiki's, and RPC's that create commits on behalf of -a user, such as merge commits. - -### Number of gitaly-ruby workers - -Gitaly-ruby has much less capacity than Gitaly itself. If your Gitaly -server has to handle a lot of request, the default setting of having -just 1 active gitaly-ruby sidecar might not be enough. If you see -ResourceExhausted errors from Gitaly it's very likely that you have not -enough gitaly-ruby capacity. - -You can increase the number of gitaly-ruby processes on your Gitaly -server with the following settings. +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). -Omnibus: +To observe what type of connections are actually being used in a +production environment you can use the following Prometheus query: -```ruby -# /etc/gitlab/gitlab.rb -# Default is 2 workers. The minimum is 2; 1 worker is always reserved as -# a passive stand-by. -gitaly['ruby_num_workers'] = 4 ``` - -Source: - -```toml -# /home/git/gitaly/config.toml -[gitaly-ruby] -num_workers = 4 +sum(rate(gitaly_connections_total[5m])) by (type) ``` -### Observing gitaly-ruby traffic - -Gitaly-ruby is a somewhat hidden, internal implementation detail of -Gitaly. There is not that much visibility into what goes on inside -gitaly-ruby processes. +## `gitaly-ruby` -If you have Prometheus set up to scrape your Gitaly process, you can see -request rates and error codes for individual RPC's in gitaly-ruby by -querying `grpc_client_handled_total`. Strictly speaking this metric does -not differentiate between gitaly-ruby and other RPC's, but in practice -(as of GitLab 11.9), all gRPC calls made by Gitaly itself are internal -calls from the main Gitaly process to one of its gitaly-ruby sidecars. +Gitaly was developed to replace the Ruby application code in GitLab. +In order to save time and/or avoid the risk of rewriting existing +application logic, in some cases we chose to copy some application code +from GitLab into Gitaly almost as-is. To be able to run that code, +`gitaly-ruby` was created, which is a "sidecar" process for the main Gitaly Go +process. Some examples of things that are implemented in `gitaly-ruby` are +RPCs that deal with wikis, and RPCs that create commits on behalf of +a user, such as merge commits. -Assuming your `grpc_client_handled_total` counter only observes Gitaly, -the following query shows you RPC's are (most likely) internally -implemented as calls to gitaly-ruby. +### Number of `gitaly-ruby` workers -``` -sum(rate(grpc_client_handled_total[5m])) by (grpc_method) > 0 -``` +`gitaly-ruby` has much less capacity than Gitaly itself. If your Gitaly +server has to handle a lot of requests, the default setting of having +just one active `gitaly-ruby` sidecar might not be enough. If you see +`ResourceExhausted` errors from Gitaly, it's very likely that you have not +enough `gitaly-ruby` capacity. -## Disabling or enabling the Gitaly service in a cluster environment +You can increase the number of `gitaly-ruby` processes on your Gitaly +server with the following settings. -If you are running Gitaly [as a remote -service](#running-gitaly-on-its-own-server) you may want to disable -the local Gitaly service that runs on your GitLab server by default. +**For Omnibus GitLab** -> 'Disabling Gitaly' only makes sense when you run GitLab in a custom -cluster configuration, where different services run on different -machines. Disabling Gitaly on all machines in the cluster is not a -valid configuration. +1. Edit `/etc/gitlab/gitlab.rb`: -If you are setting up a GitLab cluster where Gitaly does not need to -run on all machines, you can disable the Gitaly service in your -Omnibus installation, add the following line to `/etc/gitlab/gitlab.rb`: + ```ruby + # Default is 2 workers. The minimum is 2; 1 worker is always reserved as + # a passive stand-by. + gitaly['ruby_num_workers'] = 4 + ``` -```ruby -gitaly['enable'] = false -``` +1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -When you run `gitlab-ctl reconfigure` the Gitaly service will be -disabled. +**For installations from source** -To disable the Gitaly service in a GitLab cluster where you installed -GitLab from source, add the following to `/etc/default/gitlab` on the -machine where you want to disable Gitaly. +1. Edit `/home/git/gitaly/config.toml`: -```shell -gitaly_enabled=false -``` + ```toml + [gitaly-ruby] + num_workers = 4 + ``` -When you run `service gitlab restart` Gitaly will be disabled on this -particular machine. +1. Save the file and [restart GitLab](../restart_gitlab.md#installations-from-source). ## Eliminating NFS altogether @@ -440,19 +455,23 @@ If you are planning to use Gitaly without NFS for your storage needs and want to eliminate NFS from your environment altogether, there are a few things that you need to do: - 1. Make sure the [`git` user home directory](https://docs.gitlab.com/omnibus/settings/configuration.html#moving-the-home-directory-for-a-user) is on local disk. - 1. Configure [database lookup of SSH keys](../operations/fast_ssh_key_lookup.md) - to eliminate the need for a shared authorized_keys file. - 1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) - including [live tracing](../job_traces.md#new-live-trace-architecture). - 1. Configure [object storage for LFS objects](../../workflow/lfs/lfs_administration.md#storing-lfs-objects-in-remote-object-storage). - 1. Configure [object storage for uploads](../uploads.md#using-object-storage-core-only). - -NOTE: **Note:** One current feature of GitLab still requires a shared directory (NFS): [GitLab Pages](../../user/project/pages/index.md). +1. Make sure the [`git` user home directory](https://docs.gitlab.com/omnibus/settings/configuration.html#moving-the-home-directory-for-a-user) is on local disk. +1. Configure [database lookup of SSH keys](../operations/fast_ssh_key_lookup.md) + to eliminate the need for a shared authorized_keys file. +1. Configure [object storage for job artifacts](../job_artifacts.md#using-object-storage) + including [live tracing](../job_traces.md#new-live-trace-architecture). +1. Configure [object storage for LFS objects](../../workflow/lfs/lfs_administration.md#storing-lfs-objects-in-remote-object-storage). +1. Configure [object storage for uploads](../uploads.md#using-object-storage-core-only). + +NOTE: **Note:** +One current feature of GitLab that still requires a shared directory (NFS) is +[GitLab Pages](../../user/project/pages/index.md). There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) to eliminate the need for NFS to support GitLab Pages. -## Troubleshooting Gitaly in production +## Troubleshooting + +### `gitaly-debug` Since GitLab 11.6, Gitaly comes with a command-line tool called `gitaly-debug` that can be run on a Gitaly server to aid in @@ -462,3 +481,32 @@ Git clone speed for a specific repository on the server. For an up to date list of sub-commands see [the gitaly-debug README](https://gitlab.com/gitlab-org/gitaly/blob/master/cmd/gitaly-debug/README.md). + +### Client side GRPC logs + +Gitaly uses the [gRPC](https://grpc.io/) RPC framework. The Ruby gRPC +client has its own log file which may contain useful information when +you are seeing Gitaly errors. You can control the log level of the +gRPC client with the `GRPC_LOG_LEVEL` environment variable. The +default level is `WARN`. + +### Observing `gitaly-ruby` traffic + +[`gitaly-ruby`](#gitaly-ruby) is an internal implementation detail of Gitaly, +so, there's not that much visibility into what goes on inside +`gitaly-ruby` processes. + +If you have Prometheus set up to scrape your Gitaly process, you can see +request rates and error codes for individual RPCs in `gitaly-ruby` by +querying `grpc_client_handled_total`. Strictly speaking, this metric does +not differentiate between `gitaly-ruby` and other RPCs, but in practice +(as of GitLab 11.9), all gRPC calls made by Gitaly itself are internal +calls from the main Gitaly process to one of its `gitaly-ruby` sidecars. + +Assuming your `grpc_client_handled_total` counter only observes Gitaly, +the following query shows you RPCs are (most likely) internally +implemented as calls to `gitaly-ruby`: + +``` +sum(rate(grpc_client_handled_total[5m])) by (grpc_method) > 0 +``` diff --git a/doc/administration/job_artifacts.md b/doc/administration/job_artifacts.md index 5490a59ceac..2b624f8de77 100644 --- a/doc/administration/job_artifacts.md +++ b/doc/administration/job_artifacts.md @@ -115,6 +115,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a | `aws_access_key_id` | AWS credentials, or compatible | | | `aws_secret_access_key` | AWS credentials, or compatible | | | `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | +| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with AWS v4 signatures (https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | true | `region` | AWS region | us-east-1 | | `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | | `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | diff --git a/doc/administration/merge_request_diffs.md b/doc/administration/merge_request_diffs.md index 99cd9051778..0b065082ded 100644 --- a/doc/administration/merge_request_diffs.md +++ b/doc/administration/merge_request_diffs.md @@ -90,6 +90,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a | `aws_access_key_id` | AWS credentials, or compatible | | | `aws_secret_access_key` | AWS credentials, or compatible | | | `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | +| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with AWS v4 signatures (https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | true | `region` | AWS region | us-east-1 | | `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | | `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | diff --git a/doc/administration/monitoring/performance/img/request_profile_result.png b/doc/administration/monitoring/performance/img/request_profile_result.png Binary files differindex 1b06e240fa0..3b34f207974 100644 --- a/doc/administration/monitoring/performance/img/request_profile_result.png +++ b/doc/administration/monitoring/performance/img/request_profile_result.png diff --git a/doc/administration/monitoring/performance/request_profiling.md b/doc/administration/monitoring/performance/request_profiling.md index 726882fbb87..9f671e0db11 100644 --- a/doc/administration/monitoring/performance/request_profiling.md +++ b/doc/administration/monitoring/performance/request_profiling.md @@ -5,9 +5,9 @@ 1. Grab the profiling token from **Monitoring > Requests Profiles** admin page (highlighted in a blue in the image below).  -1. Pass the header `X-Profile-Token: <token>` to the request you want to profile. You can use: +1. Pass the header `X-Profile-Token: <token>` and `X-Profile-Mode: <mode>`(where <mode> can be `execution` or `memory`) to the request you want to profile. You can use: - Browser extensions. For example, [ModHeader](https://chrome.google.com/webstore/detail/modheader/idgpnmonknjnojddfkpgkljpfnnfcklj) Chrome extension. - - `curl`. For example, `curl --header 'X-Profile-Token: <token>' https://gitlab.example.com/group/project`. + - `curl`. For example, `curl --header 'X-Profile-Token: <token>' --header 'X-Profile-Mode: <mode>' https://gitlab.example.com/group/project`. 1. Once request is finished (which will take a little longer than usual), you can view the profiling output from **Monitoring > Requests Profiles** admin page.  diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 89501f20d99..054fa547704 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -34,6 +34,9 @@ The following metrics are available: | filesystem_writable | Gauge | 9.4 | Whether or not the filesystem is writable | | filesystem_read_latency_seconds | Gauge | 9.4 | Read latency of a specific filesystem | | filesystem_readable | Gauge | 9.4 | Whether or not the filesystem is readable | +| gitlab_cache_misses_total | Counter | 10.2 | Cache read miss | +| gitlab_cache_operation_duration_seconds | Histogram | 10.2 | Cache access time | +| gitlab_cache_operations_total | Counter | 12.2 | Cache operations by controller/action | | http_requests_total | Counter | 9.4 | Rack request count | | http_request_duration_seconds | Histogram | 9.4 | HTTP response time from rack middleware | | pipelines_created_total | Counter | 9.4 | Counter of pipelines created | diff --git a/doc/administration/repository_storage_types.md b/doc/administration/repository_storage_types.md index 9dea6074a3f..1669f8b128c 100644 --- a/doc/administration/repository_storage_types.md +++ b/doc/administration/repository_storage_types.md @@ -105,7 +105,7 @@ question. To start a migration, enable Hashed Storage for new projects: 1. Go to **Admin > Settings > Repository** and expand the **Repository Storage** section. -2. Select the **Use hashed storage paths for newly created and renamed projects** checkbox. +1. Select the **Use hashed storage paths for newly created and renamed projects** checkbox. Check if the change breaks any existing integration you may have that either runs on the same machine as your repositories are located, or may login to that machine @@ -133,7 +133,7 @@ Similar to the migration, to disable Hashed Storage for new projects: 1. Go to **Admin > Settings > Repository** and expand the **Repository Storage** section. -2. Uncheck the **Use hashed storage paths for newly created and renamed projects** checkbox. +1. Uncheck the **Use hashed storage paths for newly created and renamed projects** checkbox. To schedule a complete rollback, see the [rake task documentation for storage rollback](raketasks/storage.md#rollback-from-hashed-storage-to-legacy-storage) for instructions. diff --git a/doc/administration/uploads.md b/doc/administration/uploads.md index 51e267c865e..99f1c386183 100644 --- a/doc/administration/uploads.md +++ b/doc/administration/uploads.md @@ -78,6 +78,7 @@ The connection settings match those provided by [Fog](https://github.com/fog), a | `aws_access_key_id` | AWS credentials, or compatible | | | `aws_secret_access_key` | AWS credentials, or compatible | | | `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | +| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with AWS v4 signatures (https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | true | `region` | AWS region | us-east-1 | | `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | | `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | @@ -140,6 +141,22 @@ _The uploads are stored by default in 1. Save the file and [restart GitLab][] for the changes to take effect. 1. Migrate any existing local uploads to the object storage using [`gitlab:uploads:migrate` rake task](raketasks/uploads/migrate.md). +### Oracle Cloud S3 connection settings + +Note that Oracle Cloud S3 must be sure to use the following settings: + +| Setting | Value | +|---------|-------| +| `enable_signature_v4_streaming` | false | +| `path_style` | true | + +If `enable_signature_v4_streaming` is set to `true`, you may see the +following error: + +``` +STREAMING-AWS4-HMAC-SHA256-PAYLOAD is not supported +``` + ### OpenStack compatible connection settings The connection settings match those provided by [Fog](https://github.com/fog), and are as follows: diff --git a/doc/api/commits.md b/doc/api/commits.md index 6eb4c47415f..1a835c0a872 100644 --- a/doc/api/commits.md +++ b/doc/api/commits.md @@ -72,15 +72,16 @@ POST /projects/:id/repository/commits | Attribute | Type | Required | Description | | --------- | ---- | -------- | ----------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) | -| `branch` | string | yes | Name of the branch to commit into. To create a new branch, also provide `start_branch`. | +| `branch` | string | yes | Name of the branch to commit into. To create a new branch, also provide either `start_branch` or `start_sha`, and optionally `start_project`. | | `commit_message` | string | yes | Commit message | -| `start_branch` | string | no | Name of the branch to start the new commit from | -| `start_project` | integer/string | no | The project ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) to start the commit from. Defaults to the value of `id`. | +| `start_branch` | string | no | Name of the branch to start the new branch from | +| `start_sha` | string | no | SHA of the commit to start the new branch from | +| `start_project` | integer/string | no | The project ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) to start the new branch from. Defaults to the value of `id`. | | `actions[]` | array | yes | An array of action hashes to commit as a batch. See the next table for what attributes it can take. | | `author_email` | string | no | Specify the commit author's email address | | `author_name` | string | no | Specify the commit author's name | | `stats` | boolean | no | Include commit stats. Default is true | -| `force` | boolean | no | When `true` overwrites the target branch with a new commit based on the `start_branch` | +| `force` | boolean | no | When `true` overwrites the target branch with a new commit based on the `start_branch` or `start_sha` | | `actions[]` Attribute | Type | Required | Description | | --------------------- | ---- | -------- | ----------- | diff --git a/doc/api/container_registry.md b/doc/api/container_registry.md index 64ea15bca93..174b93a4f7a 100644 --- a/doc/api/container_registry.md +++ b/doc/api/container_registry.md @@ -193,13 +193,13 @@ Examples: curl --request DELETE --data 'name_regex=[0-9a-z]{40}' --data 'keep_n=5' --data 'older_than=2d' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" ``` -2. Remove all tags, but keep always the latest 5: +1. Remove all tags, but keep always the latest 5: ```bash curl --request DELETE --data 'name_regex=.*' --data 'keep_n=5' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" ``` -3. Remove all tags that are older than 1 month: +1. Remove all tags that are older than 1 month: ```bash curl --request DELETE --data 'name_regex=.*' --data 'older_than=1month' --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/5/registry/repositories/2/tags" diff --git a/doc/api/lint.md b/doc/api/lint.md index b9b49f3df27..79f5e629c7f 100644 --- a/doc/api/lint.md +++ b/doc/api/lint.md @@ -5,7 +5,7 @@ Checks if your `.gitlab-ci.yml` file is valid. ``` -POST /lint +POST /ci/lint ``` | Attribute | Type | Required | Description | diff --git a/doc/api/users.md b/doc/api/users.md index fdc84826680..b41fd106fc5 100644 --- a/doc/api/users.md +++ b/doc/api/users.md @@ -349,6 +349,9 @@ Note that `force_random_password` and `reset_password` take priority over `password`. In addition, `reset_password` and `force_random_password` can be used together. +NOTE: **Note:** +From [GitLab 12.1](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/29888/), `private_profile` will default to `false`. + ``` POST /users ``` diff --git a/doc/ci/examples/end_to_end_testing_webdriverio/index.md b/doc/ci/examples/end_to_end_testing_webdriverio/index.md index 38fcf05f519..38d0d86ffa2 100644 --- a/doc/ci/examples/end_to_end_testing_webdriverio/index.md +++ b/doc/ci/examples/end_to_end_testing_webdriverio/index.md @@ -143,7 +143,7 @@ Which brings us to the exciting part: how do we run this in GitLab CI/CD? There need to do for this: 1. Set up [CI/CD jobs](../../yaml/README.md#introduction) that actually have a browser available. -2. Update our WebdriverIO configuration to use those browsers to visit the review apps. +1. Update our WebdriverIO configuration to use those browsers to visit the review apps. For the scope of this article, we've defined an additional [CI/CD stage](../../yaml/README.md#stages) `confidence-check` that is executed _after_ the stage that deploys the review app. It uses the `node:latest` [Docker diff --git a/doc/ci/introduction/index.md b/doc/ci/introduction/index.md index fc89f0fc94f..6401ff90a0a 100644 --- a/doc/ci/introduction/index.md +++ b/doc/ci/introduction/index.md @@ -146,14 +146,14 @@ so, GitLab CI/CD: - Runs automated scripts (sequential or parallel) to: - Build and test your app. - Preview the changes per merge request with Review Apps, as you - would see in your `localhost`. + would see in your `localhost`. Once you're happy with your implementation: - Get your code reviewed and approved. - Merge the feature branch into the default branch. - GitLab CI/CD deploys your changes automatically to a production environment. -- And finally, you and your team can easily roll it back if something goes wrong. +- And finally, you and your team can easily roll it back if something goes wrong.  @@ -176,24 +176,24 @@ you'll see some of the features available in GitLab according to each stage (Verify, Package, Release). 1. **Verify**: - - Automatically build and test your application with Continuous Integration. - - Analyze your source code quality with [GitLab Code Quality](../../user/project/merge_requests/code_quality.md). **(STARTER)** - - Determine the performance impact of code changes with [Browser Performance Testing](../../user/project/merge_requests/browser_performance_testing.md). **(PREMIUM)** - - Perform a series of tests, such as [Container Scanning](../../user/application_security/container_scanning/index.md) **(ULTIMATE)**, [Dependency Scanning](../../user/application_security/dependency_scanning/index.md) **(ULTIMATE)**, and [JUnit tests](../junit_test_reports.md). - - Deploy your changes with [Review Apps](../review_apps/index.md) to preview the app changes on every branch. + - Automatically build and test your application with Continuous Integration. + - Analyze your source code quality with [GitLab Code Quality](../../user/project/merge_requests/code_quality.md). **(STARTER)** + - Determine the performance impact of code changes with [Browser Performance Testing](../../user/project/merge_requests/browser_performance_testing.md). **(PREMIUM)** + - Perform a series of tests, such as [Container Scanning](../../user/application_security/container_scanning/index.md) **(ULTIMATE)**, [Dependency Scanning](../../user/application_security/dependency_scanning/index.md) **(ULTIMATE)**, and [JUnit tests](../junit_test_reports.md). + - Deploy your changes with [Review Apps](../review_apps/index.md) to preview the app changes on every branch. 1. **Package**: - - Store Docker images with [Container Registry](../../user/project/container_registry.md). - - Store NPM packages with [NPM Registry](../../user/project/packages/npm_registry.md). **(PREMIUM)** - - Store Maven artifacts with [Maven Repository](../../user/project/packages/maven_repository.md). **(PREMIUM)** + - Store Docker images with [Container Registry](../../user/project/container_registry.md). + - Store NPM packages with [NPM Registry](../../user/project/packages/npm_registry.md). **(PREMIUM)** + - Store Maven artifacts with [Maven Repository](../../user/project/packages/maven_repository.md). **(PREMIUM)** 1. **Release**: - - Continuous Deployment, automatically deploying your app to production. - - Continuous Delivery, manually click to deploy your app to production. - - Deploy static websites with [GitLab Pages](../../user/project/pages/index.md). - - Ship features to only a portion of your pods and let a percentage of your user base to visit the temporarily deployed feature with [Canary Deployments](../../user/project/canary_deployments.md). **(PREMIUM)** - - Deploy your features behind [Feature Flags](../../user/project/operations/feature_flags.md). **(PREMIUM)** - - Add release notes to any Git tag with [GitLab Releases](../../user/project/releases/index.md). - - View of the current health and status of each CI environment running on Kubernetes with [Deploy Boards](../../user/project/deploy_boards.md). **(PREMIUM)** - - Deploy your application to a production environment in a Kubernetes cluster with [Auto Deploy](../../topics/autodevops/index.md#auto-deploy). + - Continuous Deployment, automatically deploying your app to production. + - Continuous Delivery, manually click to deploy your app to production. + - Deploy static websites with [GitLab Pages](../../user/project/pages/index.md). + - Ship features to only a portion of your pods and let a percentage of your user base to visit the temporarily deployed feature with [Canary Deployments](../../user/project/canary_deployments.md). **(PREMIUM)** + - Deploy your features behind [Feature Flags](../../user/project/operations/feature_flags.md). **(PREMIUM)** + - Add release notes to any Git tag with [GitLab Releases](../../user/project/releases/index.md). + - View of the current health and status of each CI environment running on Kubernetes with [Deploy Boards](../../user/project/deploy_boards.md). **(PREMIUM)** + - Deploy your application to a production environment in a Kubernetes cluster with [Auto Deploy](../../topics/autodevops/index.md#auto-deploy). With GitLab CI/CD you can also: diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md index a13857bee25..f63b17a9e5a 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/index.md @@ -6,7 +6,6 @@ last_update: 2019-07-03 # Pipelines for Merged Results **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/7380) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.10. -> This feature is disabled by default until we resolve issues with [contention handling](https://gitlab.com/gitlab-org/gitlab-ee/issues/11222), but [can be enabled manually](#enabling-pipelines-for-merged-results). It's possible for your source and target branches to diverge, which can result in the scenario that source branch's pipeline was green, the target's pipeline was green, @@ -25,8 +24,10 @@ new ref and a pipeline for this ref validates the result prior to merging. There are some cases where creating a combined ref is not possible or not wanted. For example, a source branch that has conflicts with the target branch -or a merge request that is still in WIP status. In this case, the merge request pipeline falls back to a "detached" state -and runs on the source branch ref as if it was a regular pipeline. +or a merge request that is still in WIP status. In this case, +GitLab doesn't create a merge commit and the pipeline runs on source branch, instead, +which is a default behavior of [Pipelines for merge requests](../index.md) + i.e. `detached` label is shown to the pipelines. The detached state serves to warn you that you are working in a situation subjected to merge problems, and helps to highlight that you should @@ -42,7 +43,7 @@ Pipelines for merged results require: In addition, pipelines for merged results have the following limitations: - Forking/cross-repo workflows are not currently supported. To follow progress, - see [#9713](https://gitlab.com/gitlab-org/gitlab-ee/issues/9713). + see [#11934](https://gitlab.com/gitlab-org/gitlab-ee/issues/11934). - This feature is not available for [fast forward merges](../../../user/project/merge_requests/fast_forward_merge.md) yet. To follow progress, see [#58226](https://gitlab.com/gitlab-org/gitlab-ce/issues/58226). diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_position_v12_0.png b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_position_v12_0.png Binary files differnew file mode 100644 index 00000000000..70916bc0e00 --- /dev/null +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/img/merge_train_position_v12_0.png diff --git a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md index 57358434c02..44cbcde264c 100644 --- a/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md +++ b/doc/ci/merge_request_pipelines/pipelines_for_merged_results/merge_trains/index.md @@ -6,7 +6,6 @@ last_update: 2019-07-03 # Merge Trains **(PREMIUM)** > [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9186) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.0. -> This feature is disabled by default, but [can be enabled manually](#enabling-merge-trains). [Pipelines for merged results](../index.md#pipelines-for-merged-results-premium) introduces running a build on the result of the merged code prior to merging, as a way to keep master green. @@ -33,35 +32,40 @@ Merge trains have the following requirements and limitations: - This feature requires that [pipelines for merged results](../index.md#pipelines-for-merged-results-premium) are **configured properly**. -- Each merge train can generate a merge ref and run a pipeline **one at a time**. - We plan to make the pipelines for merged results - [run in parallel](https://gitlab.com/gitlab-org/gitlab-ee/issues/11222) in a - future release. +- Each merge train can run a maximum of **four** pipelines in parallel. + If more than four merge requests are added to the merge train, the merge requests + will be queued until a slot in the merge train is free. There is no limit to the + number of merge requests that can be queued. +- This feature does not support [squash and merge](../../../../user/project/merge_requests/squash_and_merge.md). -## Enabling Merge Trains - -To enable merge trains at the project level: - -1. Visit your project's **Settings > General** and expand **Merge requests**. -1. Check **Allow merge trains**. -1. Click **Save changes** button. - - +<i class="fa fa-youtube-play youtube" aria-hidden="true"></i> +Watch this video for a demonstration on [how parallel execution +of Merge Trains can prevent commits from breaking the default +branch](https://www.youtube.com/watch?v=D4qCqXgZkHQ). ## How to add a merge request to a merge train To add a merge request to a merge train: -1. Click "Start/Add merge train" button in a merge request +1. Visit a merge request. +1. Click the **Start/Add to merge train** button.  ## How to remove a merge request from a merge train -1. Click "Remove from merge train" button in the merge request widget. +1. Visit a merge request. +1. Click the **Remove from merge train** button.  +## How to view a merge request's current position on the merge train + +After a merge request has been added to the merge train, the merge request's +current position will be displayed under the pipeline widget: + + + ## Start/Add to merge train when pipeline succeeds You can add a merge request to a merge train only when the latest pipeline in the @@ -70,7 +74,8 @@ the merge request to a train because the current change of the merge request may be broken thus it could affect the following merge requests. In this case, you can schedule to add the merge request to a merge train **when the latest -pipeline succeeds**. You can see the following button instead of the regular "Start/Add merge train" +pipeline succeeds** (This pipeline is [Pipelines for merged results](../index.md), not Pipelines for merge train). +You can see the following button instead of the regular **Start/Add to merge train** button while the latest pipeline is running.  diff --git a/doc/development/README.md b/doc/development/README.md index a74770ae383..ea5d9e10e2c 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -17,6 +17,7 @@ description: 'Learn how to contribute to GitLab.' - [GitLab core team & GitLab Inc. contribution process](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/PROCESS.md) - [Generate a changelog entry with `bin/changelog`](changelog.md) - [Code review guidelines](code_review.md) for reviewing code and having code reviewed +- [Database review guidelines](database_review.md) for reviewing database-related changes and complex SQL queries - [Automatic CE->EE merge](automatic_ce_ee_merge.md) - [Guidelines for implementing Enterprise Edition features](ee_features.md) - [Security process for developers](https://gitlab.com/gitlab-org/release/docs/blob/master/general/security/developer.md#security-releases-critical-non-critical-as-a-developer) @@ -63,6 +64,7 @@ description: 'Learn how to contribute to GitLab.' - [Routing](routing.md) - [Repository mirroring](repository_mirroring.md) - [Git LFS](lfs.md) +- [Developing against interacting components or features](interacting_components.md) ## Performance guides diff --git a/doc/development/chaos_endpoints.md b/doc/development/chaos_endpoints.md index b3406275937..eb6dde2d24e 100644 --- a/doc/development/chaos_endpoints.md +++ b/doc/development/chaos_endpoints.md @@ -36,6 +36,10 @@ Replace `secret` with your own secret token. Once you have enabled the chaos endpoints and restarted the application, you can start testing using the endpoints. +By default, when invoking a chaos endpoint, the web worker process which receives the request will handle it. This means, for example, that if the Kill +operation is invoked, the Puma or Unicorn worker process handling the request will be killed. To test these operations in Sidekiq, the `async` parameter on +each endpoint can be set to `true`. This will run the chaos process in a Sidekiq worker. + ## Memory leaks To simulate a memory leak in your application, use the `/-/chaos/leakmem` endpoint. @@ -47,12 +51,14 @@ The memory is not retained after the request finishes. Once the request has comp GET /-/chaos/leakmem GET /-/chaos/leakmem?memory_mb=1024 GET /-/chaos/leakmem?memory_mb=1024&duration_s=50 +GET /-/chaos/leakmem?memory_mb=1024&duration_s=50&async=true ``` -| Attribute | Type | Required | Description | -| ------------ | ------- | -------- | ---------------------------------------------------------------------------------- | -| `memory_mb` | integer | no | How much memory, in MB, should be leaked. Defaults to 100MB. | +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | ------------------------------------------------------------------------------------ | +| `memory_mb` | integer | no | How much memory, in MB, should be leaked. Defaults to 100MB. | | `duration_s` | integer | no | Minimum duration_s, in seconds, that the memory should be retained. Defaults to 30s. | +| `async` | boolean | no | Set to true to leak memory in a Sidekiq background worker process | ```bash curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024&duration_s=10 --header 'X-Chaos-Secret: secret' @@ -69,11 +75,13 @@ If you're using Unicorn, this is done by killing the worker process. ``` GET /-/chaos/cpu_spin GET /-/chaos/cpu_spin?duration_s=50 +GET /-/chaos/cpu_spin?duration_s=50&async=true ``` | Attribute | Type | Required | Description | | ------------ | ------- | -------- | --------------------------------------------------------------------- | | `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | +| `async` | boolean | no | Set to true to consume CPU in a Sidekiq background worker process | ```bash curl http://localhost:3000/-/chaos/cpu_spin?duration_s=60 --header 'X-Chaos-Secret: secret' @@ -91,12 +99,14 @@ If you're using Unicorn, this is done by killing the worker process. ``` GET /-/chaos/db_spin GET /-/chaos/db_spin?duration_s=50 +GET /-/chaos/db_spin?duration_s=50&async=true ``` -| Attribute | Type | Required | Description | -| ------------ | ------- | -------- | --------------------------------------------------------------------- | -| `interval_s` | float | no | Interval, in seconds, for every DB request. Defaults to 1s | -| `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | --------------------------------------------------------------------------- | +| `interval_s` | float | no | Interval, in seconds, for every DB request. Defaults to 1s | +| `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | +| `async` | boolean | no | Set to true to perform the operation in a Sidekiq background worker process | ```bash curl http://localhost:3000/-/chaos/db_spin?interval_s=1&duration_s=60 --header 'X-Chaos-Secret: secret' @@ -112,11 +122,13 @@ As with the CPU Spin endpoint, this may lead to your request timing out if durat ``` GET /-/chaos/sleep GET /-/chaos/sleep?duration_s=50 +GET /-/chaos/sleep?duration_s=50&async=true ``` | Attribute | Type | Required | Description | | ------------ | ------- | -------- | ---------------------------------------------------------------------- | | `duration_s` | integer | no | Duration, in seconds, that the request will sleep for. Defaults to 30s | +| `async` | boolean | no | Set to true to sleep in a Sidekiq background worker process | ```bash curl http://localhost:3000/-/chaos/sleep?duration_s=60 --header 'X-Chaos-Secret: secret' @@ -132,8 +144,13 @@ Since this endpoint uses the `KILL` signal, the worker is not given a chance to ``` GET /-/chaos/kill +GET /-/chaos/kill?async=true ``` +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | ---------------------------------------------------------------------- | +| `async` | boolean | no | Set to true to kill a Sidekiq background worker process | + ```bash curl http://localhost:3000/-/chaos/kill --header 'X-Chaos-Secret: secret' curl http://localhost:3000/-/chaos/kill?token=secret diff --git a/doc/development/code_review.md b/doc/development/code_review.md index e9c8b193309..709cd0c806b 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -62,6 +62,7 @@ from teams other than your own. **approved by a [backend maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab-ce_maintainers_backend)**. 1. If your merge request includes database migrations or changes to expensive queries [^2], it must be **approved by a [database maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab-ce_maintainers_database)**. + Read the [database review guidelines](database_review.md) for more details. 1. If your merge request includes frontend changes [^1], it must be **approved by a [frontend maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab-ce_maintainers_frontend)**. 1. If your merge request includes UX changes [^1], it must be @@ -341,8 +342,7 @@ Enterprise Edition instance. This has some implications: - [Background migrations](background_migrations.md) run in Sidekiq, and should only be done for migrations that would take an extreme amount of time at GitLab.com scale. -1. **Sidekiq workers** - [cannot change in a backwards-incompatible way](sidekiq_style_guide.md#removing-or-renaming-queues): +1. **Sidekiq workers** [cannot change in a backwards-incompatible way](sidekiq_style_guide.md#removing-or-renaming-queues): 1. Sidekiq queues are not drained before a deploy happens, so there will be workers in the queue from the previous version of GitLab. 1. If you need to change a method signature, try to do so across two releases, diff --git a/doc/development/contributing/merge_request_workflow.md b/doc/development/contributing/merge_request_workflow.md index 3325d3e074e..17328762c5b 100644 --- a/doc/development/contributing/merge_request_workflow.md +++ b/doc/development/contributing/merge_request_workflow.md @@ -103,7 +103,8 @@ If you would like quick feedback on your merge request feel free to mention some from the [core team](https://about.gitlab.com/community/core-team/) or one of the [merge request coaches](https://about.gitlab.com/team/). When having your code reviewed and when reviewing merge requests, please keep the [code review guidelines](../code_review.md) -in mind. +in mind. And if your code also makes changes to the database, or does expensive queries, +check the [database review guidelines](../database_review.md). ### Keep it simple diff --git a/doc/development/database_review.md b/doc/development/database_review.md new file mode 100644 index 00000000000..1413c2f69fb --- /dev/null +++ b/doc/development/database_review.md @@ -0,0 +1,101 @@ +# Database Review Guidelines + +This page is specific to database reviews. Please refer to our +[code review guide](code_review.md) for broader advice and best +practices for code review in general. + +## General process + +A database review is required for: + +- Changes that touch the database schema or perform data migrations, + including files in: + - `db/` + - `lib/gitlab/background_migration/` +- Changes to the database tooling, e.g.: + - migration or ActiveRecord helpers in `lib/gitlab/database/` + - load balancing +- Changes that produce SQL queries that are beyond the obvious. It is + generally up to the author of a merge request to decide whether or + not complex queries are being introduced and if they require a + database review. + +A database reviewer is expected to look out for obviously complex +queries in the change and review those closer. If the author does not +point out specific queries for review and there are no obviously +complex queries, it is enough to concentrate on reviewing the +migration only. + +It is preferable to review queries in SQL form and generally accepted +to ask the author to translate any ActiveRecord queries in SQL form +for review. + +### Roles and process + +A Merge Request author's role is to: + +- Decide whether a database review is needed. +- If database review is needed, add the ~database label. +- Use the [database changes](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.gitlab/merge_request_templates/Database%20changes.md) + merge request template, or include the appropriate items in the MR description. + +A database **reviewer**'s role is to: + +- Perform a first-pass review on the MR and suggest improvements to the author. +- Once satisfied, relabel the MR with ~"database::reviewed", approve it, and + reassign MR to the database **maintainer** suggested by Reviewer + Roulette. + +A database **maintainer**'s role is to: + +- Perform the final database review on the MR. +- Discuss further improvements or other relevant changes with the + database reviewer and the MR author. +- Finally approve the MR and relabel the MR with ~"database::approved" +- Merge the MR if no other approvals are pending or pass it on to + other maintainers as required (frontend, backend, docs). + +### Distributing review workload + +Review workload is distributed using [reviewer roulette](code_review.md#reviewer-roulette) +([example](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/25181#note_147551725)). +The MR author should then co-assign the suggested database +**reviewer**. When they give their sign-off, they will hand over to +the suggested database **maintainer**. + +If reviewer roulette didn't suggest a database reviewer & maintainer, +make sure you have applied the ~database label and rerun the +`danger-review` CI job, or pick someone from the +[`@gl-database` team](https://gitlab.com/groups/gl-database/-/group_members). + +### How to review for database + +- Check migrations + - Review relational modeling and design choices + - Review migrations follow [database migration style guide](migration_style_guide.md), + for example + - [Check ordering of columns](ordering_table_columns.md) + - [Check indexes are present for foreign keys](migration_style_guide.md#adding-foreign-key-constraints) + - Ensure that migrations execute in a transaction or only contain + concurrent index/foreign key helpers (with transactions disabled) + - Check consistency with `db/schema.rb` and that migrations are [reversible](migration_style_guide.md#reversibility) + - For data migrations, establish a time estimate for execution +- Check post deploy migration + - Make sure we can expect post deploy migrations to finish within 1 hour max. +- Check background migrations + - Review queries (for example, make sure batch sizes are fine) + - Establish a time estimate for execution +- Query performance + - Check for any obviously complex queries and queries the author specifically + points out for review (if any) + - If not present yet, ask the author to provide SQL queries and query plans + (e.g. by using [chatops](understanding_explain_plans.md#chatops) or direct + database access) + - For given queries, review parameters regarding data distribution + - [Check query plans](understanding_explain_plans.md) and suggest improvements + to queries (changing the query, schema or adding indexes and similar) + - General guideline is for queries to come in below 100ms execution time + - If queries rely on prior migrations that are not present yet on production + (eg indexes, columns), you can use a [one-off instance from the restore + pipeline](https://ops.gitlab.net/gitlab-com/gl-infra/gitlab-restore/postgres-gprd) + in order to establish a proper testing environment. diff --git a/doc/development/documentation/improvement-workflow.md b/doc/development/documentation/improvement-workflow.md index a12c3d5ea7b..80fbd4b6427 100644 --- a/doc/development/documentation/improvement-workflow.md +++ b/doc/development/documentation/improvement-workflow.md @@ -52,9 +52,9 @@ To request a post-merge review, [create an issue for one using the Doc Review te **3. Maintainer** 1. Review by assigned maintainer, who can always request/require the above reviews. Maintainer review can occur before or after a technical writer review. -2. Ensure a release milestone of the format XX.Y is set. If the freeze for that release has begun, add the label `pick into <XX.Y>` unless this change is not required for the release. In that case, simply change the milestone. -3. If EE and CE MRs exist, merge the EE MR first, then the CE MR. -4. After merging, if there has not been a technical writer review and an issue for a follow-up review was not already created and linked from the MR, [create the issue using the Doc Review template](https://gitlab.com/gitlab-org/gitlab-ce/issues/new?issuable_template=Doc%20Review) and link it from the MR. +1. Ensure a release milestone of the format XX.Y is set. If the freeze for that release has begun, add the label `pick into <XX.Y>` unless this change is not required for the release. In that case, simply change the milestone. +1. If EE and CE MRs exist, merge the EE MR first, then the CE MR. +1. After merging, if there has not been a technical writer review and an issue for a follow-up review was not already created and linked from the MR, [create the issue using the Doc Review template](https://gitlab.com/gitlab-org/gitlab-ce/issues/new?issuable_template=Doc%20Review) and link it from the MR. ## Other ways to help diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index 36a2f47a55b..8ce855594df 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -488,15 +488,17 @@ Currently, the following tests are in place: that all cURL examples in API docs use the full switches. It's recommended to [check locally](#previewing-the-changes-live) before pushing to GitLab by executing the command `bundle exec nanoc check internal_links` on your local - [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs) directory. + [`gitlab-docs`](https://gitlab.com/gitlab-org/gitlab-docs) directory. In addition, + `docs-lint` also runs [markdownlint](styleguide.md#markdown-rules) to ensure the + markdown is consistent across all documentation. 1. [`ee_compat_check`](../automatic_ce_ee_merge.md#avoiding-ce-ee-merge-conflicts-beforehand) (runs on CE only): - When you submit a merge request to GitLab Community Edition (CE), - there is this additional job that runs against Enterprise Edition (EE) - and checks if your changes can apply cleanly to the EE codebase. - If that job fails, read the instructions in the job log for what to do next. - As CE is merged into EE once a day, it's important to avoid merge conflicts. - Submitting an EE-equivalent merge request cherry-picking all commits from CE to EE is - essential to avoid them. + When you submit a merge request to GitLab Community Edition (CE), + there is this additional job that runs against Enterprise Edition (EE) + and checks if your changes can apply cleanly to the EE codebase. + If that job fails, read the instructions in the job log for what to do next. + As CE is merged into EE once a day, it's important to avoid merge conflicts. + Submitting an EE-equivalent merge request cherry-picking all commits from CE to EE is + essential to avoid them. 1. [`ee-files-location-check`/`ee-specific-lines-check`](#ee-specific-lines-check) (runs on EE only): This test ensures that no new files/directories are created/changed in EE. All docs should be submitted in CE instead, regardless the tier they are on. @@ -559,15 +561,16 @@ A file with `proselint` configuration must be placed in a #### `markdownlint` `markdownlint` checks that certain rules ([example](https://github.com/DavidAnson/markdownlint/blob/master/README.md#rules--aliases)) - are followed for Markdown syntax. - Our [Documentation Style Guide](styleguide.md) and [Markdown Guide](https://about.gitlab.com/handbook/product/technical-writing/markdown-guide/) - elaborate on which choices must be made when selecting Markdown syntax for - GitLab documentation. This tool helps catch deviations from those guidelines. +are followed for Markdown syntax. Our [Documentation Style Guide](styleguide.md) and +[Markdown Guide](https://about.gitlab.com/handbook/product/technical-writing/markdown-guide/) +elaborate on which choices must be made when selecting Markdown syntax for GitLab +documentation. This tool helps catch deviations from those guidelines, and matches the +tests run on the documentation by [`docs-lint`](#testing). `markdownlint` can be used [on the command line](https://github.com/igorshubovych/markdownlint-cli#markdownlint-cli--), - either on a single Markdown file or on all Markdown files in a project. For example, to run - `markdownlint` on all documentation in the [`gitlab-ce` project](https://gitlab.com/gitlab-org/gitlab-ce), - run the following commands from within the `gitlab-ce` project: +either on a single Markdown file or on all Markdown files in a project. For example, to run +`markdownlint` on all documentation in the [`gitlab-ce` project](https://gitlab.com/gitlab-org/gitlab-ce), +run the following commands from within the `gitlab-ce` project: ```sh cd doc @@ -597,7 +600,7 @@ The following sample `markdownlint` configuration modifies the available default "line-length": false, "no-trailing-punctuation": false, "ol-prefix": { "style": "one" }, - "blanks-around-fences": false, + "blanks-around-fences": true, "no-inline-html": { "allowed_elements": [ "table", @@ -612,11 +615,15 @@ The following sample `markdownlint` configuration modifies the available default "a", "strong", "i", - "div" + "div", + "b" ] }, "hr-style": { "style": "---" }, - "fenced-code-language": false + "code-block-style": { "style": "fenced" }, + "fenced-code-language": false, + "no-duplicate-header": { "allow_different_nesting": true }, + "commands-show-output": false } ``` diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 1ca965f8ae9..36ffc02644e 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -44,9 +44,9 @@ Include any media types/sources if the content is relevant to readers. You can f In the software industry, it is a best practice to organize documentatioin in different types. For example, [Divio recommends](https://www.divio.com/blog/documentation/): 1. Tutorials -2. How-to guides -3. Explanation -4. Reference (for example, a glossary) +1. How-to guides +1. Explanation +1. Reference (for example, a glossary) At GitLab, we have so many product changes in our monthly releases that we can't afford to continually update multiple types of information. If we have multiple types, the information will become outdated. Therefore, we have a [single template](structure.md) for documentation. @@ -107,6 +107,22 @@ Hard-coded HTML is valid, although it's discouraged to be used while we have `/h - Special styling is required. - Reviewed and approved by a technical writer. +### Markdown Rules + +GitLab ensures that the Markdown used across all documentation is consistent, as +well as easy to review and maintain, by testing documentation changes with +[Markdownlint (mdl)](https://github.com/markdownlint/markdownlint). This lint test +checks many common problems with Markdown, and fails when any document has an issue +with Markdown formatting that may cause the page to render incorrectly within GitLab. +It will also fail when a document is using non-standard Markdown (which may render +correctly, but is not the current standard in GitLab documentation). + +Each formatting issue that mdl checks has an associated [rule](https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md), +and the rules that are currently enabled for GitLab documentation are listed in the +[`.mdlrc.style`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.mdlrc.style) file. +Configuration options are set in the [`.mdlrc`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.mdlrc.style) +file. + ## Structure ### Organize by topic, not by type @@ -142,9 +158,9 @@ The table below shows what kind of documentation goes where. ### Working with directories and files 1. When you create a new directory, always start with an `index.md` file. - Do not use another file name and **do not** create `README.md` files. + Do not use another file name and **do not** create `README.md` files. 1. **Do not** use special characters and spaces, or capital letters in file names, - directory names, branch names, and anything that generates a path. + directory names, branch names, and anything that generates a path. 1. When creating a new document and it has more than one word in its name, make sure to use underscores instead of spaces or dashes (`-`). For example, a proper naming would be `import_projects_from_github.md`. The same rule diff --git a/doc/development/fe_guide/frontend_faq.md b/doc/development/fe_guide/frontend_faq.md index e4225f2bc39..0d2aeffeac0 100644 --- a/doc/development/fe_guide/frontend_faq.md +++ b/doc/development/fe_guide/frontend_faq.md @@ -5,12 +5,12 @@ 1. **You talk about Frontend FAQ.** Please share links to it whenever applicable, so more eyes catch when content gets outdated. -2. **Keep it short and simple.** +1. **Keep it short and simple.** Whenever an answer needs more than two sentences it does not belong here. -3. **Provide background when possible.** +1. **Provide background when possible.** Linking to relevant source code, issue / epic, or other documentation helps to understand the answer. -4. **If you see something, do something.** +1. **If you see something, do something.** Please remove or update any content that is outdated as soon as you see it. ## FAQ diff --git a/doc/development/fe_guide/style_guide_js.md b/doc/development/fe_guide/style_guide_js.md index b50159c2b75..18ef754642d 100644 --- a/doc/development/fe_guide/style_guide_js.md +++ b/doc/development/fe_guide/style_guide_js.md @@ -548,7 +548,7 @@ Please check this [rules][eslint-plugin-vue-rules] for more documentation. <component @click="eventHandler"/> ``` -2. Shorthand `:` is preferable over `v-bind` +1. Shorthand `:` is preferable over `v-bind` ```javascript // bad @@ -558,7 +558,7 @@ Please check this [rules][eslint-plugin-vue-rules] for more documentation. <component :class="btn"/> ``` -3. Shorthand `#` is preferable over `v-slot` +1. Shorthand `#` is preferable over `v-slot` ```javascript // bad diff --git a/doc/development/fe_guide/style_guide_scss.md b/doc/development/fe_guide/style_guide_scss.md index 5220c9eeea3..f895cfd36ac 100644 --- a/doc/development/fe_guide/style_guide_scss.md +++ b/doc/development/fe_guide/style_guide_scss.md @@ -35,7 +35,7 @@ New utility classes should be added to [`utilities.scss`](https://gitlab.com/git We recommend a "utility-first" approach. 1. Start with utility classes. -2. If composing utility classes into a component class removes code duplication and encapsulates a clear responsibility, do it. +1. If composing utility classes into a component class removes code duplication and encapsulates a clear responsibility, do it. This encourages an organic growth of component classes and prevents the creation of one-off unreusable classes. Also, the kind of classes that emerge from "utility-first" tend to be design-centered (e.g. `.button`, `.alert`, `.card`) rather than domain-centered (e.g. `.security-report-widget`, `.commit-header-icon`). diff --git a/doc/development/gotchas.md b/doc/development/gotchas.md index 13dda17bb7d..941eea2609e 100644 --- a/doc/development/gotchas.md +++ b/doc/development/gotchas.md @@ -53,7 +53,7 @@ When run, this spec doesn't do what we might expect: (compared using ==) ``` -That's because FactoryBot sequences are not reseted for each example. +This is because FactoryBot sequences are not reset for each example. Please remember that sequence-generated values exist only to avoid having to explicitly set attributes that have a uniqueness constraint when using a factory. diff --git a/doc/development/import_export.md b/doc/development/import_export.md index 64c91f151c5..4c78c62f5f5 100644 --- a/doc/development/import_export.md +++ b/doc/development/import_export.md @@ -229,8 +229,8 @@ meaning that we want to bump the version up in the next version (or patch releas For example: 1. Add rename to `RelationRenameService` in X.Y -2. Remove it from `RelationRenameService` in X.Y + 1 -3. Bump Import/Export version in X.Y + 1 +1. Remove it from `RelationRenameService` in X.Y + 1 +1. Bump Import/Export version in X.Y + 1 ```ruby module Gitlab diff --git a/doc/development/interacting_components.md b/doc/development/interacting_components.md new file mode 100644 index 00000000000..74d52d808e2 --- /dev/null +++ b/doc/development/interacting_components.md @@ -0,0 +1,29 @@ +# Developing against interacting components or features + +It's not uncommon that a single code change can reflect and interact with multiple parts of GitLab +codebase. Furthermore, an existing feature might have an underlying integration or behavior that +might go unnoticed even by reviewers and maintainers. + +The goal of this section is to briefly list interacting pieces to think about +when making _backend_ changes that might involve multiple features or [components](architecture.md#components). + +## Uploads + +GitLab supports uploads to [object storage]. That means every feature and +change that affects uploads should also be tested against [object storage], +which is _not_ enabled by default in [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit). + +When working on a related feature, make sure to enable and test it +against [Minio](https://gitlab.com/gitlab-org/gitlab-development-kit/blob/master/doc/howto/object_storage.md). + +See also [File Storage in GitLab](file_storage.md). + +## Merge requests + +### Forks + +GitLab supports a great amount of features for [merge requests](../user/project/merge_requests/index.md). One +of them is the ability to create merge requests from and to [forks](../gitlab-basics/fork-project.md), +which should also be highly considered and tested upon development phase. + +[object storage]: https://docs.gitlab.com/charts/advanced/external-object-storage/ diff --git a/doc/development/licensed_feature_availability.md b/doc/development/licensed_feature_availability.md index 80ec7b8c0cf..29e4ace157b 100644 --- a/doc/development/licensed_feature_availability.md +++ b/doc/development/licensed_feature_availability.md @@ -14,7 +14,7 @@ it should be restricted on namespace scope. 1. Add the feature symbol on `EES_FEATURES`, `EEP_FEATURES` or `EEU_FEATURES` constants in `ee/app/models/license.rb`. Note on `ee/app/models/ee/namespace.rb` that _Bronze_ GitLab.com features maps to on-premise _EES_, _Silver_ to _EEP_ and _Gold_ to _EEU_. -2. Check using: +1. Check using: ```ruby project.feature_available?(:feature_symbol) @@ -29,8 +29,8 @@ the instance license. 1. Add the feature symbol on `EES_FEATURES`, `EEP_FEATURES` or `EEU_FEATURES` constants in `ee/app/models/license.rb`. -2. Add the same feature symbol to `GLOBAL_FEATURES` -3. Check using: +1. Add the same feature symbol to `GLOBAL_FEATURES` +1. Check using: ```ruby License.feature_available?(:feature_symbol) diff --git a/doc/development/performance.md b/doc/development/performance.md index 8b569a677b6..14b3f8204d2 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -267,7 +267,7 @@ piece of code is worth optimizing. The only two things you can do are: 1. Think about what the code does, how it's used, how many times it's called and how much time is spent in it relative to the total execution time (e.g., the total time spent in a web request). -2. Ask others (preferably in the form of an issue). +1. Ask others (preferably in the form of an issue). Some examples of changes that aren't really important/worth the effort: @@ -285,10 +285,10 @@ directly in a web request as much as possible. This has numerous benefits such as: 1. An error won't prevent the request from completing. -2. The process being slow won't affect the loading time of a page. -3. In case of a failure it's easy to re-try the process (Sidekiq takes care of +1. The process being slow won't affect the loading time of a page. +1. In case of a failure it's easy to re-try the process (Sidekiq takes care of this automatically). -4. By isolating the code from a web request it will hopefully be easier to test +1. By isolating the code from a web request it will hopefully be easier to test and maintain. It's especially important to use Sidekiq as much as possible when dealing with diff --git a/doc/development/policies.md b/doc/development/policies.md index c4ac42bb40a..833b0acb13e 100644 --- a/doc/development/policies.md +++ b/doc/development/policies.md @@ -89,8 +89,8 @@ Each line represents a rule that was evaluated. There are a few things to note: 1. The `-` or `+` symbol indicates whether the rule block was evaluated to be `false` or `true`, respectively. -2. The number inside the brackets indicates the score. -3. The last part of the line (e.g. `@john : Issue/1`) shows the username +1. The number inside the brackets indicates the score. +1. The last part of the line (e.g. `@john : Issue/1`) shows the username and subject for that rule. Here you can see that the first four rules were evaluated `false` for diff --git a/doc/development/testing_guide/end_to_end/page_objects.md b/doc/development/testing_guide/end_to_end/page_objects.md index 29ad49403fe..52957d1a1ab 100644 --- a/doc/development/testing_guide/end_to_end/page_objects.md +++ b/doc/development/testing_guide/end_to_end/page_objects.md @@ -27,7 +27,7 @@ When someone later changes `t.text_field :login` in the view associated with this page to `t.text_field :username` it will generate a different field identifier, what would effectively break all tests. -Because we are using `Page::Main::Login.act { sign_in_using_credentials }` +Because we are using `Page::Main::Login.perform(&:sign_in_using_credentials)` everywhere, when we want to sign into GitLab, the page object is the single source of truth, and we will need to update `fill_in :user_login` to `fill_in :user_username` only in a one place. diff --git a/doc/development/testing_guide/end_to_end/quick_start_guide.md b/doc/development/testing_guide/end_to_end/quick_start_guide.md index 3bbf8feab39..14a169dcc1d 100644 --- a/doc/development/testing_guide/end_to_end/quick_start_guide.md +++ b/doc/development/testing_guide/end_to_end/quick_start_guide.md @@ -116,9 +116,9 @@ end Below are the steps that the test covers: 1. The test finds the 'Edit' link for the labels and clicks on it. -2. Then it fills in the 'Assign labels' input field with the value 'animal::dolphin' and press enters. -3. Then it clicks in the content body to apply the label and refreshes the page. -4. Finally, the expectations check that the previous scoped label was removed and that the new one was added. +1. Then it fills in the 'Assign labels' input field with the value 'animal::dolphin' and press enters. +1. Then it clicks in the content body to apply the label and refreshes the page. +1. Finally, the expectations check that the previous scoped label was removed and that the new one was added. Let's now see how the second test case would look. @@ -144,9 +144,9 @@ end Below are the steps that the test covers: 1. The test finds the 'Edit' link for the labels and clicks on it. -2. Then it fills in the 'Assign labels' input field with the value 'plant::orchid' and press enters. -3. Then it clicks in the content body to apply the label and refreshes the page. -4. Finally, the expectations check that both scoped labels are present. +1. Then it fills in the 'Assign labels' input field with the value 'plant::orchid' and press enters. +1. Then it clicks in the content body to apply the label and refreshes the page. +1. Finally, the expectations check that both scoped labels are present. > Similar to the previous test, this one is also very straightforward, but there is some code duplication. Let's address it. @@ -290,7 +290,7 @@ As already mentioned in the [best practices](best_practices.md) document, end-to Some improvements that we could make in our test suite to optimize its time to run are: 1. Having a single test case (an `it` block) that exercises both scenarios to avoid "wasting" time in the tests' pre-conditions, instead of having two different test cases. -2. Making the selection of labels more performant by allowing for the selection of more than one label in the same reusable method. +1. Making the selection of labels more performant by allowing for the selection of more than one label in the same reusable method. Let's look at a suggestion that addresses the above points, one by one: @@ -332,8 +332,8 @@ To address point 1, we changed the test implementation from two `it` blocks into > Notice that the implementation of the new and unique `it` block had to change a little bit. Below we describe in details what it does. 1. It selects two scoped labels simultaneously, one from the same scope of the one already applied in the issue during the setup phase (in the `before` block), and another one from a different scope. -2. It asserts that the correct labels are visible in the `labels_block`, and that the labels were correctly added and removed; -3. Finally, the `select_label_and_refresh` method is changed to `select_labels_and_refresh`, which accepts an array of labels instead of a single label, and it iterates on them for faster label selection (this is what is used in step 1 explained above.) +1. It asserts that the correct labels are visible in the `labels_block`, and that the labels were correctly added and removed; +1. Finally, the `select_label_and_refresh` method is changed to `select_labels_and_refresh`, which accepts an array of labels instead of a single label, and it iterates on them for faster label selection (this is what is used in step 1 explained above.) ### 7. Resources @@ -542,9 +542,9 @@ end Notice that we have not only moved the `select_labels_and_refresh` method, but we have also changed its implementation to: 1. Click the `:edit_link_labels` element previously defined, instead of using `find('.block.labels .edit-link').click` -2. Use `within_element(:dropdown_menu_labels, text: label)`, and inside of it, we call `send_keys_to_element(:dropdown_input_field, [label, :enter])`, which is a method that we will implement in the `QA::Page::Base` class to replace `find('.dropdown-menu-labels .dropdown-input-field').send_keys [label, :enter]` -3. Use `click_body` after iterating on each label, instead of using `find('#content-body').click` -4. Iterate on every label again, and then we use `has_element?(:labels_block, text: label)` after clicking the page body (which applies the labels), and before refreshing the page, to avoid test flakiness due to refreshing too fast. +1. Use `within_element(:dropdown_menu_labels, text: label)`, and inside of it, we call `send_keys_to_element(:dropdown_input_field, [label, :enter])`, which is a method that we will implement in the `QA::Page::Base` class to replace `find('.dropdown-menu-labels .dropdown-input-field').send_keys [label, :enter]` +1. Use `click_body` after iterating on each label, instead of using `find('#content-body').click` +1. Iterate on every label again, and then we use `has_element?(:labels_block, text: label)` after clicking the page body (which applies the labels), and before refreshing the page, to avoid test flakiness due to refreshing too fast. ##### Details of `text_of_labels_block` diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index c909745b1ab..ff28c2ea5e2 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -80,18 +80,20 @@ describe('Component', () => { Remember that the performance of each test depends on the environment. ### Manual module mocks + Jest supports [manual module mocks](https://jestjs.io/docs/en/manual-mocks) by placing a mock in a `__mocks__/` directory next to the source module. **Don't do this.** We want to keep all of our test-related code in one place (the `spec/` folder), and the logic that Jest uses to apply mocks from `__mocks__/` is rather inconsistent. Instead, our test runner detects manual mocks from `spec/frontend/mocks/`. Any mock placed here is automatically picked up and injected whenever you import its source module. - Files in `spec/frontend/mocks/ce` will mock the corresponding CE module from `app/assets/javascripts`, mirroring the source module's path. - - Example: `spec/frontend/mocks/ce/lib/utils/axios_utils` will mock the module `~/lib/utils/axios_utils`. + - Example: `spec/frontend/mocks/ce/lib/utils/axios_utils` will mock the module `~/lib/utils/axios_utils`. - Files in `spec/frontend/mocks/node` will mock NPM packages of the same name or path. - We don't support mocking EE modules yet. If a mock is found for which a source module doesn't exist, the test suite will fail. 'Virtual' mocks, or mocks that don't have a 1-to-1 association with a source module, are not supported yet. #### Writing a mock + Create a JS module in the appropriate place in `spec/frontend/mocks/`. That's it. It will automatically mock its source package in all tests. Make sure that your mock's export has the same format as the mocked module. So, if you're mocking a CommonJS module, you'll need to use `module.exports` instead of the ES6 `export`. @@ -99,14 +101,15 @@ Make sure that your mock's export has the same format as the mocked module. So, It might be useful for a mock to expose a property that indicates if the mock was loaded. This way, tests can assert the presence of a mock without calling any logic and causing side-effects. The `~/lib/utils/axios_utils` module mock has such a property, `isMock`, that is `true` in the mock and undefined in the original class. Jest's mock functions also have a `mock` property that you can test. #### Bypassing mocks + If you ever need to import the original module in your tests, use [`jest.requireActual()`](https://jestjs.io/docs/en/jest-object#jestrequireactualmodulename) (or `jest.requireActual().default` for the default export). The `jest.mock()` and `jest.unmock()` won't have an effect on modules that have a manual mock, because mocks are imported and cached before any tests are run. #### Keep mocks light + Global mocks introduce magic and can affect how modules are imported in your tests. Try to keep them as light as possible and dependency-free. A global mock should be useful for any unit test. For example, the `axios_utils` and `jquery` module mocks throw an error when an HTTP request is attempted, since this is useful behaviour in >99% of tests. When in doubt, construct mocks in your test file using [`jest.mock()`](https://jestjs.io/docs/en/jest-object#jestmockmodulename-factory-options), [`jest.spyOn()`](https://jestjs.io/docs/en/jest-object#jestspyonobject-methodname), etc. - ## Karma test suite GitLab uses the [Karma][karma] test runner with [Jasmine] as its test @@ -462,7 +465,7 @@ See this [section][vue-test]. For running the frontend tests, you need the following commands: -- `rake karma:fixtures` (re-)generates [fixtures](#frontend-test-fixtures). +- `rake frontend:fixtures` (re-)generates [fixtures](#frontend-test-fixtures). - `yarn test` executes the tests. As long as the fixtures don't change, `yarn test` is sufficient (and saves you some time). @@ -515,8 +518,8 @@ Information on setting up and running RSpec integration tests with Code that is added to HAML templates (in `app/views/`) or makes Ajax requests to the backend has tests that require HTML or JSON from the backend. Fixtures for these tests are located at: -- `spec/javascripts/fixtures/`, for running tests in CE. -- `ee/spec/javascripts/fixtures/`, for running tests in EE. +- `spec/frontend/fixtures/`, for running tests in CE. +- `ee/spec/frontend/fixtures/`, for running tests in EE. Fixture files in: @@ -527,7 +530,7 @@ The following are examples of tests that work for both Karma and Jest: ```javascript it('makes a request', () => { - const responseBody = getJSONFixture('some/fixture.json'); // loads spec/javascripts/fixtures/some/fixture.json + const responseBody = getJSONFixture('some/fixture.json'); // loads spec/frontend/fixtures/some/fixture.json axiosMock.onGet(endpoint).reply(200, responseBody); myButton.click(); @@ -536,7 +539,7 @@ it('makes a request', () => { }); it('uses some HTML element', () => { - loadFixtures('some/page.html'); // loads spec/javascripts/fixtures/some/page.html and adds it to the DOM + loadFixtures('some/page.html'); // loads spec/frontend/fixtures/some/page.html and adds it to the DOM const element = document.getElementById('#my-id'); @@ -544,12 +547,12 @@ it('uses some HTML element', () => { }); ``` -HTML and JSON fixtures are generated from backend views and controllers using RSpec (see `spec/javascripts/fixtures/*.rb`). +HTML and JSON fixtures are generated from backend views and controllers using RSpec (see `spec/frontend/fixtures/*.rb`). For each fixture, the content of the `response` variable is stored in the output file. This variable gets automagically set if the test is marked as `type: :request` or `type: :controller`. -Fixtures are regenerated using the `bin/rake karma:fixtures` command but you can also generate them individually, -for example `bin/rspec spec/javascripts/fixtures/merge_requests.rb`. +Fixtures are regenerated using the `bin/rake frontend:fixtures` command but you can also generate them individually, +for example `bin/rspec spec/frontend/fixtures/merge_requests.rb`. When creating a new fixture, it often makes sense to take a look at the corresponding tests for the endpoint in `(ee/)spec/controllers/` or `(ee/)spec/requests/`. ## Gotchas diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index 20caac4cb74..626bd259ed6 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -354,6 +354,8 @@ There are several rake tasks available to you via the command line: - Does the same thing as `sudo gitlab-rake gitlab:elastic:create_empty_index` - [sudo gitlab-rake gitlab:elastic:index_snippets](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) - Performs an Elasticsearch import that indexes the snippets data. +- [sudo gitlab-rake gitlab:elastic:projects_not_indexed](https://gitlab.com/gitlab-org/gitlab-ee/blob/master/ee/lib/tasks/gitlab/elastic.rake) + - Displays which projects are not indexed. ### Environment Variables @@ -469,6 +471,10 @@ Here are some common pitfalls and how to overcome them: The more data present in your GitLab instance, the longer the indexing process takes. +- **There are some projects that weren't indexed, but we don't know which ones** + + You can run `sudo gitlab-rake gitlab:elastic:projects_not_indexed` to display projects that aren't indexed. + - **No new data is added to the Elasticsearch index when I push code** When performing the initial indexing of blobs, we lock all projects until the project finishes indexing. It could diff --git a/doc/integration/jenkins_deprecated.md b/doc/integration/jenkins_deprecated.md index eae705c9637..bac89ae2dc6 100644 --- a/doc/integration/jenkins_deprecated.md +++ b/doc/integration/jenkins_deprecated.md @@ -19,7 +19,7 @@ Requirements: ## Jenkins 1. Install [GitLab Hook plugin](https://wiki.jenkins.io/display/JENKINS/GitLab+Hook+Plugin) -2. Set up jenkins project +1. Set up jenkins project  diff --git a/doc/integration/jira_development_panel.md b/doc/integration/jira_development_panel.md index 60c7bdabf93..5e906e5af16 100644 --- a/doc/integration/jira_development_panel.md +++ b/doc/integration/jira_development_panel.md @@ -59,7 +59,7 @@ There are no special requirements if you are using GitLab.com.  - Check `api` in the Scopes section. -2. Click `Save application`. You will see the generated 'Application Id' and 'Secret' values. +1. Click `Save application`. You will see the generated 'Application Id' and 'Secret' values. Copy these values that you will use on the Jira configuration side. ## Jira Configuration @@ -70,7 +70,7 @@ There are no special requirements if you are using GitLab.com.  -2. Complete the form +1. Complete the form Select GitHub Enterprise for the `Host` field. @@ -92,7 +92,7 @@ There are no special requirements if you are using GitLab.com. Ensure that the rest of the checkboxes are checked. -3. Click `Add` to complete and create the integration. +1. Click `Add` to complete and create the integration. Jira takes up to a few minutes to know about (import behind the scenes) all the commits and branches for all the projects in the GitLab group you specified in the previous step. These are refreshed diff --git a/doc/security/reset_root_password.md b/doc/security/reset_root_password.md index 6a6c5262179..ec360e2d338 100644 --- a/doc/security/reset_root_password.md +++ b/doc/security/reset_root_password.md @@ -23,7 +23,7 @@ user = User.where(id: 1).first or ```bash -user = User.find_by(email: 'admin@local.host') +user = User.find_by(email: 'admin@example.com') ``` Now you can change your password: diff --git a/doc/subscriptions/index.md b/doc/subscriptions/index.md index 745a2253e84..68e62fff106 100644 --- a/doc/subscriptions/index.md +++ b/doc/subscriptions/index.md @@ -69,6 +69,14 @@ Once your GitLab.com account is linked, you can go to your [Subscriptions](https Please note that you need to be a group owner to associate a group to your subscription. +### GitLab.com: Upgrade your subscription plan + +GitLab.com subscriptions can be upgraded directly through the [Subscriptions portal](https://customers.gitlab.com/subscriptions). + +The Subscriptions portal provides an **Upgrade** button below each GitLab.com +subscription, which will lead you to a simple +checkout process. + ### Confirm or upgrade your GitLab.com subscription details within GitLab To see the status of your GitLab.com subscription, you can click on the Billings diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index f9ad952aaad..7a92a672801 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -789,11 +789,11 @@ To configure your application variables: 1. Go to your project's **Settings > CI/CD**, then expand the section called **Variables**. -2. Create a CI Variable, ensuring the key is prefixed with +1. Create a CI Variable, ensuring the key is prefixed with `K8S_SECRET_`. For example, you can create a variable with key `K8S_SECRET_RAILS_MASTER_KEY`. -3. Run an Auto Devops pipeline either by manually creating a new +1. Run an Auto Devops pipeline either by manually creating a new pipeline or by pushing a code change to GitLab. Auto DevOps pipelines will take your application secret variables to diff --git a/doc/user/admin_area/settings/img/email_confirmation.png b/doc/user/admin_area/settings/img/email_confirmation.png Binary files differnew file mode 100644 index 00000000000..4d888da3416 --- /dev/null +++ b/doc/user/admin_area/settings/img/email_confirmation.png diff --git a/doc/user/admin_area/settings/sign_up_restrictions.md b/doc/user/admin_area/settings/sign_up_restrictions.md index d77e91156f8..1b1bcbcd6e8 100644 --- a/doc/user/admin_area/settings/sign_up_restrictions.md +++ b/doc/user/admin_area/settings/sign_up_restrictions.md @@ -4,19 +4,26 @@ type: reference # Sign-up restrictions -By implementing sign-up restrictions, you can blacklist or whitelist email addresses -belonging to specific domains. +You can use sign-up restrictions to require user email confirmation, as well as +to blacklist or whitelist email addresses belonging to specific domains. >**Note**: These restrictions are only applied during sign-up. An admin is able to add a user through the admin panel with a disallowed domain. Also note that the users can change their email addresses after signup to disallowed domains. +## Require email confirmation + +You can send confirmation emails during sign-up and require that users confirm +their email address before they are allowed to sign in. + + + ## Whitelist email domains > [Introduced][ce-598] in GitLab 7.11.0 -You can restrict users to only signup using email addresses matching the given +You can restrict users to only sign up using email addresses matching the given domains list. ## Blacklist email domains @@ -24,7 +31,9 @@ domains list. > [Introduced][ce-5259] in GitLab 8.10. With this feature enabled, you can block email addresses of a specific domain -from creating an account on your GitLab server. This is particularly useful to prevent spam. Disposable email addresses are usually used by malicious users to create dummy accounts and spam issues. +from creating an account on your GitLab server. This is particularly useful +to prevent malicious users from creating spam accounts with disposable email +addresses. ## Settings @@ -33,10 +42,10 @@ To access this feature: 1. Navigate to the **Settings > General** in the Admin area. 1. Expand the **Sign-up restrictions** section. -For the: +For the blacklist, you can enter the list manually or upload a `.txt` file that +contains list entries. -- Blacklist, you can enter the list manually, or upload a `.txt` file with it. -- Whitelist you must enter the list manually. +For the whitelist, you must enter the list manually. Both the whitelist and blacklist accept wildcards. For example, you can use `*.company.com` to accept every `company.com` subdomain, or `*.io` to block all diff --git a/doc/user/application_security/dast/index.md b/doc/user/application_security/dast/index.md index a0a917c5ebd..4b98dd73d76 100644 --- a/doc/user/application_security/dast/index.md +++ b/doc/user/application_security/dast/index.md @@ -162,6 +162,28 @@ As the DAST job belongs to a separate `dast` stage that runs after all [default stages](../../../ci/yaml/README.md#stages), don't forget to add `stage: dast` when you override the template job definition. +## Available variables + +DAST can be [configured](#customizing-the-dast-settings) using environment variables. +Since it's a wrapper around the ZAP scanning scripts +([baseline](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan) +or [full](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) scan), it +accepts all arguments those scripts recognize (the arguments are the same). +The choice of the scan type depends on the `DAST_FULL_SCAN_ENABLED` environment +variable value. + +| Environment variable | Required | Description | +|-----------------------------| ----------|--------------------------------------------------------------------------------| +| `DAST_WEBSITE` | yes | The URL of the website to scan. | +| `DAST_AUTH_URL` | no | The authentication URL of the website to scan. | +| `DAST_USERNAME` | no | The username to authenticate to in the website. | +| `DAST_PASSWORD` | no | The password to authenticate to in the website. | +| `DAST_USERNAME_FIELD` | no | The name of username field at the sign-in HTML form. | +| `DAST_PASSWORD_FIELD` | no | The name of password field at the sign-in HTML form. | +| `DAST_AUTH_EXCLUDE_URLS` | no | The URLs to skip during the authenticated scan; comma-separated, no spaces in between. | +| `DAST_TARGET_AVAILABILITY_TIMEOUT` | no | Time limit in seconds to wait for target availability. Scan is attempted nevertheless if it runs out. Integer. Defaults to `60`. | +| `DAST_FULL_SCAN_ENABLED` | no | Switches the tool to execute [ZAP Full Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Full-Scan) instead of [ZAP Baseline Scan](https://github.com/zaproxy/zaproxy/wiki/ZAP-Baseline-Scan). Boolean. `true`, `True`, or `1` are considered as true value, otherwise false. Defaults to `false`. | + ## Security Dashboard The Security Dashboard is a good place to get an overview of all the security diff --git a/doc/user/application_security/index.md b/doc/user/application_security/index.md index 91e79f6c23b..56a4cbd26d2 100644 --- a/doc/user/application_security/index.md +++ b/doc/user/application_security/index.md @@ -115,9 +115,9 @@ generated by GitLab. To apply the fix: 1. Click on the vulnerability. 1. Download and review the patch file `remediation.patch`. -2. Ensure your local project has the same commit checked out that was used to generate the patch. -3. Run `git apply remediation.patch`. -4. Verify and commit the changes to your branch. +1. Ensure your local project has the same commit checked out that was used to generate the patch. +1. Run `git apply remediation.patch`. +1. Verify and commit the changes to your branch.  diff --git a/doc/user/application_security/sast/analyzers.md b/doc/user/application_security/sast/analyzers.md new file mode 100644 index 00000000000..59835aeba01 --- /dev/null +++ b/doc/user/application_security/sast/analyzers.md @@ -0,0 +1,143 @@ +--- +table_display_block: true +--- + +# SAST Analyzers **(ULTIMATE)** + +SAST relies on underlying third party tools that are wrapped into what we call +"Analyzers". An analyzer is a +[dedicated project](https://gitlab.com/gitlab-org/security-products/analyzers) +that wraps a particular tool to: + +- Expose its detection logic. +- Handle its execution. +- Convert its output to the common format. + +This is achieved by implementing the [common API](https://gitlab.com/gitlab-org/security-products/analyzers/common). + +SAST supports the following official analyzers: + +- [Bandit](https://gitlab.com/gitlab-org/security-products/analyzers/bandit) +- [Brakeman](https://gitlab.com/gitlab-org/security-products/analyzers/brakeman) +- [ESLint (Javascript)](https://gitlab.com/gitlab-org/security-products/analyzers/eslint) +- [SpotBugs with the Find Sec Bugs plugin (Ant, Gradle and wrapper, Grails, Maven and wrapper, SBT)](https://gitlab.com/gitlab-org/security-products/analyzers/spotbugs) +- [Flawfinder](https://gitlab.com/gitlab-org/security-products/analyzers/flawfinder) +- [Gosec](https://gitlab.com/gitlab-org/security-products/analyzers/gosec) +- [NodeJsScan](https://gitlab.com/gitlab-org/security-products/analyzers/nodejs-scan) +- [PHP CS security-audit](https://gitlab.com/gitlab-org/security-products/analyzers/phpcs-security-audit) +- [Secrets (Gitleaks, TruffleHog & Diffence secret detectors)](https://gitlab.com/gitlab-org/security-products/analyzers/secrets) +- [Security Code Scan (.NET)](https://gitlab.com/gitlab-org/security-products/analyzers/security-code-scan) +- [TSLint (Typescript)](https://gitlab.com/gitlab-org/security-products/analyzers/tslint) +- [Sobelow (Elixir Phoenix)](https://gitlab.com/gitlab-org/security-products/analyzers/sobelow) + +The analyzers are published as Docker images that SAST will use to launch +dedicated containers for each analysis. + +SAST is pre-configured with a set of **default images** that are maintained by +GitLab, but users can also integrate their own **custom images**. + +## Official default analyzers + +Any custom change to the official analyzers can be achieved by using an +[environment variable in your `.gitlab-ci.yml`](index.md#customizing-the-sast-settings). + +### Using a custom Docker mirror + +You can switch to a custom Docker registry that provides the official analyzer +images under a different prefix. For instance, the following instructs +SAST to pull `my-docker-registry/gl-images/bandit` +instead of `registry.gitlab.com/gitlab-org/security-products/analyzers/bandit`. +In `.gitlab-ci.yml` define: + +```yaml +include: + template: SAST.gitlab-ci.yml + +variables: + SAST_ANALYZER_IMAGE_PREFIX: my-docker-registry/gl-images +``` + +This configuration requires that your custom registry provides images for all +the official analyzers. + +### Selecting specific analyzers + +You can select the official analyzers you want to run. Here's how to enable +`bandit` and `flawfinder` while disabling all the other default ones. +In `.gitlab-ci.yml` define: + +```yaml +include: + template: SAST.gitlab-ci.yml + +variables: + SAST_DEFAULT_ANALYZERS: "bandit,flawfinder" +``` + +`bandit` runs first. When merging the reports, SAST will +remove the duplicates and will keep the `bandit` entries. + +### Disabling default analyzers + +Setting `SAST_DEFAULT_ANALYZERS` to an empty string will disable all the official +default analyzers. In `.gitlab-ci.yml` define: + +```yaml +include: + template: SAST.gitlab-ci.yml + +variables: + SAST_DEFAULT_ANALYZERS: "" +``` + +That's needed when one totally relies on [custom analyzers](#custom-analyzers). + +## Custom Analyzers + +You can provide your own analyzers as a comma separated list of Docker images. +Here's how to add `analyzers/csharp` and `analyzers/perl` to the default images: +In `.gitlab-ci.yml` define: + +```yaml +include: + template: SAST.gitlab-ci.yml + +variables: + SAST_ANALYZER_IMAGES: "my-docker-registry/analyzers/csharp,amy-docker-registry/analyzers/perl" +``` + +The values must be the full path to the container registry images, +like what you would feed to the `docker pull` command. + +NOTE: **Note:** +This configuration doesn't benefit from the integrated detection step. +SAST has to fetch and spawn each Docker image to establish whether the +custom analyzer can scan the source code. + +## Analyzers Data + +| Property \ Tool | Bandit | Brakeman | ESLint security | Find Sec Bugs | Flawfinder | Go AST Scanner | NodeJsScan | Php CS Security Audit | Security code Scan (.NET) | TSLint Security | Sobelow | +| --------------------------------------- | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :------------------: | :---------------------: | :-------------------------: | :-------------: | :----------------: | +| Severity | ✓ | 𐄂 | 𐄂 | ✓ | 𐄂 | ✓ | 𐄂 | ✓ | 𐄂 | ✓ | 𐄂 | +| Title | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Description | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | ✓ | 𐄂 | 𐄂 | ✓ | ✓ | +| File | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Start line | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| End line | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | 𐄂 | +| Start column | 𐄂 | 𐄂 | ✓ | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | 𐄂 | +| End column | 𐄂 | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | 𐄂 | +| External id (e.g. CVE) | 𐄂 | ⚠ | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| URLs | 𐄂 | ✓ | 𐄂 | ⚠ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Internal doc/explanation | ⚠ | ✓ | 𐄂 | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | +| Solution | 𐄂 | 𐄂 | 𐄂 | ⚠ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Confidence | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | ✓ | +| Affected item (e.g. class or package) | 𐄂 | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Source code extract | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | 𐄂 | 𐄂 | 𐄂 | 𐄂 | 𐄂 | +| Internal ID | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | 𐄂 | ✓ | ✓ | ✓ | ✓ | + +- ✓ => we have that data +- ⚠ => we have that data but it's partially reliable, or we need to extract it from unstructured content +- 𐄂 => we don't have that data or it would need to develop specific or inefficient/unreliable logic to obtain it. + +The values provided by these tools are heterogeneous so they are sometimes +normalized into common values (e.g., `severity`, `confidence`, etc). diff --git a/doc/user/application_security/sast/index.md b/doc/user/application_security/sast/index.md index 1f9fd9d4e18..7df86eedd18 100644 --- a/doc/user/application_security/sast/index.md +++ b/doc/user/application_security/sast/index.md @@ -135,6 +135,58 @@ sast: CI_DEBUG_TRACE: "true" ``` +### Available variables + +SAST can be [configured](#customizing-the-sast-settings) using environment variables. + +#### Docker images + +The following are Docker image-related variables. + +| Environment variable | Description | +|-------------------------------|--------------------------------------------------------------------------------| +| `SAST_ANALYZER_IMAGES` | Comma separated list of custom images. Default images are still enabled. Read more about [customizing analyzers](analyzers.md). | +| `SAST_ANALYZER_IMAGE_PREFIX` | Override the name of the Docker registry providing the default images (proxy). Read more about [customizing analyzers](analyzers.md). | +| `SAST_ANALYZER_IMAGE_TAG` | Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md). | +| `SAST_DEFAULT_ANALYZERS` | Override the names of default images. Read more about [customizing analyzers](analyzers.md). | +| `SAST_PULL_ANALYZER_IMAGES` | Pull the images from the Docker registry (set to 0 to disable). Read more about [customizing analyzers](analyzers.md). | + +### Vulnerability filters + +Some analyzers make it possible to filter out vulnerabilities under a given threshold. + +| `SAST_BANDIT_EXCLUDED_PATHS` | - | comma-separated list of paths to exclude from scan. Uses Python's [`fnmatch` syntax](https://docs.python.org/2/library/fnmatch.html) | +| `SAST_BRAKEMAN_LEVEL` | 1 | Ignore Brakeman vulnerabilities under given confidence level. Integer, 1=Low 3=High. | +| `SAST_FLAWFINDER_LEVEL` | 1 | Ignore Flawfinder vulnerabilities under given risk level. Integer, 0=No risk, 5=High risk. | +| `SAST_GITLEAKS_ENTROPY_LEVEL` | 8.0 | Minimum entropy for secret detection. Float, 0.0 = low, 8.0 = high. | +| `SAST_GOSEC_LEVEL` | 0 | Ignore gosec vulnerabilities under given confidence level. Integer, 0=Undefined, 1=Low, 1=Medium, 3=High. | +| `SAST_EXCLUDED_PATHS` | - | Exclude vulnerabilities from output based on the paths. This is a comma-separated list of patterns. Patterns can be globs, file or folder paths. Parent directories will also match patterns. | + +### Timeouts + +The following variables configure timeouts. + +| `SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT` | 2m | Time limit for Docker client negotiation. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m". | +| `SAST_PULL_ANALYZER_IMAGE_TIMEOUT` | 5m | Time limit when pulling the image of an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m". | +| `SAST_RUN_ANALYZER_TIMEOUT` | 20m | Time limit when running an analyzer. Timeouts are parsed using Go's [`ParseDuration`](https://golang.org/pkg/time/#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". For example, "300ms", "1.5h" or "2h45m".| + +### Analyzer settings + +Some analyzers can be customized with environment variables. + +| Environment variable | Analyzer | Description | +|-------------------------|----------|----------| +| `ANT_HOME` | spotbugs | The `ANT_HOME` environment variable. | +| `ANT_PATH` | spotbugs | Path to the `ant` executable. | +| `GRADLE_PATH` | spotbugs | Path to the `gradle` executable. | +| `JAVA_OPTS` | spotbugs | Additional arguments for the `java` executable. | +| `JAVA_PATH` | spotbugs | Path to the `java` executable. | +| `MAVEN_CLI_OPTS` | spotbugs | Additional arguments for the `mvn` or `mvnw` executable. | +| `MAVEN_PATH` | spotbugs | Path to the `mvn` executable. | +| `MAVEN_REPO_PATH` | spotbugs | Path to the Maven local repository (shortcut for the `maven.repo.local` property). | +| `SBT_PATH` | spotbugs | Path to the `sbt` executable. | +| `FAIL_NEVER` | spotbugs | Set to `1` to ignore compilation failure. | + ## Reports JSON format CAUTION: **Caution:** diff --git a/doc/user/application_security/security_dashboard/index.md b/doc/user/application_security/security_dashboard/index.md index 3b01fe66e03..2a2385c00ae 100644 --- a/doc/user/application_security/security_dashboard/index.md +++ b/doc/user/application_security/security_dashboard/index.md @@ -26,8 +26,8 @@ To use the project or group security dashboard: 1. At least one project inside a group must be configured with at least one of the [supported reports](#supported-reports). -2. The configured jobs must use the [new `reports` syntax](../../../ci/yaml/README.md#artifactsreports). -3. [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 or newer must be used. +1. The configured jobs must use the [new `reports` syntax](../../../ci/yaml/README.md#artifactsreports). +1. [GitLab Runner](https://docs.gitlab.com/runner/) 11.5 or newer must be used. If you're using the shared Runners on GitLab.com, this is already the case. ## Project Security Dashboard diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 6956086c382..d5c836554ce 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -19,8 +19,8 @@ This namespace: To see a list of available applications to install: 1. For a: - - Project-level cluster, navigate to your project's **Operations > Kubernetes**. - - Group-level cluster, navigate to your group's **Kubernetes** page. + - Project-level cluster, navigate to your project's **Operations > Kubernetes**. + - Group-level cluster, navigate to your group's **Kubernetes** page. Install Helm first as it's used to install other applications. @@ -232,8 +232,8 @@ The applications below can be upgraded. To upgrade an application: 1. For a: - - Project-level cluster, navigate to your project's **Operations > Kubernetes**. - - Group-level cluster, navigate to your group's **Kubernetes** page. + - Project-level cluster, navigate to your project's **Operations > Kubernetes**. + - Group-level cluster, navigate to your group's **Kubernetes** page. 1. Select your cluster. 1. If an upgrade is available, the **Upgrade** button is displayed. Click the button to upgrade. @@ -259,8 +259,8 @@ The applications below can be uninstalled. To uninstall an application: 1. For a: - - Project-level cluster, navigate to your project's **Operations > Kubernetes**. - - Group-level cluster, navigate to your group's **Kubernetes** page. + - Project-level cluster, navigate to your project's **Operations > Kubernetes**. + - Group-level cluster, navigate to your group's **Kubernetes** page. 1. Select your cluster. 1. Click the **Uninstall** button for the application. diff --git a/doc/user/group/bulk_editing/img/bulk-editing.png b/doc/user/group/bulk_editing/img/bulk-editing.png Binary files differnew file mode 100644 index 00000000000..d08503dc312 --- /dev/null +++ b/doc/user/group/bulk_editing/img/bulk-editing.png diff --git a/doc/user/group/bulk_editing/index.md b/doc/user/group/bulk_editing/index.md new file mode 100644 index 00000000000..117a46da0ea --- /dev/null +++ b/doc/user/group/bulk_editing/index.md @@ -0,0 +1,22 @@ +# Bulk editing issue milestones **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/7249) in +[GitLab Ultimate](https://about.gitlab.com/pricing/) 12.1. + +NOTE: **Note:** +A permission level of `Reporter` or higher is required in order to manage issues. + +Milestones can be updated simultaneously across multiple issues by using the bulk editing feature. + + + +To bulk update group issue milestones: + +1. Navigate to the issues list. +1. Click **Edit issues**. + - This will open a sidebar on the right-hand side of your screen where an editable field + for milestones will be displayed. + - Checkboxes will also appear beside each issue. +1. Check the checkbox beside each issue to be edited. +1. Select the desired milestone from the sidebar. +1. Click **Update all**. diff --git a/doc/user/group/index.md b/doc/user/group/index.md index db348c678eb..7597105f36f 100644 --- a/doc/user/group/index.md +++ b/doc/user/group/index.md @@ -78,6 +78,10 @@ Issues and merge requests are part of projects. For a given group, you can view [issues](../project/issues/index.md#issues-list) and [merge requests](../project/merge_requests/index.md#merge-requests-per-group) across all projects in that group, together in a single list view. +### Bulk editing issues + +For details, see [bulk editing issues](../group/bulk_editing/index.md). + ## Create a new group > For a list of words that are not allowed to be used as group names see the diff --git a/doc/user/group/subgroups/index.md b/doc/user/group/subgroups/index.md index 1c6cca049c5..7c80c4b7928 100644 --- a/doc/user/group/subgroups/index.md +++ b/doc/user/group/subgroups/index.md @@ -28,39 +28,39 @@ only 1 parent group. It resembles a directory behavior or a nested items list: - Group 1 - Group 1.1 - Group 1.2 - - Group 1.2.1 - - Group 1.2.2 - - Group 1.2.2.1 + - Group 1.2.1 + - Group 1.2.2 + - Group 1.2.2.1 In a real world example, imagine maintaining a GNU/Linux distribution with the first group being the name of the distribution, and subsequent groups split as follows: - Organization Group - GNU/Linux distro - Category Subgroup - Packages - - (project) Package01 - - (project) Package02 + - (project) Package01 + - (project) Package02 - Category Subgroup - Software - - (project) Core - - (project) CLI - - (project) Android app - - (project) iOS app + - (project) Core + - (project) CLI + - (project) Android app + - (project) iOS app - Category Subgroup - Infra tools - - (project) Ansible playbooks + - (project) Ansible playbooks Another example of GitLab as a company would be the following: - Organization Group - GitLab - Category Subgroup - Marketing - - (project) Design - - (project) General + - (project) Design + - (project) General - Category Subgroup - Software - - (project) GitLab CE - - (project) GitLab EE - - (project) Omnibus GitLab - - (project) GitLab Runner - - (project) GitLab Pages daemon + - (project) GitLab CE + - (project) GitLab EE + - (project) Omnibus GitLab + - (project) GitLab Runner + - (project) GitLab Pages daemon - Category Subgroup - Infra tools - - (project) Chef cookbooks + - (project) Chef cookbooks - Category Subgroup - Executive team --- @@ -88,13 +88,13 @@ To create a subgroup: 1. In the group's dashboard expand the **New project** split button, select **New subgroup** and click the **New subgroup** button. -  +  1. Create a new group like you would normally do. Notice that the parent group namespace is fixed under **Group path**. The visibility level can differ from the parent group. -  +  1. Click the **Create group** button and you will be taken to the new group's dashboard page. diff --git a/doc/user/markdown.md b/doc/user/markdown.md index 96e74125801..37f3f21f539 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -1160,13 +1160,15 @@ Examples: + Or pluses ``` +<!-- The "2." and "4." in the example above are changed to "1." below, only to match the standards on docs.gitlab.com --> + 1. First ordered list item -2. Another item +1. Another item - Unordered sub-list. 1. Actual numbers don't matter, just that it's a number 1. Ordered sub-list 1. Next ordered sub-list item -4. And another item. +1. And another item. * Unordered lists can use asterisks - Or minuses @@ -1184,14 +1186,14 @@ Example: Second paragraph of first item. -2. Another item +1. Another item ``` 1. First ordered list item Second paragraph of first item. -2. Another item +1. Another item --- @@ -1205,14 +1207,14 @@ Example: Paragraph of first item. -2. Another item +1. Another item ``` 1. First ordered list item Paragraph of first item. -2. Another item +1. Another item ### Superscripts / Subscripts diff --git a/doc/user/project/badges.md b/doc/user/project/badges.md index 8849dd2d684..cd3e5f5a63c 100644 --- a/doc/user/project/badges.md +++ b/doc/user/project/badges.md @@ -17,10 +17,10 @@ If you find that you have to add the same badges to several projects, you may wa To add a new badge to a project: -1. Navigate to your project's **Settings > General > Badges**. -1. Under "Link", enter the URL that the badges should point to and under - "Badge image URL" the URL of the image that should be displayed. -1. Submit the badge by clicking the **Add badge** button. +1. Navigate to your project's **Settings > General > Badges**. +1. Under "Link", enter the URL that the badges should point to and under + "Badge image URL" the URL of the image that should be displayed. +1. Submit the badge by clicking the **Add badge** button. After adding a badge to a project, you can see it in the list below the form. You can edit it by clicking on the pen icon next to it or to delete it by @@ -39,10 +39,10 @@ project, consider adding them on the [project level](#project-badges) or use To add a new badge to a group: -1. Navigate to your group's **Settings > General > Badges**. -1. Under "Link", enter the URL that the badges should point to and under - "Badge image URL" the URL of the image that should be displayed. -1. Submit the badge by clicking the **Add badge** button. +1. Navigate to your group's **Settings > General > Badges**. +1. Under "Link", enter the URL that the badges should point to and under + "Badge image URL" the URL of the image that should be displayed. +1. Submit the badge by clicking the **Add badge** button. After adding a badge to a group, you can see it in the list below the form. You can edit the badge by clicking on the pen icon next to it or to delete it diff --git a/doc/user/project/bulk_editing.md b/doc/user/project/bulk_editing.md index d0c7daf4692..1783f81df3a 100644 --- a/doc/user/project/bulk_editing.md +++ b/doc/user/project/bulk_editing.md @@ -13,13 +13,20 @@ by using the bulk editing feature.  NOTE: **Note:** -Bulk editing of issues and merge requests is only available at the project level. +Bulk editing of merge requests is only available at the project level. +For more details, see [bulk editing group issues](../group/bulk_editing/index.md). -To update multiple project issues or merge requests at the same time, navigate to -their respective lists and click **Edit issues** or **Edit merge requests** available -in the tab bar. This will open a sidebar on the right-hand side of your screen -where editable fields will be displayed. Checkboxes will also appear to the left-hand -side of eachissue or merge request for you to select the items you want to update. +To update multiple project issues or merge requests at the same time: -Once you have selected all relevant items, choose the appropriate fields and their -values from the sidebar and click **Update all** to apply your changes. +1. Navigate to their respective list. + +1. Click **Edit issues** or **Edit merge requests**. + + - This will open a sidebar on the right-hand side of your screen + where editable fields will be displayed. + + - Checkboxes will also appear beside each issue or merge request. + +1. Check the checkboxes of each items to be edited. +1. Choose the appropriate fields and their values from the sidebar. +1. Click **Update all**. diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index 14ee6303bf9..a32759c7bdc 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -41,8 +41,7 @@ To run Knative on Gitlab, you will need: external IP address or hostname for that domain. 1. **`.gitlab-ci.yml`:** GitLab uses [Kaniko](https://github.com/GoogleContainerTools/kaniko) to build the application. We also use [gitlabktl](https://gitlab.com/gitlab-org/gitlabktl) - and [TriggerMesh CLI](https://github.com/triggermesh/tm) CLIs to simplify the - deployment of services and functions to Knative. + CLI to simplify the deployment of services and functions to Knative. 1. **`serverless.yml`** (for [functions only](#deploying-functions)): When using serverless to deploy functions, the `serverless.yml` file will contain the information for all the functions being hosted in the repository as well as a reference to the runtime being used. @@ -249,7 +248,7 @@ Explanation of the fields used above: | Parameter | Description | |-----------|-------------| -| `name` | Indicates which provider is used to execute the `serverless.yml` file. In this case, the TriggerMesh `tm` CLI. | +| `name` | Indicates which provider is used to execute the `serverless.yml` file. In this case, the TriggerMesh middleware. | | `environment` | Includes the environment variables to be passed as part of function execution for **all** functions in the file, where `FOO` is the variable name and `BAR` are he variable contents. You may replace this with you own variables. | ### `functions` @@ -343,27 +342,23 @@ Go to the **CI/CD > Pipelines** and click on the pipeline that deployed your app The output will look like this: ```bash -Running with gitlab-runner 11.5.0~beta.844.g96d88322 (96d88322) - on docker-auto-scale 72989761 -Using Docker executor with image gcr.io/triggermesh/tm@sha256:e3ee74db94d215bd297738d93577481f3e4db38013326c90d57f873df7ab41d5 ... -Pulling docker image gcr.io/triggermesh/tm@sha256:e3ee74db94d215bd297738d93577481f3e4db38013326c90d57f873df7ab41d5 ... -Using docker image sha256:6b3f6590a9b30bd7aafb9573f047d930c70066e43955b4beb18a1eee175f6de1 for gcr.io/triggermesh/tm@sha256:e3ee74db94d215bd297738d93577481f3e4db38013326c90d57f873df7ab41d5 ... -Running on runner-72989761-project-4342902-concurrent-0 via runner-72989761-stg-srm-1541795796-27929c96... -Cloning repository... -Cloning into '/builds/danielgruesso/knative'... -Checking out 8671ad20 as master... -Skipping Git submodules setup -$ echo "$CI_REGISTRY_IMAGE" -registry.staging.gitlab.com/danielgruesso/knative -$ tm -n "$KUBE_NAMESPACE" --config "$KUBECONFIG" deploy service "$CI_PROJECT_NAME" --from-image "$CI_REGISTRY_IMAGE" --wait -Deployment started. Run "tm -n knative-4342902 describe service knative" to see the details -Waiting for ready state....... -Service domain: knative.knative-4342902.example.com +Running with gitlab-runner 12.1.0-rc1 (6da35412) + on prm-com-gitlab-org ae3bfce3 +Using Docker executor with image registry.gitlab.com/gitlab-org/gitlabktl:latest ... +Running on runner-ae3bfc-concurrent-0 via runner-ae3bfc ... +Fetching changes... +Authenticating with credentials from job payload (GitLab Registry) +$ /usr/bin/gitlabktl application deploy +Welcome to gitlabktl tool +time="2019-07-15T10:51:07Z" level=info msg="deploying registry credentials" +Creating app-hello function +Waiting for app-hello ready state +Service app-hello URL: http://app-hello.serverless.example.com Job succeeded ``` -The second to last line, labeled **Service domain** contains the URL for the deployment. Copy and paste the domain into your -browser to see the app live. +The second to last line, labeled **Service domain** contains the URL for the +deployment. Copy and paste the domain into your browser to see the app live.  diff --git a/doc/user/project/cycle_analytics.md b/doc/user/project/cycle_analytics.md index 5d36e1d4be3..6707b88c317 100644 --- a/doc/user/project/cycle_analytics.md +++ b/doc/user/project/cycle_analytics.md @@ -21,19 +21,19 @@ Analytics** tab. There are seven stages that are tracked as part of the Cycle Analytics calculations. - **Issue** (Tracker) - - Time to schedule an issue (by milestone or by adding it to an issue board) + - Time to schedule an issue (by milestone or by adding it to an issue board) - **Plan** (Board) - - Time to first commit + - Time to first commit - **Code** (IDE) - - Time to create a merge request + - Time to create a merge request - **Test** (CI) - - Time it takes GitLab CI/CD to test your code + - Time it takes GitLab CI/CD to test your code - **Review** (Merge Request/MR) - - Time spent on code review + - Time spent on code review - **Staging** (Continuous Deployment) - - Time between merging and deploying to production + - Time between merging and deploying to production - **Production** (Total) - - Total lifecycle time; i.e. the velocity of the project or team + - Total lifecycle time; i.e. the velocity of the project or team ## How the data is measured diff --git a/doc/user/project/import/svn.md b/doc/user/project/import/svn.md index 7359487e1bf..d175ee87f26 100644 --- a/doc/user/project/import/svn.md +++ b/doc/user/project/import/svn.md @@ -9,13 +9,13 @@ between the two, for more information consult your favorite search engine. There are two approaches to SVN to Git migration: 1. [Git/SVN Mirror](#smooth-migration-with-a-gitsvn-mirror-using-subgit) which: - - Makes the GitLab repository to mirror the SVN project. - - Git and SVN repositories are kept in sync; you can use either one. - - Smoothens the migration process and allows to manage migration risks. + - Makes the GitLab repository to mirror the SVN project. + - Git and SVN repositories are kept in sync; you can use either one. + - Smoothens the migration process and allows to manage migration risks. 1. [Cut over migration](#cut-over-migration-with-svn2git) which: - - Translates and imports the existing data and history from SVN to Git. - - Is a fire and forget approach, good for smaller teams. + - Translates and imports the existing data and history from SVN to Git. + - Is a fire and forget approach, good for smaller teams. ## Smooth migration with a Git/SVN mirror using SubGit diff --git a/doc/user/project/index.md b/doc/user/project/index.md index 7307c5b8991..45e96437517 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -66,15 +66,15 @@ When you create a project in GitLab, you'll have access to a large number of to automatically set up your app's deployment - [Enable and disable GitLab CI](../../ci/enable_or_disable_ci.md) - [Pipelines](../../ci/pipelines.md): Configure and visualize - your GitLab CI/CD pipelines from the UI - - [Scheduled Pipelines](pipelines/schedules.md): Schedule a pipeline - to start at a chosen time - - [Pipeline Graphs](../../ci/pipelines.md#visualizing-pipelines): View your - entire pipeline from the UI - - [Job artifacts](pipelines/job_artifacts.md): Define, - browse, and download job artifacts - - [Pipeline settings](pipelines/settings.md): Set up Git strategy (choose the default way your repository is fetched from GitLab in a job), - timeout (defines the maximum amount of time in minutes that a job is able run), custom path for `.gitlab-ci.yml`, test coverage parsing, pipeline's visibility, and much more + your GitLab CI/CD pipelines from the UI + - [Scheduled Pipelines](pipelines/schedules.md): Schedule a pipeline + to start at a chosen time + - [Pipeline Graphs](../../ci/pipelines.md#visualizing-pipelines): View your + entire pipeline from the UI + - [Job artifacts](pipelines/job_artifacts.md): Define, + browse, and download job artifacts + - [Pipeline settings](pipelines/settings.md): Set up Git strategy (choose the default way your repository is fetched from GitLab in a job), + timeout (defines the maximum amount of time in minutes that a job is able run), custom path for `.gitlab-ci.yml`, test coverage parsing, pipeline's visibility, and much more - [Kubernetes cluster integration](clusters/index.md): Connecting your GitLab project with a Kubernetes cluster - [Feature Flags](operations/feature_flags.md): Feature flags allow you to ship a project in diff --git a/doc/user/project/integrations/img/prometheus_dashboard_area_panel_type.png b/doc/user/project/integrations/img/prometheus_dashboard_area_panel_type.png Binary files differnew file mode 100644 index 00000000000..7260b11f07b --- /dev/null +++ b/doc/user/project/integrations/img/prometheus_dashboard_area_panel_type.png diff --git a/doc/user/project/integrations/img/prometheus_dashboard_single_stat_panel_type.png b/doc/user/project/integrations/img/prometheus_dashboard_single_stat_panel_type.png Binary files differnew file mode 100644 index 00000000000..ce4c54f909d --- /dev/null +++ b/doc/user/project/integrations/img/prometheus_dashboard_single_stat_panel_type.png diff --git a/doc/user/project/integrations/mock_ci.md b/doc/user/project/integrations/mock_ci.md index 1c64b275d6e..886094a6531 100644 --- a/doc/user/project/integrations/mock_ci.md +++ b/doc/user/project/integrations/mock_ci.md @@ -5,9 +5,9 @@ To set up the mock CI service server, respond to the following endpoints - `commit_status`: `#{project.namespace.path}/#{project.path}/status/#{sha}.json` - - Have your service return `200 { status: ['failed'|'canceled'|'running'|'pending'|'success'|'success-with-warnings'|'skipped'|'not_found'] }` + - Have your service return `200 { status: ['failed'|'canceled'|'running'|'pending'|'success'|'success-with-warnings'|'skipped'|'not_found'] }` - If the service returns a 404, it is interpreted as `pending` - `build_page`: `#{project.namespace.path}/#{project.path}/status/#{sha}` - - Just where the build is linked to, doesn't matter if implemented + - Just where the build is linked to, doesn't matter if implemented For an example of a mock CI server, see [`gitlab-org/gitlab-mock-ci-service`](https://gitlab.com/gitlab-org/gitlab-mock-ci-service) diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index 72f12972596..61c30e0b0ef 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -191,7 +191,7 @@ The following tables outline the details of expected properties. | Property | Type | Required | Description | | ------ | ------ | ------ | ------- | -| `type` | enum | no, defaults to `area-chart` | Specifies the chart type to use. Only `area-chart` is currently supported. | +| `type` | enum | no, defaults to `area-chart` | Specifies the chart type to use. | | `title` | string | yes | Heading for the panel. | | `y_label` | string | no, but highly encouraged | Y-Axis label for the panel. | | `weight` | number | no, defaults to order in file | Order to appear within the grouping. Lower number means higher priority, which will be higher on the page. Numbers do not need to be consecutive. | @@ -207,6 +207,65 @@ The following tables outline the details of expected properties. | `query` | string | yes if `query_range` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | | `query_range` | string | yes if `query` is not defined | Defines the Prometheus query to be used to populate the chart/panel. If defined, the `query_range` endpoint of the [Prometheus API](https://prometheus.io/docs/prometheus/latest/querying/api/) will be utilized. | +#### Panel types for dashboards + +The below panel types are supported in monitoring dashboards. + +##### Area + +To add an area panel type to a dashboard, look at the following sample dashboard file: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group Title' + panels: + - type: area-chart + title: "Chart Title" + y_label: "Y-Axis" + metrics: + - id: 10 + query_range: 'http_requests_total' + label: "Metric of Ages" + unit: "count" +``` + +Note the following properties: + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| type | string | no | Type of panel to be rendered. Optional for area panel types | +| query_range | yes | required | For area panel types, you must use a [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) | + + + +##### Single Stat + +To add a single stat panel type to a dashboard, look at the following sample dashboard file: + +```yaml +dashboard: 'Dashboard Title' +panel_groups: + - group: 'Group Title' + panels: + - title: "Single Stat" + type: "single-stat" + metrics: + - id: 10 + query: 'max(go_memstats_alloc_bytes{job="prometheus"})' + unit: MB + label: "Total" +``` + +Note the following properties: + +| Property | Type | Required | Description | +| ------ | ------ | ------ | ------ | +| type | string | yes | Type of panel to be rendered. For single stat panel types, set to `single-stat` | +| query | string | yes | For single stat panel types, you must use an [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries) | + + + ### Setting up alerts for Prometheus metrics **(ULTIMATE)** #### Managed Prometheus instances diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 30940b65454..84adb9637fc 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -58,13 +58,13 @@ Navigate to the webhooks page by going to your project's If you are writing your own endpoint (web server) that will receive GitLab webhooks keep in mind the following things: -- Your endpoint should send its HTTP response as fast as possible. If - you wait too long, GitLab may decide the hook failed and retry it. -- Your endpoint should ALWAYS return a valid HTTP response. If you do - not do this then GitLab will think the hook failed and retry it. - Most HTTP libraries take care of this for you automatically but if - you are writing a low-level hook this is important to remember. -- GitLab ignores the HTTP status code returned by your endpoint. +- Your endpoint should send its HTTP response as fast as possible. If + you wait too long, GitLab may decide the hook failed and retry it. +- Your endpoint should ALWAYS return a valid HTTP response. If you do + not do this then GitLab will think the hook failed and retry it. + Most HTTP libraries take care of this for you automatically but if + you are writing a low-level hook this is important to remember. +- GitLab ignores the HTTP status code returned by your endpoint. ## Secret token diff --git a/doc/user/project/issue_board.md b/doc/user/project/issue_board.md index 3eb5581912a..eaca5f8cfb8 100644 --- a/doc/user/project/issue_board.md +++ b/doc/user/project/issue_board.md @@ -143,10 +143,10 @@ Create lists for each of your team members and quickly drag-and-drop issues onto - **Issue Board** - Each board represents a unique view for your issues. It can have multiple lists with each list consisting of issues represented by cards. - **List** - A column on the issue board that displays issues matching certain attributes. In addition to the default lists of 'Open' and 'Closed' issue, each additional list will show issues matching your chosen label or assignee. On the top of that list you can see the number of issues that belong to it. - - **Label list**: a list based on a label. It shows all opened issues with that label. - - **Assignee list**: a list which includes all issues assigned to a user. - - **Open** (default): shows all open issues that do not belong to one of the other lists. Always appears as the leftmost list. - - **Closed** (default): shows all closed issues. Always appears as the rightmost list. + - **Label list**: a list based on a label. It shows all opened issues with that label. + - **Assignee list**: a list which includes all issues assigned to a user. + - **Open** (default): shows all open issues that do not belong to one of the other lists. Always appears as the leftmost list. + - **Closed** (default): shows all closed issues. Always appears as the rightmost list. - **Card** - A box in the list that represents an individual issue. The information you can see on a card consists of the issue number, the issue title, the assignee, and the labels associated with the issue. You can drag cards from one list to another to change their label or assignee from that of the source list to that of the destination list. ## Permissions @@ -386,6 +386,10 @@ a given board inside your GitLab instance, any time those two issues are subsequ loaded in any board in the same instance (could be a different project board or a different group board, for example), that ordering will be maintained. +This ordering also affects [issue lists](issues/sorting_issue_lists.md). +Changing the order in an issue board changes the ordering in an issue list, +and vice versa. + ### Filtering issues You should be able to use the filters on top of your Issue Board to show only diff --git a/doc/user/project/issues/confidential_issues.md b/doc/user/project/issues/confidential_issues.md index 2c755e0fb4d..c10ef564f0d 100644 --- a/doc/user/project/issues/confidential_issues.md +++ b/doc/user/project/issues/confidential_issues.md @@ -77,3 +77,49 @@ project's search results respectively. | Maintainer access | Guest access | | :-----------: | :----------: | |  |  | + +## Merge Requests for Confidential Issues + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/58583) in GitLab 12.1. + +To help prevent confidential information being leaked from a public project +in the process of resolving a confidential issue, confidential issues can be +resolved by creating a merge request from a private fork. + +The merge request created will target the default branch of the private fork, +not the default branch of the public upstream project. This prevents the merge +request, branch, and commits entering the public repository, and revealing +confidential information prematurely. When the confidential commits are ready +to be made public, this can be done by opening a merge request from the private +fork to the public upstream project. + +TIP: **Best practice:** +If you create a long-lived private fork in the same group or in a sub-group of +the original upstream, all the users with Developer membership to the public +project will also have the same permissions in the private project. This way, +all the Developers, who have access to view confidential issues, will have a +streamlined workflow for fixing them. + +### How it works + +On a confidential issue, a **Create confidential merge request** button is +available. Clicking on it will open a dropdown where you can choose to +**Create confidential merge request and branch** or **Create branch**: + +| Create confidential merge request | Create branch | +| :-------------------------------: | :-----------: | +|  |  | + +The **Project** dropdown includes the list of private forks the user is a member +of as at least a Developer and merge requests are enabled. + +Whenever the **Branch name** and **Source (branch or tag)** fields change, the +availability of the target or source branch will be checked. Both branches should +be available in the private fork selected. + +By clicking the **Create confidential merge request** button, GitLab will create +the branch and merge request in the private fork. When you choose +**Create branch**, GitLab will only create the branch. + +Once the branch is created in the private fork, developers can now push code to +that branch to fix the confidential issue. diff --git a/doc/user/project/issues/img/confidential_mr_branch_dropdown_v12_1.png b/doc/user/project/issues/img/confidential_mr_branch_dropdown_v12_1.png Binary files differnew file mode 100644 index 00000000000..7c24226a6c4 --- /dev/null +++ b/doc/user/project/issues/img/confidential_mr_branch_dropdown_v12_1.png diff --git a/doc/user/project/issues/img/confidential_mr_dropdown_v12_1.png b/doc/user/project/issues/img/confidential_mr_dropdown_v12_1.png Binary files differnew file mode 100644 index 00000000000..d6d391c6dd9 --- /dev/null +++ b/doc/user/project/issues/img/confidential_mr_dropdown_v12_1.png diff --git a/doc/user/project/issues/index.md b/doc/user/project/issues/index.md index e917697e973..6706e1c2e2b 100644 --- a/doc/user/project/issues/index.md +++ b/doc/user/project/issues/index.md @@ -89,7 +89,7 @@ Key actions for Issues include:  On an issue's page, you can view [all aspects of the issue](issue_data_and_actions.md), -and modify them if you you have the necessary [permissions](../../permissions.md). +and modify them if you have the necessary [permissions](../../permissions.md). ### Issues list @@ -104,7 +104,8 @@ view, you can also make certain changes [in bulk](../bulk_editing.md) to the dis For more information, see the [Issue Data and Actions](issue_data_and_actions.md) page for a rundown of all the fields and information in an issue. -For sorting by issue priority, see [Label Priority](../labels.md#label-priority). +You can sort a list of issues several ways, including by issue creation date, milestone due date, +etc. For more information, see the [Sorting and Ordering Issue Lists](sorting_issue_lists.md) page. ### Issue boards diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 709588959c1..e9b4e591384 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -7,7 +7,9 @@ you can do with issues. ## Create a new Issue -When you create a new issue, you'll be prompted to fill in the [data and fields of the issue](issue_data_and_actions.md#parts-of-an-issue), as illustrated below. +When you create a new issue, you'll be prompted to fill in the [data and fields of the issue](issue_data_and_actions.md#parts-of-an-issue), as illustrated below. If you know +the values you want to assign to an issue, you can use the [Quick actions](../quick_actions.md) +feature to input values, instead of selecting them from lists.  diff --git a/doc/user/project/issues/sorting_issue_lists.md b/doc/user/project/issues/sorting_issue_lists.md new file mode 100644 index 00000000000..ba120783430 --- /dev/null +++ b/doc/user/project/issues/sorting_issue_lists.md @@ -0,0 +1,30 @@ +# Sorting and Ordering Issue Lists + +You can sort a list of issues several ways, including by issue creation date, milestone due date, +etc. The available sorting options can change based on the context of the list. +For sorting by issue priority, see [Label Priority](../labels.md#label-priority). + +In group and project issue lists, it is also possible to order issues manually, +similar to [issue boards](../issue_board.md#issue-ordering-in-a-list). + +## Manual sorting + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/62178) in GitLab 12.1. + +When you select **Manual** sorting, you can change +the order by dragging and dropping the issues. The changed order will persist. Everyone who visits the same list will see the reordered list, with some exceptions. + +Each issue is assigned a relative order value, representing its relative +order with respect to the other issues in the list. When you drag-and-drop reorder +an issue, its relative order value changes accordingly. + +In addition, any time that issue appears in a manually sorted list, +the updated relative order value will be used for the ordering. This means that +if issue `A` is drag-and-drop reordered to be above issue `B` by any user in +a given list inside your GitLab instance, any time those two issues are subsequently +loaded in any list in the same instance (could be a different project issue list or a +different group issue list, for example), that ordering will be maintained. + +This ordering also affects [issue boards](../issue_board.md#issue-ordering-in-a-list). +Changing the order in an issue list changes the ordering in an issue board, +and vice versa. diff --git a/doc/user/project/merge_requests/index.md b/doc/user/project/merge_requests/index.md index f593046fa8b..8c28e24683f 100644 --- a/doc/user/project/merge_requests/index.md +++ b/doc/user/project/merge_requests/index.md @@ -365,6 +365,11 @@ have been marked as a **Work In Progress**. [Learn more about setting a merge request as "Work In Progress".](work_in_progress_merge_requests.md) +## Merge Requests for Confidential Issues + +Create [merge requests to resolve confidential issues](../issues/confidential_issues.md#merge-requests-for-confidential-issues) +for preventing leakage or early release of sentive data through regular merge requests. + ## Merge request approvals **(STARTER)** > Included in [GitLab Starter][products]. @@ -386,6 +391,17 @@ can show the Code Climate report right in the merge request widget area. [Read more about Code Quality reports.](code_quality.md) +## Metrics Reports **(PREMIUM)** + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/9788) in [GitLab Premium][products] 11.10. +Requires GitLab Runner 11.10 and above. + +If you are using [GitLab CI][ci], you can configure your job to output custom +metrics and GitLab will display the Metrics Report on the merge request so +that it's fast and easy to identify changes to important metrics. + +[Read more about Metrics Report](../../../ci/metrics_reports.md). + ## Browser Performance Testing **(PREMIUM)** > Introduced in [GitLab Premium][products] 10.3. diff --git a/doc/user/project/pages/introduction.md b/doc/user/project/pages/introduction.md index 9451b5349c0..e8984cb8b9f 100644 --- a/doc/user/project/pages/introduction.md +++ b/doc/user/project/pages/introduction.md @@ -83,23 +83,23 @@ You can enable Pages access control on your project, so that only 1. Navigate to your project's **Settings > General > Permissions**. 1. Toggle the **Pages** button to enable the access control. - NOTE: **Note:** - If you don't see the toggle button, that means that it's not enabled. - Ask your administrator to [enable it](../../../administration/pages/index.md#access-control). + NOTE: **Note:** + If you don't see the toggle button, that means that it's not enabled. + Ask your administrator to [enable it](../../../administration/pages/index.md#access-control). 1. The Pages access control dropdown allows you to set who can view pages hosted with GitLab Pages, depending on your project's visibility: - - If your project is private: - - **Only project members**: Only project members will be able to browse the website. - - **Everyone**: Everyone, both logged into and logged out of GitLab, will be able to browse the website, no matter their project membership. - - If your project is internal: - - **Only project members**: Only project members will be able to browse the website. - - **Everyone with access**: Everyone logged into GitLab will be able to browse the website, no matter their project membership. - - **Everyone**: Everyone, both logged into and logged out of GitLab, will be able to browse the website, no matter their project membership. - - If your project is public: - - **Only project members**: Only project members will be able to browse the website. - - **Everyone with access**: Everyone, both logged into and logged out of GitLab, will be able to browse the website, no matter their project membership. + - If your project is private: + - **Only project members**: Only project members will be able to browse the website. + - **Everyone**: Everyone, both logged into and logged out of GitLab, will be able to browse the website, no matter their project membership. + - If your project is internal: + - **Only project members**: Only project members will be able to browse the website. + - **Everyone with access**: Everyone logged into GitLab will be able to browse the website, no matter their project membership. + - **Everyone**: Everyone, both logged into and logged out of GitLab, will be able to browse the website, no matter their project membership. + - If your project is public: + - **Only project members**: Only project members will be able to browse the website. + - **Everyone with access**: Everyone, both logged into and logged out of GitLab, will be able to browse the website, no matter their project membership. 1. Click **Save changes**. diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index 948ac91a886..f81669a41c9 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -59,6 +59,12 @@ discussions, and descriptions: | `/create_merge_request <branch name>` | Create a new merge request starting from the current issue | ✓ | | | `/relate #issue1 #issue2` | Mark issues as related **(STARTER)** | ✓ | | +## Autocomplete characters + +Many quick actions require a parameter, for example: username, milestone, and +label. [Autocomplete characters](autocomplete_characters.md) can make it easier +to enter a parameter, compared to selecting items from a list. + ## Quick actions for commit messages The following quick actions are applicable for commit messages: diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md index 7241df613eb..35d5320c0b1 100644 --- a/doc/user/project/settings/import_export.md +++ b/doc/user/project/settings/import_export.md @@ -116,3 +116,8 @@ For more details on the specific data persisted in a project export, see the 1. Click on **Import project** to begin importing. Your newly imported project page will appear soon. + +NOTE: **Note:** +If use of the `Internal` visibility level +[is restricted](../../../public_access/public_access.md#restricting-the-use-of-public-or-internal-projects), +all imported projects are given the visibility of `Private`. diff --git a/doc/user/search/advanced_search_syntax.md b/doc/user/search/advanced_search_syntax.md index ad9f065b19b..c3d22e4fd29 100644 --- a/doc/user/search/advanced_search_syntax.md +++ b/doc/user/search/advanced_search_syntax.md @@ -50,9 +50,9 @@ here's a quick guide: The Advanced Syntax Search also supports the use of filters. The available filters are: - - filename: Filters by filename. You can use the glob (`*`) operator for fuzzy matching. - - path: Filters by path. You can use the glob (`*`) operator for fuzzy matching. - - extension: Filters by extension in the filename. Please write the extension without a leading dot. Exact match only. +- filename: Filters by filename. You can use the glob (`*`) operator for fuzzy matching. +- path: Filters by path. You can use the glob (`*`) operator for fuzzy matching. +- extension: Filters by extension in the filename. Please write the extension without a leading dot. Exact match only. To use them, simply add them to your query in the format `<filter_name>:<value>` without any spaces between the colon (`:`) and the value. diff --git a/doc/user/search/index.md b/doc/user/search/index.md index c34b9ae3d7e..8d7b4a429aa 100644 --- a/doc/user/search/index.md +++ b/doc/user/search/index.md @@ -55,12 +55,12 @@ Selecting **Any** does the opposite. It returns results that have a non-empty va You can filter issues and merge requests by specific terms included in titles or descriptions. - Syntax - - Searches look for all the words in a query, in any order. E.g.: searching - issues for `display bug` will return all issues matching both those words, in any order. - - To find the exact term, use double quotes: `"display bug"` + - Searches look for all the words in a query, in any order. E.g.: searching + issues for `display bug` will return all issues matching both those words, in any order. + - To find the exact term, use double quotes: `"display bug"` - Limitation - - For performance reasons, terms shorter than 3 chars are ignored. E.g.: searching - issues for `included in titles` is same as `included titles` + - For performance reasons, terms shorter than 3 chars are ignored. E.g.: searching + issues for `included in titles` is same as `included titles`  diff --git a/doc/workflow/forking_workflow.md b/doc/workflow/forking_workflow.md index 02be0ad191d..869a0a621c3 100644 --- a/doc/workflow/forking_workflow.md +++ b/doc/workflow/forking_workflow.md @@ -10,31 +10,25 @@ document more information about using branches to work together. Forking a project is in most cases a two-step process. -1. Click on the fork button located in the middle of the page or a project's - home page right next to the stars button. +1. Click on the fork button located in the middle of the page or a project's + home page right next to the stars button. -  +  - --- +1. Once you do that, you'll be presented with a screen where you can choose + the namespace to fork to. Only namespaces (groups and your own + namespace) where you have write access to, will be shown. Click on the + namespace to create your fork there. -1. Once you do that, you'll be presented with a screen where you can choose - the namespace to fork to. Only namespaces (groups and your own - namespace) where you have write access to, will be shown. Click on the - namespace to create your fork there. +  -  + **Note:** + If the namespace you chose to fork the project to has another project with + the same path name, you will be presented with a warning that the forking + could not be completed. Try to resolve the error and repeat the forking + process. - --- - - **Note:** - If the namespace you chose to fork the project to has another project with - the same path name, you will be presented with a warning that the forking - could not be completed. Try to resolve the error and repeat the forking - process. - -  - - --- +  After the forking is done, you can start working on the newly created repository. There, you will have full [Owner](../user/permissions.md) diff --git a/doc/workflow/lfs/lfs_administration.md b/doc/workflow/lfs/lfs_administration.md index 55183408a10..3160b0c4deb 100644 --- a/doc/workflow/lfs/lfs_administration.md +++ b/doc/workflow/lfs/lfs_administration.md @@ -91,6 +91,7 @@ Here is a configuration example with S3. | `aws_access_key_id` | AWS credentials, or compatible | `ABC123DEF456` | | `aws_secret_access_key` | AWS credentials, or compatible | `ABC123DEF456ABC123DEF456ABC123DEF456` | | `aws_signature_version` | AWS signature version to use. 2 or 4 are valid options. Digital Ocean Spaces and other providers may need 2. | 4 | +| `enable_signature_v4_streaming` | Set to true to enable HTTP chunked transfers with AWS v4 signatures (https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be false | true | `region` | AWS region | us-east-1 | | `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com` | s3.amazonaws.com | | `endpoint` | Can be used when configuring an S3 compatible service such as [Minio](https://www.minio.io), by entering a URL such as `http://127.0.0.1:9000` | (optional) | diff --git a/doc/workflow/repository_mirroring.md b/doc/workflow/repository_mirroring.md index 87ca46e94be..0b8e7d851b0 100644 --- a/doc/workflow/repository_mirroring.md +++ b/doc/workflow/repository_mirroring.md @@ -92,9 +92,9 @@ The repository will push soon. To force a push, click the appropriate button. 1. On the destination GitLab instance, create a [personal access token](../user/profile/personal_access_tokens.md) with `API` scope. 1. On the source GitLab instance: - 1. Fill in the **Git repository URL** field using this format: `https://oauth2@<destination host>/<your_gitlab_group_or_name>/<your_gitlab_project>.git`. - 1. Fill in **Password** field with the GitLab personal access token created on the destination GitLab instance. - 1. Click the **Mirror repository** button. + 1. Fill in the **Git repository URL** field using this format: `https://oauth2@<destination host>/<your_gitlab_group_or_name>/<your_gitlab_project>.git`. + 1. Fill in **Password** field with the GitLab personal access token created on the destination GitLab instance. + 1. Click the **Mirror repository** button. ## Pulling from a remote repository **(STARTER)** @@ -118,9 +118,9 @@ To configure mirror pulling for an existing project: 1. Select **Pull** from the **Mirror direction** dropdown. 1. Select an authentication method from the **Authentication method** dropdown, if necessary. 1. If necessary, check the following boxes: - - **Overwrite diverged branches**. - - **Trigger pipelines for mirror updates**. - - **Only mirror protected branches**. + - **Overwrite diverged branches**. + - **Trigger pipelines for mirror updates**. + - **Only mirror protected branches**. 1. Click the **Mirror repository** button to save the configuration.  diff --git a/lib/api/commits.rb b/lib/api/commits.rb index fe910d46f6c..e4f4e79cd46 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -76,7 +76,7 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide either `start_branch` or `start_sha`, and optionally `start_project`.', allow_blank: false requires :commit_message, type: String, desc: 'Commit message' requires :actions, type: Array, desc: 'Actions to perform in commit' do requires :action, type: String, desc: 'The action to perform, `create`, `delete`, `move`, `update`, `chmod`', values: %w[create update move delete chmod].freeze @@ -98,12 +98,16 @@ module API requires :execute_filemode, type: Boolean, desc: 'When `true/false` enables/disables the execute flag on the file.' end end - optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' - optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the commit from' + + optional :start_branch, type: String, desc: 'Name of the branch to start the new branch from' + optional :start_sha, type: String, desc: 'SHA of the commit to start the new branch from' + mutually_exclusive :start_branch, :start_sha + + optional :start_project, types: [Integer, String], desc: 'The ID or path of the project to start the new branch from' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' optional :stats, type: Boolean, default: true, desc: 'Include commit stats' - optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch`' + optional :force, type: Boolean, default: false, desc: 'When `true` overwrites the target branch with a new commit based on the `start_branch` or `start_sha`' end post ':id/repository/commits' do if params[:start_project] @@ -118,7 +122,7 @@ module API attrs = declared_params attrs[:branch_name] = attrs.delete(:branch) - attrs[:start_branch] ||= attrs[:branch_name] + attrs[:start_branch] ||= attrs[:branch_name] unless attrs[:start_sha] attrs[:start_project] = start_project if start_project result = ::Files::MultiService.new(user_project, current_user, attrs).execute diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 833e3b9ebaf..51b7cf05c8f 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -71,6 +71,7 @@ module API :build_timeout, :builds_access_level, :ci_config_path, + :ci_default_git_depth, :container_registry_enabled, :default_branch, :description, diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb new file mode 100644 index 00000000000..4f47cdef971 --- /dev/null +++ b/lib/gitlab/chaos.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + # Chaos methods for GitLab. + # See https://docs.gitlab.com/ee/development/chaos_endpoints.html for more details. + class Chaos + # leak_mem will retain the specified amount of memory and sleep. + # On return, the memory will be released. + def self.leak_mem(memory_mb, duration_s) + start_time = Time.now + + retainer = [] + # Add `n` 1mb chunks of memory to the retainer array + memory_mb.times { retainer << "x" * 1.megabyte } + + duration_left = [start_time + duration_s - Time.now, 0].max + Kernel.sleep(duration_left) + end + + # cpu_spin will consume all CPU on a single core for the specified duration + def self.cpu_spin(duration_s) + expected_end_time = Time.now + duration_s + + rand while Time.now < expected_end_time + end + + # db_spin will query the database in a tight loop for the specified duration + def self.db_spin(duration_s, interval_s) + expected_end_time = Time.now + duration_s + + while Time.now < expected_end_time + ActiveRecord::Base.connection.execute("SELECT 1") + + end_interval_time = Time.now + [duration_s, interval_s].min + rand while Time.now < end_interval_time + end + end + + # sleep will sleep for the specified duration + def self.sleep(duration_s) + Kernel.sleep(duration_s) + end + + # Kill will send a SIGKILL signal to the current process + def self.kill + Process.kill("KILL", Process.pid) + end + end +end diff --git a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml index a3db2705bf6..280e75d46f5 100644 --- a/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Serverless.gitlab-ci.yml @@ -8,26 +8,23 @@ stages: - deploy .serverless:build:image: - stage: build image: registry.gitlab.com/gitlab-org/gitlabktl:latest + stage: build script: /usr/bin/gitlabktl app build .serverless:deploy:image: + image: registry.gitlab.com/gitlab-org/gitlabktl:latest stage: deploy - image: gcr.io/triggermesh/tm@sha256:3cfdd470a66b741004fb02354319d79f1598c70117ce79978d2e07e192bfb336 # v0.0.11 environment: development - script: - - echo "$CI_REGISTRY_IMAGE" - - tm -n "$KUBE_NAMESPACE" --config "$KUBECONFIG" deploy service "$CI_PROJECT_NAME" --from-image "$CI_REGISTRY_IMAGE" --wait + script: /usr/bin/gitlabktl app deploy .serverless:build:functions: - stage: build - environment: development image: registry.gitlab.com/gitlab-org/gitlabktl:latest + stage: build script: /usr/bin/gitlabktl serverless build .serverless:deploy:functions: + image: registry.gitlab.com/gitlab-org/gitlabktl:latest stage: deploy environment: development - image: registry.gitlab.com/gitlab-org/gitlabktl:latest script: /usr/bin/gitlabktl serverless deploy diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 0fc145534bf..c0a12318990 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -46,6 +46,16 @@ module Gitlab ee? ? 'gitlab-ee' : 'gitlab-ce' end + def markdown_list(items) + list = items.map { |item| "* `#{item}`" }.join("\n") + + if items.size > 10 + "\n<details>\n\n#{list}\n\n</details>\n" + else + list + end + end + # @return [Hash<String,Array<String>>] def changes_by_category all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| @@ -132,6 +142,22 @@ module Gitlab def new_teammates(usernames) usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } end + + def missing_database_labels(current_mr_labels) + labels = if has_database_scoped_labels?(current_mr_labels) + ['database'] + else + ['database', 'database::review pending'] + end + + labels - current_mr_labels + end + + private + + def has_database_scoped_labels?(current_mr_labels) + current_mr_labels.any? { |label| label.start_with?('database::') } + end end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 0c5f33e1b2a..2ae807697bc 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -6,31 +6,45 @@ module Gitlab BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time + PERMITTED_TIMESTAMP_COLUMNS = %i[created_at updated_at deleted_at].to_set.freeze + DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze + # Adds `created_at` and `updated_at` columns with timezone information. # # This method is an improved version of Rails' built-in method `add_timestamps`. # + # By default, adds `created_at` and `updated_at` columns, but these can be specified as: + # + # add_timestamps_with_timezone(:my_table, columns: [:created_at, :deleted_at]) + # + # This allows you to create just the timestamps you need, saving space. + # # Available options are: - # default - The default value for the column. - # null - When set to `true` the column will allow NULL values. + # :default - The default value for the column. + # :null - When set to `true` the column will allow NULL values. # The default is to not allow NULL values. + # :columns - the column names to create. Must be one + # of `Gitlab::Database::MigrationHelpers::PERMITTED_TIMESTAMP_COLUMNS`. + # Default value: `DEFAULT_TIMESTAMP_COLUMNS` + # + # All options are optional. def add_timestamps_with_timezone(table_name, options = {}) options[:null] = false if options[:null].nil? + columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS) + default_value = options[:default] - [:created_at, :updated_at].each do |column_name| - if options[:default] && transaction_open? - raise '`add_timestamps_with_timezone` with default value cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` ' \ - 'in the body of your migration class' - end + validate_not_in_transaction!(:add_timestamps_with_timezone, 'with default value') if default_value + + columns.each do |column_name| + validate_timestamp_column_name!(column_name) # If default value is presented, use `add_column_with_default` method instead. - if options[:default] + if default_value add_column_with_default( table_name, column_name, :datetime_with_timezone, - default: options[:default], + default: default_value, allow_null: options[:null] ) else @@ -39,6 +53,21 @@ module Gitlab end end + # To be used in the `#down` method of migrations that + # use `#add_timestamps_with_timezone`. + # + # Available options are: + # :columns - the column names to remove. Must be one + # Default value: `DEFAULT_TIMESTAMP_COLUMNS` + # + # All options are optional. + def remove_timestamps(table_name, options = {}) + columns = options.fetch(:columns, DEFAULT_TIMESTAMP_COLUMNS) + columns.each do |column_name| + remove_column(table_name, column_name) + end + end + # Creates a new index, concurrently when supported # # On PostgreSQL this method creates an index concurrently, on MySQL this @@ -1098,6 +1127,28 @@ into similar problems in the future (e.g. when new tables are created). def mysql_compatible_index_length Gitlab::Database.mysql? ? 20 : nil end + + private + + def validate_timestamp_column_name!(column_name) + return if PERMITTED_TIMESTAMP_COLUMNS.member?(column_name) + + raise <<~MESSAGE + Illegal timestamp column name! Got #{column_name}. + Must be one of: #{PERMITTED_TIMESTAMP_COLUMNS.to_a} + MESSAGE + end + + def validate_not_in_transaction!(method_name, modifier = nil) + return unless transaction_open? + + raise <<~ERROR + #{["`#{method_name}`", modifier].compact.join(' ')} cannot be run inside a transaction. + + You can disable transactions by calling `disable_ddl_transaction!` in the body of + your migration class + ERROR + end end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 44a62586a23..df9f33baec2 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -9,6 +9,7 @@ module Gitlab # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze BLANK_SHA = ('0' * 40).freeze + COMMIT_ID = /\A[0-9a-f]{40}\z/.freeze TAG_REF_PREFIX = "refs/tags/".freeze BRANCH_REF_PREFIX = "refs/heads/".freeze @@ -65,6 +66,10 @@ module Gitlab ref == BLANK_SHA end + def commit_id?(ref) + COMMIT_ID.match?(ref) + end + def version Gitlab::Git::Version.git_version end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index a7d9ba51277..6e8aa5d578e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -873,13 +873,13 @@ module Gitlab def multi_action( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, - start_branch_name: nil, start_repository: self, + start_branch_name: nil, start_sha: nil, start_repository: self, force: false) wrapped_gitaly_errors do gitaly_operation_client.user_commit_files(user, branch_name, message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) end end # rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index 902fa3c7822..badf943e39c 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -13,7 +13,18 @@ module Gitlab def wrap_rugged_call(&block) Gitlab::GitalyClient::StorageSettings.allow_disk_access do - yield + start = Gitlab::Metrics::System.monotonic_time + + result = yield + + duration = Gitlab::Metrics::System.monotonic_time - start + + if Gitlab::RuggedInstrumentation.active? + Gitlab::RuggedInstrumentation.increment_query_count + Gitlab::RuggedInstrumentation.query_time += duration + end + + result end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index e7319a7b7f0..c98de722fe1 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -392,7 +392,7 @@ module Gitlab @can_use_disk[storage] end - return cached_value if cached_value.present? + return cached_value unless cached_value.nil? gitaly_filesystem_id = filesystem_id(storage) direct_filesystem_id = filesystem_id_from_disk(storage) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 783c2ff0915..33ca428a942 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -325,11 +325,11 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force = false) + start_branch_name, start_repository, force = false, start_sha = nil) req_enum = Enumerator.new do |y| header = user_commit_files_request_header(user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) y.yield Gitaly::UserCommitFilesRequest.new(header: header) @@ -445,7 +445,7 @@ module Gitlab # rubocop:disable Metrics/ParameterLists def user_commit_files_request_header( user, branch_name, commit_message, actions, author_email, author_name, - start_branch_name, start_repository, force) + start_branch_name, start_repository, force, start_sha) Gitaly::UserCommitFilesRequestHeader.new( repository: @gitaly_repo, @@ -456,7 +456,8 @@ module Gitlab commit_author_email: encode_binary(author_email), start_branch_name: encode_binary(start_branch_name), start_repository: start_repository.gitaly_repository, - force: force + force: force, + start_sha: encode_binary(start_sha) ) end # rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/grape_logging/loggers/perf_logger.rb b/lib/gitlab/grape_logging/loggers/perf_logger.rb index 18ea3a8d2f3..7e86b35a215 100644 --- a/lib/gitlab/grape_logging/loggers/perf_logger.rb +++ b/lib/gitlab/grape_logging/loggers/perf_logger.rb @@ -6,11 +6,30 @@ module Gitlab module Loggers class PerfLogger < ::GrapeLogging::Loggers::Base def parameters(_, _) + gitaly_data.merge(rugged_data) + end + + def gitaly_data + gitaly_calls = Gitlab::GitalyClient.get_request_count + + return {} if gitaly_calls.zero? + { gitaly_calls: Gitlab::GitalyClient.get_request_count, gitaly_duration: Gitlab::GitalyClient.query_time_ms } end + + def rugged_data + rugged_calls = Gitlab::RuggedInstrumentation.query_count + + return {} if rugged_calls.zero? + + { + rugged_calls: rugged_calls, + rugged_duration_ms: Gitlab::RuggedInstrumentation.query_time_ms + } + end end end end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index dec99c23a2d..91fe4e5d074 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -130,6 +130,7 @@ module Gitlab def visibility_level level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level level = @project.group.visibility_level if @project.group && level.to_i > @project.group.visibility_level + level = Gitlab::VisibilityLevel::PRIVATE if level == Gitlab::VisibilityLevel::INTERNAL && Gitlab::CurrentSettings.restricted_visibility_levels.include?(level) { 'visibility_level' => level } end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index eef802caabb..79f756c8f8a 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -7,7 +7,7 @@ module Gitlab module Samplers class RubySampler < BaseSampler def initialize(interval) - metrics[:process_start_time_seconds].set(labels.merge(worker_label), Time.now.to_i) + metrics[:process_start_time_seconds].set(labels, Time.now.to_i) super end @@ -50,9 +50,9 @@ module Gitlab def sample start_time = System.monotonic_time - metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) - metrics[:process_cpu_seconds_total].set(labels.merge(worker_label), ::Gitlab::Metrics::System.cpu_time) - metrics[:process_max_fds].set(labels.merge(worker_label), ::Gitlab::Metrics::System.max_open_file_descriptors) + metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:process_cpu_seconds_total].set(labels, ::Gitlab::Metrics::System.cpu_time) + metrics[:process_max_fds].set(labels, ::Gitlab::Metrics::System.max_open_file_descriptors) set_memory_usage_metrics sample_gc @@ -75,22 +75,9 @@ module Gitlab def set_memory_usage_metrics memory_usage = System.memory_usage - memory_labels = labels.merge(worker_label) - metrics[:memory_bytes].set(memory_labels, memory_usage) - metrics[:process_resident_memory_bytes].set(memory_labels, memory_usage) - end - - def worker_label - return { worker: 'sidekiq' } if Sidekiq.server? - return {} unless defined?(Unicorn::Worker) - - worker_no = ::Prometheus::Client::Support::Unicorn.worker_id - if worker_no - { worker: worker_no } - else - { worker: 'master' } - end + metrics[:memory_bytes].set(labels, memory_usage) + metrics[:process_resident_memory_bytes].set(labels, memory_usage) end end end diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index 01db507761b..2ee7144fe2f 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -50,7 +50,8 @@ module Gitlab def observe(key, duration) return unless current_transaction - metric_cache_operation_duration_seconds.observe(current_transaction.labels.merge({ operation: key }), duration / 1000.0) + metric_cache_operations_total.increment(current_transaction.labels.merge({ operation: key })) + metric_cache_operation_duration_seconds.observe({ operation: key }, duration / 1000.0) current_transaction.increment(:cache_duration, duration, false) current_transaction.increment(:cache_count, 1, false) current_transaction.increment("cache_#{key}_duration".to_sym, duration, false) @@ -63,12 +64,20 @@ module Gitlab Transaction.current end + def metric_cache_operations_total + @metric_cache_operations_total ||= ::Gitlab::Metrics.counter( + :gitlab_cache_operations_total, + 'Cache operations', + Transaction::BASE_LABELS + ) + end + def metric_cache_operation_duration_seconds @metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram( :gitlab_cache_operation_duration_seconds, 'Cache access time', - Transaction::BASE_LABELS.merge({ action: nil }), - [0.001, 0.01, 0.1, 1, 10] + {}, + [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0] ) end diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 7615f6f443b..99958d7a211 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'ruby-prof' +require 'memory_profiler' module Gitlab module RequestProfiler @@ -28,22 +29,73 @@ module Gitlab end def call_with_profiling(env) + case env['HTTP_X_PROFILE_MODE'] + when 'execution', nil + call_with_call_stack_profiling(env) + when 'memory' + call_with_memory_profiling(env) + else + raise ActionController::BadRequest, invalid_profile_mode(env) + end + end + + def invalid_profile_mode(env) + <<~HEREDOC + Invalid X-Profile-Mode: #{env['HTTP_X_PROFILE_MODE']}. + Supported profile mode request header: + - X-Profile-Mode: execution + - X-Profile-Mode: memory + HEREDOC + end + + def call_with_call_stack_profiling(env) ret = nil - result = RubyProf::Profile.profile do + report = RubyProf::Profile.profile do ret = catch(:warden) do @app.call(env) end end - printer = RubyProf::CallStackPrinter.new(result) - file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}.html" + generate_report(env, 'execution', 'html') do |file| + printer = RubyProf::CallStackPrinter.new(report) + printer.print(file) + end + + handle_request_ret(ret) + end + + def call_with_memory_profiling(env) + ret = nil + report = MemoryProfiler.report do + ret = catch(:warden) do + @app.call(env) + end + end + + generate_report(env, 'memory', 'txt') do |file| + report.pretty_print(to_file: file) + end + + handle_request_ret(ret) + end + + def generate_report(env, report_type, extension) + file_name = "#{env['PATH_INFO'].tr('/', '|')}_#{Time.current.to_i}"\ + "_#{report_type}.#{extension}" file_path = "#{PROFILES_DIR}/#{file_name}" FileUtils.mkdir_p(PROFILES_DIR) - File.open(file_path, 'wb') do |file| - printer.print(file) + + begin + File.open(file_path, 'wb') do |file| + yield(file) + end + rescue + FileUtils.rm(file_path) end + end + def handle_request_ret(ret) if ret.is_a?(Array) ret else diff --git a/lib/gitlab/request_profiler/profile.rb b/lib/gitlab/request_profiler/profile.rb index 46996ef8c51..74f2ec1d083 100644 --- a/lib/gitlab/request_profiler/profile.rb +++ b/lib/gitlab/request_profiler/profile.rb @@ -3,28 +3,26 @@ module Gitlab module RequestProfiler class Profile - attr_reader :name, :time, :request_path + attr_reader :name, :time, :file_path, :request_path, :profile_mode, :type alias_method :to_param, :name def self.all - Dir["#{PROFILES_DIR}/*.html"].map do |path| + Dir["#{PROFILES_DIR}/*.{html,txt}"].map do |path| new(File.basename(path)) end end def self.find(name) - name_dup = name.dup - name_dup << '.html' unless name.end_with?('.html') - - file_path = "#{PROFILES_DIR}/#{name_dup}" + file_path = File.join(PROFILES_DIR, name) return unless File.exist?(file_path) - new(name_dup) + new(name) end def initialize(name) @name = name + @file_path = File.join(PROFILES_DIR, name) set_attributes end @@ -33,12 +31,23 @@ module Gitlab File.read("#{PROFILES_DIR}/#{name}") end + def content_type + case type + when 'html' + 'text/html' + when 'txt' + 'text/plain' + end + end + private def set_attributes - _, path, timestamp = name.split(/(.*)_(\d+)\.html$/) + _, path, timestamp, profile_mode, type = name.split(/(.*)_(\d+)_(.*)\.(html|txt)$/) @request_path = path.tr('|', '/') @time = Time.at(timestamp.to_i).utc + @profile_mode = profile_mode + @type = type end end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb new file mode 100644 index 00000000000..70c06e8b308 --- /dev/null +++ b/lib/gitlab/rugged_instrumentation.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module RuggedInstrumentation + def self.query_time + SafeRequestStore[:rugged_query_time] ||= 0 + end + + def self.query_time=(duration) + SafeRequestStore[:rugged_query_time] = duration + end + + def self.query_time_ms + (self.query_time * 1000).round(2) + end + + def self.query_count + SafeRequestStore[:rugged_call_count] ||= 0 + end + + def self.increment_query_count + SafeRequestStore[:rugged_call_count] ||= 0 + SafeRequestStore[:rugged_call_count] += 1 + end + + def self.active? + Gitlab::SafeRequestStore.active? + end + end +end diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb new file mode 100644 index 00000000000..1b87d3bb626 --- /dev/null +++ b/lib/gitlab/slug/environment.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# An environment name is not necessarily suitable for use in URLs, DNS +# or other third-party contexts, so provide a slugified version. A slug has +# the following properties: +# * contains only lowercase letters (a-z), numbers (0-9), and '-' +# * begins with a letter +# * has a maximum length of 24 bytes (OpenShift limitation) +# * cannot end with `-` +module Gitlab + module Slug + class Environment + attr_reader :name + + def initialize(name) + @name = name + end + + def generate + # Lowercase letters and numbers only + slugified = name.to_s.downcase.gsub(/[^a-z0-9]/, '-') + + # Must start with a letter + slugified = 'env-' + slugified unless slugified.match?(/^[a-z]/) + + # Repeated dashes are invalid (OpenShift limitation) + slugified.squeeze!('-') + + slugified = + if slugified.size > 24 || slugified != name + # Maximum length: 24 characters (OpenShift limitation) + shorten_and_add_suffix(slugified) + else + # Cannot end with a dash (Kubernetes label limitation) + slugified.chomp('-') + end + + slugified + end + + private + + def shorten_and_add_suffix(slug) + slug = slug[0..16] + slug << '-' unless slug.ends_with?('-') + slug << suffix + end + + # Slugifying a name may remove the uniqueness guarantee afforded by it being + # based on name (which must be unique). To compensate, we add a predictable + # 6-byte suffix in those circumstances. This is not *guaranteed* uniqueness, + # but the chance of collisions is vanishingly small + def suffix + Digest::SHA2.hexdigest(name.to_s).to_i(16).to_s(36).last(6) + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 7572c0bdbfd..db1086c9cae 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -6,7 +6,9 @@ module Gitlab class << self def data(force_refresh: false) - Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } + Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do + uncached_data + end end def uncached_data @@ -128,10 +130,15 @@ module Gitlab } end + # @return [Hash<Symbol, Integer>] def usage_counters - { - web_ide_commits: Gitlab::UsageDataCounters::WebIdeCounter.total_commits_count - } + usage_data_counters.map(&:totals).reduce({}) { |a, b| a.merge(b) } + end + + # @return [Array<#totals>] An array of objects that respond to `#totals` + def usage_data_counters + [Gitlab::UsageDataCounters::WikiPageCounter, + Gitlab::UsageDataCounters::WebIdeCounter] end def components_usage_data diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb index d10871f704c..75d5a75e3a4 100644 --- a/lib/gitlab/usage_data_counters/redis_counter.rb +++ b/lib/gitlab/usage_data_counters/redis_counter.rb @@ -4,6 +4,8 @@ module Gitlab module UsageDataCounters module RedisCounter def increment(redis_counter_key) + return unless Gitlab::CurrentSettings.usage_ping_enabled + Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb index 6fbffb94c58..0718c1dd761 100644 --- a/lib/gitlab/usage_data_counters/web_ide_counter.rb +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -6,6 +6,8 @@ module Gitlab extend RedisCounter COMMITS_COUNT_KEY = 'WEB_IDE_COMMITS_COUNT' + MERGE_REQUEST_COUNT_KEY = 'WEB_IDE_MERGE_REQUESTS_COUNT' + VIEWS_COUNT_KEY = 'WEB_IDE_VIEWS_COUNT' class << self def increment_commits_count @@ -15,6 +17,30 @@ module Gitlab def total_commits_count total_count(COMMITS_COUNT_KEY) end + + def increment_merge_requests_count + increment(MERGE_REQUEST_COUNT_KEY) + end + + def total_merge_requests_count + total_count(MERGE_REQUEST_COUNT_KEY) + end + + def increment_views_count + increment(VIEWS_COUNT_KEY) + end + + def total_views_count + total_count(VIEWS_COUNT_KEY) + end + + def totals + { + web_ide_commits: total_commits_count, + web_ide_views: total_views_count, + web_ide_merge_requests: total_merge_requests_count + } + end end end end diff --git a/lib/gitlab/usage_data_counters/wiki_page_counter.rb b/lib/gitlab/usage_data_counters/wiki_page_counter.rb new file mode 100644 index 00000000000..c8b59a3160c --- /dev/null +++ b/lib/gitlab/usage_data_counters/wiki_page_counter.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class WikiPageCounter + extend RedisCounter + + KNOWN_EVENTS = %w[create update delete].map(&:freeze).freeze + + UnknownEvent = Class.new(StandardError) + + class << self + # Each event gets a unique Redis key + def redis_key(event) + raise UnknownEvent, event unless KNOWN_EVENTS.include?(event.to_s) + + "USAGE_WIKI_PAGES_#{event}".upcase + end + + def count(event) + increment(redis_key event) + end + + def read(event) + total_count(redis_key event) + end + + def totals + KNOWN_EVENTS.map { |e| ["wiki_pages_#{e}".to_sym, read(e)] }.to_h + end + end + end +end diff --git a/lib/peek/views/redis.rb b/lib/peek/views/redis_detailed.rb index 73de8672fa4..12760c9b75e 100644 --- a/lib/peek/views/redis.rb +++ b/lib/peek/views/redis_detailed.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'redis' -require 'peek-redis' module Gitlab module Peek @@ -36,11 +35,42 @@ end module Peek module Views - module RedisDetailed + class RedisDetailed < View REDACTED_MARKER = "<redacted>" + def key + 'redis' + end + def results - super.merge(details: details) + { + calls: calls, + duration: formatted_duration, + details: details + } + end + + def detail_store + ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] + end + + private + + def formatted_duration + ms = duration * 1000 + if ms >= 1000 + "%.2fms" % ms + else + "%.0fms" % ms + end + end + + def duration + detail_store.map { |entry| entry[:duration] }.sum # rubocop:disable CodeReuse/ActiveRecord + end + + def calls + detail_store.count end def details @@ -49,10 +79,6 @@ module Peek .map(&method(:format_call_details)) end - def detail_store - ::Gitlab::SafeRequestStore['redis_call_details'] ||= [] - end - def format_call_details(call) call.merge(cmd: format_command(call[:cmd]), duration: (call[:duration] * 1000).round(3)) @@ -76,11 +102,3 @@ end class Redis::Client prepend Gitlab::Peek::RedisInstrumented end - -module Peek - module Views - class Redis < View - prepend Peek::Views::RedisDetailed - end - end -end diff --git a/lib/prometheus/pid_provider.rb b/lib/prometheus/pid_provider.rb new file mode 100644 index 00000000000..c92522c73c5 --- /dev/null +++ b/lib/prometheus/pid_provider.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'prometheus/client/support/unicorn' + +module Prometheus + module PidProvider + extend self + + def worker_id + if Sidekiq.server? + 'sidekiq' + elsif defined?(Unicorn::Worker) + "unicorn_#{unicorn_worker_id}" + elsif defined?(::Puma) + "puma_#{puma_worker_id}" + else + "process_#{Process.pid}" + end + end + + private + + # This is not fully accurate as we don't really know if the nil returned + # is actually means we're on master or not. + # Follow up issue was created to address this problem and + # to introduce more structrured approach to a current process discovery: + # https://gitlab.com/gitlab-org/gitlab-ce/issues/64740 + def unicorn_worker_id + ::Prometheus::Client::Support::Unicorn.worker_id || 'master' + end + + # See the comment for #unicorn_worker_id + def puma_worker_id + match = process_name.match(/cluster worker ([0-9]+):/) + match ? match[1] : 'master' + end + + def process_name + $0 + end + end +end diff --git a/lib/tasks/frontend.rake b/lib/tasks/frontend.rake new file mode 100644 index 00000000000..1cac7520227 --- /dev/null +++ b/lib/tasks/frontend.rake @@ -0,0 +1,21 @@ +unless Rails.env.production? + namespace :frontend do + desc 'GitLab | Frontend | Generate fixtures for JavaScript tests' + RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args| + args.with_defaults(pattern: '{spec,ee/spec}/frontend/fixtures/*.rb') + ENV['NO_KNAPSACK'] = 'true' + t.pattern = args[:pattern] + t.rspec_opts = '--format documentation' + end + + desc 'GitLab | Frontend | Run JavaScript tests' + task tests: ['yarn:check'] do + sh "yarn test" do |ok, res| + abort('rake frontend:tests failed') unless ok + end + end + end + + desc 'GitLab | Frontend | Shortcut for frontend:fixtures and frontend:tests' + task frontend: ['frontend:fixtures', 'frontend:tests'] +end diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake index 2dc14183fa3..36590010406 100644 --- a/lib/tasks/karma.rake +++ b/lib/tasks/karma.rake @@ -1,15 +1,8 @@ unless Rails.env.production? namespace :karma do + # alias exists for legacy reasons desc 'GitLab | Karma | Generate fixtures for JavaScript tests' - task fixtures: ['karma:rspec_fixtures'] - - desc 'GitLab | Karma | Generate fixtures using RSpec' - RSpec::Core::RakeTask.new(:rspec_fixtures, [:pattern]) do |t, args| - args.with_defaults(pattern: '{spec,ee/spec}/javascripts/fixtures/*.rb') - ENV['NO_KNAPSACK'] = 'true' - t.pattern = args[:pattern] - t.rspec_opts = '--format documentation' - end + task fixtures: ['frontend:fixtures'] desc 'GitLab | Karma | Run JavaScript tests' task tests: ['yarn:check'] do diff --git a/package.json b/package.json index 4ba9a0d9a1b..aebb52dd07c 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "@babel/preset-env": "^7.4.4", "@gitlab/csslab": "^1.9.0", "@gitlab/svgs": "^1.67.0", - "@gitlab/ui": "^5.6.0", + "@gitlab/ui": "^5.7.0", "apollo-cache-inmemory": "^1.5.1", "apollo-client": "^2.5.1", "apollo-link": "^1.2.11", diff --git a/qa/qa/page/main/login.rb b/qa/qa/page/main/login.rb index ffce3b72275..c2b0482d789 100644 --- a/qa/qa/page/main/login.rb +++ b/qa/qa/page/main/login.rb @@ -6,7 +6,7 @@ module QA class Login < Page::Base view 'app/views/devise/passwords/edit.html.haml' do element :password_field - element :password_confirmation + element :password_confirmation_field element :change_password_button end @@ -154,7 +154,7 @@ module QA return unless has_content?('Change your password') fill_element :password_field, Runtime::User.password - fill_element :password_confirmation, Runtime::User.password + fill_element :password_confirmation_field, Runtime::User.password click_element :change_password_button end end diff --git a/qa/qa/page/main/menu.rb b/qa/qa/page/main/menu.rb index c98d85d7911..d86d554356e 100644 --- a/qa/qa/page/main/menu.rb +++ b/qa/qa/page/main/menu.rb @@ -6,7 +6,7 @@ module QA class Menu < Page::Base view 'app/views/layouts/header/_current_user_dropdown.html.haml' do element :sign_out_link - element :settings_link, 'link_to s_("CurrentUser|Settings")' # rubocop:disable QA/ElementWithPattern + element :settings_link end view 'app/views/layouts/header/_default.html.haml' do diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb index 5eceeb9661c..425fb861456 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/check_mentions_for_xss_spec.rb @@ -3,17 +3,25 @@ module QA context 'Plan' do describe 'check xss occurence in @mentions in issues' do - let(:issue_title) { 'issue title' } + before do + QA::Runtime::Env.personal_access_token = QA::Runtime::Env.admin_personal_access_token - it 'user mentions a user in comment' do - Runtime::Browser.visit(:gitlab, Page::Main::Login) - Page::Main::Login.perform(&:sign_in_using_credentials) + unless QA::Runtime::Env.personal_access_token + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.perform(&:sign_in_using_admin_credentials) + end user = Resource::User.fabricate_via_api! do |user| user.name = "eve <img src=x onerror=alert(2)<img src=x onerror=alert(1)>" user.password = "test1234" end + QA::Runtime::Env.personal_access_token = nil + + Page::Main::Menu.perform(&:sign_out) if Page::Main::Menu.perform { |p| p.has_personal_area?(wait: 0) } + + Page::Main::Login.perform(&:sign_in_using_credentials) + project = Resource::Project.fabricate_via_api! do |resource| resource.name = 'xss-test-for-mentions-project' end @@ -25,17 +33,19 @@ module QA end issue = Resource::Issue.fabricate_via_api! do |issue| - issue.title = issue_title + issue.title = 'issue title' issue.project = project end issue.visit! + end - Page::Project::Issue::Show.perform do |show_page| - show_page.select_all_activities_filter - show_page.comment('cc-ing you here @eve') + it 'user mentions a user in comment' do + Page::Project::Issue::Show.perform do |show| + show.select_all_activities_filter + show.comment('cc-ing you here @eve') expect do - expect(show_page).to have_content("cc-ing you here") + expect(show).to have_content("cc-ing you here") end.not_to raise_error # Selenium::WebDriver::Error::UnhandledAlertError end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb index 2101311f065..ad70f6813fb 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/collapse_comments_in_discussions_spec.rb @@ -3,39 +3,40 @@ module QA context 'Plan' do describe 'collapse comments in issue discussions' do - let(:issue_title) { 'issue title' } + let(:my_first_reply) { 'My first reply' } - it 'user collapses reply for comments in an issue' do + before do Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.perform(&:sign_in_using_credentials) issue = Resource::Issue.fabricate_via_api! do |issue| - issue.title = issue_title + issue.title = 'issue title' end issue.visit! - expect(page).to have_content(issue_title) + Page::Project::Issue::Show.perform do |show| + my_first_discussion = 'My first discussion' - Page::Project::Issue::Show.perform do |show_page| - my_first_discussion = "My first discussion" - my_first_reply = "My First Reply" - one_reply = "1 reply" - - show_page.select_all_activities_filter - show_page.start_discussion(my_first_discussion) - expect(show_page).to have_content(my_first_discussion) + show.select_all_activities_filter + show.start_discussion(my_first_discussion) + page.assert_text(my_first_discussion) + show.reply_to_discussion(my_first_reply) + page.assert_text(my_first_reply) + end + end - show_page.reply_to_discussion(my_first_reply) - expect(show_page).to have_content(my_first_reply) + it 'user collapses and expands reply for comments in an issue' do + Page::Project::Issue::Show.perform do |show| + one_reply = "1 reply" - show_page.collapse_replies - expect(show_page).to have_content(one_reply) - expect(show_page).not_to have_content(my_first_reply) + show.collapse_replies + expect(show).to have_content(one_reply) + expect(show).not_to have_content(my_first_reply) - show_page.expand_replies - expect(show_page).to have_content(my_first_reply) - expect(show_page).not_to have_content(one_reply) + show.expand_replies + expect(show).to have_content(my_first_reply) + expect(show).not_to have_content(one_reply) end end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb index 2e71ecb5718..0b1bd00ac8d 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/comment_issue_spec.rb @@ -3,7 +3,7 @@ module QA context 'Plan' do describe 'Issue comments' do - it 'user comments on an issue and edits the comment' do + before do Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.perform(&:sign_in_using_credentials) @@ -11,19 +11,21 @@ module QA issue.title = 'issue title' end issue.visit! + end - Page::Project::Issue::Show.perform do |issue_show_page| + it 'user comments on an issue and edits the comment' do + Page::Project::Issue::Show.perform do |show| first_version_of_comment = 'First version of the comment' second_version_of_comment = 'Second version of the comment' - issue_show_page.comment(first_version_of_comment) + show.comment(first_version_of_comment) - expect(issue_show_page).to have_content(first_version_of_comment) + expect(show).to have_content(first_version_of_comment) - issue_show_page.edit_comment(second_version_of_comment) + show.edit_comment(second_version_of_comment) - expect(issue_show_page).to have_content(second_version_of_comment) - expect(issue_show_page).not_to have_content(first_version_of_comment) + expect(show).to have_content(second_version_of_comment) + expect(show).not_to have_content(first_version_of_comment) end end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb index 342f65593f6..04ae4963d3a 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/create_issue_spec.rb @@ -5,8 +5,15 @@ module QA describe 'Issue creation' do let(:issue_title) { 'issue title' } + before do + Runtime::Browser.visit(:gitlab, Page::Main::Login) + Page::Main::Login.perform(&:sign_in_using_credentials) + end + it 'user creates an issue' do - create_issue + Resource::Issue.fabricate_via_browser_ui! do |issue| + issue.title = issue_title + end Page::Project::Menu.perform(&:click_issues) @@ -18,9 +25,15 @@ module QA File.absolute_path(File.join('spec', 'fixtures', 'banana_sample.gif')) end - it 'user comments on an issue with an attachment' do - create_issue + before do + issue = Resource::Issue.fabricate_via_api! do |issue| + issue.title = issue_title + end + + issue.visit! + end + it 'user comments on an issue with an attachment' do Page::Project::Issue::Show.perform do |show| show.comment('See attached banana for scale', attachment: file_to_attach) @@ -36,15 +49,6 @@ module QA end end end - - def create_issue - Runtime::Browser.visit(:gitlab, Page::Main::Login) - Page::Main::Login.perform(&:sign_in_using_credentials) - - Resource::Issue.fabricate_via_browser_ui! do |issue| - issue.title = issue_title - end - end end end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/filter_issue_comments_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/filter_issue_comments_spec.rb index af9514008f9..317e31feea8 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/filter_issue_comments_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/filter_issue_comments_spec.rb @@ -5,7 +5,7 @@ module QA describe 'filter issue comments activities' do let(:issue_title) { 'issue title' } - it 'user filters comments and activities in an issue' do + before do Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.perform(&:sign_in_using_credentials) @@ -14,28 +14,28 @@ module QA end issue.visit! + end - expect(page).to have_content(issue_title) - - Page::Project::Issue::Show.perform do |show_page| + it 'user filters comments and activities in an issue' do + Page::Project::Issue::Show.perform do |show| my_own_comment = "My own comment" made_the_issue_confidential = "made the issue confidential" - show_page.comment('/confidential', filter: :comments_only) - show_page.comment(my_own_comment, filter: :comments_only) + show.comment('/confidential', filter: :comments_only) + show.comment(my_own_comment, filter: :comments_only) - expect(show_page).not_to have_content(made_the_issue_confidential) - expect(show_page).to have_content(my_own_comment) + expect(show).not_to have_content(made_the_issue_confidential) + expect(show).to have_content(my_own_comment) - show_page.select_all_activities_filter + show.select_all_activities_filter - expect(show_page).to have_content(made_the_issue_confidential) - expect(show_page).to have_content(my_own_comment) + expect(show).to have_content(made_the_issue_confidential) + expect(show).to have_content(my_own_comment) - show_page.select_history_only_filter + show.select_history_only_filter - expect(show_page).to have_content(made_the_issue_confidential) - expect(show_page).not_to have_content(my_own_comment) + expect(show).to have_content(made_the_issue_confidential) + expect(show).not_to have_content(my_own_comment) end end end diff --git a/qa/qa/specs/features/browser_ui/2_plan/issue/issue_suggestions_spec.rb b/qa/qa/specs/features/browser_ui/2_plan/issue/issue_suggestions_spec.rb index 24dcb32f63f..c42c2cedde0 100644 --- a/qa/qa/specs/features/browser_ui/2_plan/issue/issue_suggestions_spec.rb +++ b/qa/qa/specs/features/browser_ui/2_plan/issue/issue_suggestions_spec.rb @@ -5,7 +5,7 @@ module QA describe 'issue suggestions' do let(:issue_title) { 'Issue Lists are awesome' } - it 'user sees issue suggestions when creating a new issue' do + before do Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.perform(&:sign_in_using_credentials) @@ -20,14 +20,16 @@ module QA end project.visit! + end + it 'user sees issue suggestions when creating a new issue' do Page::Project::Show.perform(&:go_to_new_issue) - Page::Project::Issue::New.perform do |new_issue_page| - new_issue_page.add_title("issue") - expect(new_issue_page).to have_content(issue_title) + Page::Project::Issue::New.perform do |new| + new.add_title("issue") + expect(new).to have_content(issue_title) - new_issue_page.add_title("Issue Board") - expect(new_issue_page).not_to have_content(issue_title) + new.add_title("Issue Board") + expect(new).not_to have_content(issue_title) end end end diff --git a/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb b/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb index 7a36f9ea420..3ce291bf8bc 100644 --- a/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb +++ b/qa/qa/specs/features/browser_ui/3_create/merge_request/rebase_merge_request_spec.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true module QA - context 'Create' do + # Failure issue: https://gitlab.com/gitlab-org/quality/staging/issues/66 + context 'Create', :quarantine do describe 'Merge request rebasing' do it 'user rebases source branch of merge request' do Runtime::Browser.visit(:gitlab, Page::Main::Login) diff --git a/scripts/lint-rugged b/scripts/lint-rugged index 9466c62a415..e1605b4501b 100755 --- a/scripts/lint-rugged +++ b/scripts/lint-rugged @@ -10,7 +10,12 @@ ALLOWED = [ # Reverted Rugged calls due to Gitaly atop NFS performance # See https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code. 'lib/gitlab/git/rugged_impl/', - 'lib/gitlab/gitaly_client/storage_settings.rb' + 'lib/gitlab/gitaly_client/storage_settings.rb', + + # Needed for logging + 'config/initializers/lograge.rb', + 'lib/gitlab/grape_logging/loggers/perf_logger.rb', + 'lib/gitlab/rugged_instrumentation.rb' ].freeze rugged_lines = IO.popen(%w[git grep -i -n rugged -- app config lib], &:read).lines diff --git a/spec/controllers/admin/requests_profiles_controller_spec.rb b/spec/controllers/admin/requests_profiles_controller_spec.rb index 10850cb4603..289bb58c5a8 100644 --- a/spec/controllers/admin/requests_profiles_controller_spec.rb +++ b/spec/controllers/admin/requests_profiles_controller_spec.rb @@ -10,38 +10,63 @@ describe Admin::RequestsProfilesController do end describe '#show' do - let(:basename) { "profile_#{Time.now.to_i}.html" } let(:tmpdir) { Dir.mktmpdir('profiler-test') } let(:test_file) { File.join(tmpdir, basename) } - let(:profile) { Gitlab::RequestProfiler::Profile.new(basename) } - let(:sample_data) do - <<~HTML - <!DOCTYPE html> - <html> - <body> - <h1>My First Heading</h1> - <p>My first paragraph.</p> - </body> - </html> - HTML + + subject do + get :show, params: { name: basename } end before do stub_const('Gitlab::RequestProfiler::PROFILES_DIR', tmpdir) - output = File.open(test_file, 'w') - output.write(sample_data) - output.close + File.write(test_file, sample_data) end after do File.unlink(test_file) end - it 'loads an HTML profile' do - get :show, params: { name: basename } + context 'when loading HTML profile' do + let(:basename) { "profile_#{Time.now.to_i}_execution.html" } + + let(:sample_data) do + '<html> <body> <h1>Heading</h1> <p>paragraph.</p> </body> </html>' + end + + it 'renders the data' do + subject + + expect(response).to have_gitlab_http_status(200) + expect(response.body).to eq(sample_data) + end + end + + context 'when loading TXT profile' do + let(:basename) { "profile_#{Time.now.to_i}_memory.txt" } + + let(:sample_data) do + <<~TXT + Total allocated: 112096396 bytes (1080431 objects) + Total retained: 10312598 bytes (53567 objects) + TXT + end + + it 'renders the data' do + subject + + expect(response).to have_gitlab_http_status(200) + expect(response.body).to eq(sample_data) + end + end + + context 'when loading PDF profile' do + let(:basename) { "profile_#{Time.now.to_i}_anything.pdf" } + + let(:sample_data) { 'mocked pdf content' } - expect(response).to have_gitlab_http_status(200) - expect(response.body).to eq(sample_data) + it 'fails to render the data' do + expect { subject }.to raise_error(ActionController::UrlGenerationError, /No route matches.*unmatched constraints:/) + end end end end diff --git a/spec/controllers/chaos_controller_spec.rb b/spec/controllers/chaos_controller_spec.rb new file mode 100644 index 00000000000..bafd4a70862 --- /dev/null +++ b/spec/controllers/chaos_controller_spec.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe ChaosController do + describe '#leakmem' do + it 'calls synchronously' do + expect(Gitlab::Chaos).to receive(:leak_mem).with(100, 30.seconds) + + get :leakmem + + expect(response).to have_gitlab_http_status(200) + end + + it 'call synchronously with params' do + expect(Gitlab::Chaos).to receive(:leak_mem).with(1, 2.seconds) + + get :leakmem, params: { memory_mb: 1, duration_s: 2 } + + expect(response).to have_gitlab_http_status(200) + end + + it 'calls asynchronously' do + expect(Chaos::LeakMemWorker).to receive(:perform_async).with(100, 30.seconds) + + get :leakmem, params: { async: 1 } + + expect(response).to have_gitlab_http_status(200) + end + end + + describe '#cpu_spin' do + it 'calls synchronously' do + expect(Gitlab::Chaos).to receive(:cpu_spin).with(30.seconds) + + get :cpu_spin + + expect(response).to have_gitlab_http_status(200) + end + + it 'calls synchronously with params' do + expect(Gitlab::Chaos).to receive(:cpu_spin).with(3.seconds) + + get :cpu_spin, params: { duration_s: 3 } + + expect(response).to have_gitlab_http_status(200) + end + + it 'calls asynchronously' do + expect(Chaos::CpuSpinWorker).to receive(:perform_async).with(30.seconds) + + get :cpu_spin, params: { async: 1 } + + expect(response).to have_gitlab_http_status(200) + end + end + + describe '#db_spin' do + it 'calls synchronously' do + expect(Gitlab::Chaos).to receive(:db_spin).with(30.seconds, 1.second) + + get :db_spin + + expect(response).to have_gitlab_http_status(200) + end + + it 'calls synchronously with params' do + expect(Gitlab::Chaos).to receive(:db_spin).with(4.seconds, 5.seconds) + + get :db_spin, params: { duration_s: 4, interval_s: 5 } + + expect(response).to have_gitlab_http_status(200) + end + + it 'calls asynchronously' do + expect(Chaos::DbSpinWorker).to receive(:perform_async).with(30.seconds, 1.second) + + get :db_spin, params: { async: 1 } + + expect(response).to have_gitlab_http_status(200) + end + end + + describe '#sleep' do + it 'calls synchronously' do + expect(Gitlab::Chaos).to receive(:sleep).with(30.seconds) + + get :sleep + + expect(response).to have_gitlab_http_status(200) + end + + it 'calls synchronously with params' do + expect(Gitlab::Chaos).to receive(:sleep).with(5.seconds) + + get :sleep, params: { duration_s: 5 } + + expect(response).to have_gitlab_http_status(200) + end + + it 'calls asynchronously' do + expect(Chaos::SleepWorker).to receive(:perform_async).with(30.seconds) + + get :sleep, params: { async: 1 } + + expect(response).to have_gitlab_http_status(200) + end + end + + describe '#kill' do + it 'calls synchronously' do + expect(Gitlab::Chaos).to receive(:kill).with(no_args) + + get :kill + + expect(response).to have_gitlab_http_status(200) + end + + it 'calls asynchronously' do + expect(Chaos::KillWorker).to receive(:perform_async).with(no_args) + + get :kill, params: { async: 1 } + + expect(response).to have_gitlab_http_status(200) + end + end +end diff --git a/spec/controllers/dashboard/milestones_controller_spec.rb b/spec/controllers/dashboard/milestones_controller_spec.rb index 4de537ae6f8..67939aa4e6a 100644 --- a/spec/controllers/dashboard/milestones_controller_spec.rb +++ b/spec/controllers/dashboard/milestones_controller_spec.rb @@ -47,6 +47,8 @@ describe Dashboard::MilestonesController do describe "#index" do let(:public_group) { create(:group, :public) } let!(:public_milestone) { create(:milestone, group: public_group) } + let!(:closed_group_milestone) { create(:milestone, group: group, state: 'closed') } + let!(:closed_project_milestone) { create(:milestone, project: project, state: 'closed') } render_views @@ -59,6 +61,15 @@ describe Dashboard::MilestonesController do expect(json_response.map { |i| i["group_name"] }.compact).to match_array(group.name) end + it 'returns closed group and project milestones to which the user belongs' do + get :index, params: { state: 'closed' }, format: :json + + expect(response).to have_gitlab_http_status(200) + expect(json_response.size).to eq(2) + expect(json_response.map { |i| i["name"] }).to match_array([closed_group_milestone.name, closed_project_milestone.name]) + expect(json_response.map { |i| i["group_name"] }.compact).to match_array(group.name) + end + it 'searches legacy project milestones by title when search_title is given' do project_milestone = create(:milestone, title: 'Project milestone title', project: project) @@ -77,11 +88,11 @@ describe Dashboard::MilestonesController do expect(response.body).not_to include(project_milestone.title) end - it 'shows counts of group and project milestones to which the user belongs to' do + it 'shows counts of open and closed group and project milestones to which the user belongs to' do get :index expect(response.body).to include("Open\n<span class=\"badge badge-pill\">2</span>") - expect(response.body).to include("Closed\n<span class=\"badge badge-pill\">0</span>") + expect(response.body).to include("Closed\n<span class=\"badge badge-pill\">2</span>") end context 'external authorization' do diff --git a/spec/controllers/ide_controller_spec.rb b/spec/controllers/ide_controller_spec.rb new file mode 100644 index 00000000000..0462f9520d5 --- /dev/null +++ b/spec/controllers/ide_controller_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe IdeController do + let(:user) { create(:user) } + + before do + sign_in(user) + end + + it 'increases the views counter' do + expect(Gitlab::UsageDataCounters::WebIdeCounter).to receive(:increment_views_count) + + get :index + end +end diff --git a/spec/controllers/projects/merge_requests/creations_controller_spec.rb b/spec/controllers/projects/merge_requests/creations_controller_spec.rb index 5fefad86ef3..3816e1c7a31 100644 --- a/spec/controllers/projects/merge_requests/creations_controller_spec.rb +++ b/spec/controllers/projects/merge_requests/creations_controller_spec.rb @@ -212,4 +212,46 @@ describe Projects::MergeRequests::CreationsController do expect(response).to have_gitlab_http_status(200) end end + + describe 'POST create' do + let(:params) do + { + namespace_id: fork_project.namespace.to_param, + project_id: fork_project, + merge_request: { + title: 'Test merge request', + source_branch: 'remove-submodule', + target_branch: 'master' + } + } + end + + it 'creates merge request' do + expect do + post_request(params) + end.to change { MergeRequest.count }.by(1) + end + + context 'when the merge request is not created from the web ide' do + it 'counter is not increased' do + expect(Gitlab::UsageDataCounters::WebIdeCounter).not_to receive(:increment_merge_requests_count) + + post_request(params) + end + end + + context 'when the merge request is created from the web ide' do + let(:nav_source) { { nav_source: 'webide' } } + + it 'counter is increased' do + expect(Gitlab::UsageDataCounters::WebIdeCounter).to receive(:increment_merge_requests_count) + + post_request(params.merge(nav_source)) + end + end + + def post_request(merge_request_params) + post :create, params: merge_request_params + end + end end diff --git a/spec/features/admin/admin_requests_profiles_spec.rb b/spec/features/admin/admin_requests_profiles_spec.rb index 2503fc9067d..a962af4952b 100644 --- a/spec/features/admin/admin_requests_profiles_spec.rb +++ b/spec/features/admin/admin_requests_profiles_spec.rb @@ -19,42 +19,97 @@ describe 'Admin::RequestsProfilesController' do expect(page).to have_content("X-Profile-Token: #{Gitlab::RequestProfiler.profile_token}") end - it 'lists all available profiles' do - time1 = 1.hour.ago - time2 = 2.hours.ago - time3 = 3.hours.ago - profile1 = "|gitlab-org|gitlab-ce_#{time1.to_i}.html" - profile2 = "|gitlab-org|gitlab-ce_#{time2.to_i}.html" - profile3 = "|gitlab-com|infrastructure_#{time3.to_i}.html" - - FileUtils.touch("#{Gitlab::RequestProfiler::PROFILES_DIR}/#{profile1}") - FileUtils.touch("#{Gitlab::RequestProfiler::PROFILES_DIR}/#{profile2}") - FileUtils.touch("#{Gitlab::RequestProfiler::PROFILES_DIR}/#{profile3}") - - visit admin_requests_profiles_path + context 'when having multiple profiles' do + let(:time1) { 1.hour.ago } + let(:time2) { 2.hours.ago } + + let(:profiles) do + [ + { + request_path: '/gitlab-org/gitlab-ce', + name: "|gitlab-org|gitlab-ce_#{time1.to_i}_execution.html", + created: time1, + profile_mode: 'Execution' + }, + { + request_path: '/gitlab-org/gitlab-ce', + name: "|gitlab-org|gitlab-ce_#{time2.to_i}_execution.html", + created: time2, + profile_mode: 'Execution' + }, + { + request_path: '/gitlab-org/gitlab-ce', + name: "|gitlab-org|gitlab-ce_#{time1.to_i}_memory.html", + created: time1, + profile_mode: 'Memory' + }, + { + request_path: '/gitlab-org/gitlab-ce', + name: "|gitlab-org|gitlab-ce_#{time2.to_i}_memory.html", + created: time2, + profile_mode: 'Memory' + }, + { + request_path: '/gitlab-org/infrastructure', + name: "|gitlab-org|infrastructure_#{time1.to_i}_execution.html", + created: time1, + profile_mode: 'Execution' + }, + { + request_path: '/gitlab-org/infrastructure', + name: "|gitlab-org|infrastructure_#{time2.to_i}_memory.html", + created: time2, + profile_mode: 'Memory' + } + ] + end - within('.card', text: '/gitlab-org/gitlab-ce') do - expect(page).to have_selector("a[href='#{admin_requests_profile_path(profile1)}']", text: time1.to_s(:long)) - expect(page).to have_selector("a[href='#{admin_requests_profile_path(profile2)}']", text: time2.to_s(:long)) + before do + profiles.each do |profile| + FileUtils.touch(File.join(Gitlab::RequestProfiler::PROFILES_DIR, profile[:name])) + end end - within('.card', text: '/gitlab-com/infrastructure') do - expect(page).to have_selector("a[href='#{admin_requests_profile_path(profile3)}']", text: time3.to_s(:long)) + it 'lists all available profiles' do + visit admin_requests_profiles_path + + profiles.each do |profile| + within('.card', text: profile[:request_path]) do + expect(page).to have_selector( + "a[href='#{admin_requests_profile_path(profile[:name])}']", + text: "#{profile[:created].to_s(:long)} #{profile[:profile_mode]}") + end + end end end end describe 'GET /admin/requests_profiles/:profile' do context 'when a profile exists' do - it 'displays the content of the profile' do - content = 'This is a request profile' - profile = "|gitlab-org|gitlab-ce_#{Time.now.to_i}.html" - + before do File.write("#{Gitlab::RequestProfiler::PROFILES_DIR}/#{profile}", content) + end + + context 'when is valid call stack profile' do + let(:content) { 'This is a call stack request profile' } + let(:profile) { "|gitlab-org|gitlab-ce_#{Time.now.to_i}_execution.html" } + + it 'displays the content' do + visit admin_requests_profile_path(profile) + + expect(page).to have_content(content) + end + end + + context 'when is valid memory profile' do + let(:content) { 'This is a memory request profile' } + let(:profile) { "|gitlab-org|gitlab-ce_#{Time.now.to_i}_memory.txt" } - visit admin_requests_profile_path(profile) + it 'displays the content' do + visit admin_requests_profile_path(profile) - expect(page).to have_content(content) + expect(page).to have_content(content) + end end end diff --git a/spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb b/spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb index 10fe60cb075..3e72b2ce71f 100644 --- a/spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb +++ b/spec/features/merge_request/user_resolves_diff_notes_and_discussions_resolve_spec.rb @@ -362,14 +362,14 @@ describe 'Merge request > User resolves diff notes and threads', :js do end end - it 'shows jump to next thread button except on last thread' do + it 'shows jump to next discussion button on all discussions' do wait_for_requests all_discussion_replies = page.all('.discussion-reply-holder') expect(all_discussion_replies.count).to eq(2) - expect(all_discussion_replies.first.all('.discussion-next-btn').count).to eq(2) - expect(all_discussion_replies.last.all('.discussion-next-btn').count).to eq(2) + expect(all_discussion_replies.first.all('.discussion-next-btn').count).to eq(1) + expect(all_discussion_replies.last.all('.discussion-next-btn').count).to eq(1) end it 'displays next thread even if hidden' do diff --git a/spec/fixtures/security-reports/deprecated/gl-sast-report.json b/spec/fixtures/security-reports/deprecated/gl-sast-report.json index a85b9be8b5f..2f7e47281e2 100644 --- a/spec/fixtures/security-reports/deprecated/gl-sast-report.json +++ b/spec/fixtures/security-reports/deprecated/gl-sast-report.json @@ -838,6 +838,11 @@ }, "identifiers": [ { + "type": "flawfinder_func_name", + "name": "Flawfinder - char", + "value": "char" + }, + { "type": "cwe", "name": "CWE-119", "value": "119", @@ -870,6 +875,11 @@ }, "identifiers": [ { + "type": "flawfinder_func_name", + "name": "Flawfinder - fopen", + "value": "fopen" + }, + { "type": "cwe", "name": "CWE-362", "value": "362", @@ -897,6 +907,11 @@ }, "identifiers": [ { + "type": "flawfinder_func_name", + "name": "Flawfinder - char", + "value": "char" + }, + { "type": "cwe", "name": "CWE-119", "value": "119", @@ -930,6 +945,11 @@ }, "identifiers": [ { + "type": "flawfinder_func_name", + "name": "Flawfinder - strcpy", + "value": "strcpy" + }, + { "type": "cwe", "name": "CWE-120", "value": "120", diff --git a/spec/fixtures/security-reports/master/gl-sast-report.json b/spec/fixtures/security-reports/master/gl-sast-report.json index 4bef3d22f70..345e1e9f83a 100644 --- a/spec/fixtures/security-reports/master/gl-sast-report.json +++ b/spec/fixtures/security-reports/master/gl-sast-report.json @@ -840,6 +840,11 @@ }, "identifiers": [ { + "type": "flawfinder_func_name", + "name": "Flawfinder - char", + "value": "char" + }, + { "type": "cwe", "name": "CWE-119", "value": "119", @@ -872,6 +877,11 @@ }, "identifiers": [ { + "type": "flawfinder_func_name", + "name": "Flawfinder - fopen", + "value": "fopen" + }, + { "type": "cwe", "name": "CWE-362", "value": "362", @@ -899,6 +909,11 @@ }, "identifiers": [ { + "type": "flawfinder_func_name", + "name": "Flawfinder - char", + "value": "char" + }, + { "type": "cwe", "name": "CWE-119", "value": "119", @@ -932,6 +947,11 @@ }, "identifiers": [ { + "type": "flawfinder_func_name", + "name": "Flawfinder - strcpy", + "value": "strcpy" + }, + { "type": "cwe", "name": "CWE-120", "value": "120", diff --git a/spec/frontend/environment.js b/spec/frontend/environment.js index a8c8688441d..290c0e797cb 100644 --- a/spec/frontend/environment.js +++ b/spec/frontend/environment.js @@ -1,8 +1,11 @@ /* eslint-disable import/no-commonjs */ +const path = require('path'); const { ErrorWithStack } = require('jest-util'); const JSDOMEnvironment = require('jest-environment-jsdom'); +const ROOT_PATH = path.resolve(__dirname, '../..'); + class CustomEnvironment extends JSDOMEnvironment { constructor(config, context) { super(config, context); @@ -35,9 +38,8 @@ class CustomEnvironment extends JSDOMEnvironment { this.rejectedPromises.push(error); }; - this.global.fixturesBasePath = `${process.cwd()}/${ - IS_EE ? 'ee/' : '' - }spec/javascripts/fixtures`; + this.global.fixturesBasePath = `${ROOT_PATH}/tmp/tests/frontend/fixtures${IS_EE ? '-ee' : ''}`; + this.global.staticFixturesBasePath = `${ROOT_PATH}/spec/frontend/fixtures`; // Not yet supported by JSDOM: https://github.com/jsdom/jsdom/issues/317 this.global.document.createRange = () => ({ diff --git a/spec/javascripts/fixtures/abuse_reports.rb b/spec/frontend/fixtures/abuse_reports.rb index e0aaecf626a..21356390cae 100644 --- a/spec/javascripts/fixtures/abuse_reports.rb +++ b/spec/frontend/fixtures/abuse_reports.rb @@ -21,6 +21,6 @@ describe Admin::AbuseReportsController, '(JavaScript fixtures)', type: :controll it 'abuse_reports/abuse_reports_list.html' do get :index - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/admin_users.rb b/spec/frontend/fixtures/admin_users.rb index 22a5de66577..0209594dadc 100644 --- a/spec/javascripts/fixtures/admin_users.rb +++ b/spec/frontend/fixtures/admin_users.rb @@ -23,6 +23,6 @@ describe Admin::UsersController, '(JavaScript fixtures)', type: :controller do get :new - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/application_settings.rb b/spec/frontend/fixtures/application_settings.rb index d4651fa6ece..38a060580c1 100644 --- a/spec/javascripts/fixtures/application_settings.rb +++ b/spec/frontend/fixtures/application_settings.rb @@ -28,6 +28,6 @@ describe Admin::ApplicationSettingsController, '(JavaScript fixtures)', type: :c get :show - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/autocomplete_sources.rb b/spec/frontend/fixtures/autocomplete_sources.rb index b20a0159d7d..9e04328e2b9 100644 --- a/spec/javascripts/fixtures/autocomplete_sources.rb +++ b/spec/frontend/fixtures/autocomplete_sources.rb @@ -34,6 +34,6 @@ describe Projects::AutocompleteSourcesController, '(JavaScript fixtures)', type: type_id: issue.id } - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/blob.rb b/spec/frontend/fixtures/blob.rb index 07670552cd5..ce5030efbf8 100644 --- a/spec/javascripts/fixtures/blob.rb +++ b/spec/frontend/fixtures/blob.rb @@ -29,6 +29,6 @@ describe Projects::BlobController, '(JavaScript fixtures)', type: :controller do id: 'add-ipython-files/files/ipython/basic.ipynb' }) - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/boards.rb b/spec/frontend/fixtures/boards.rb index 5835721d3d5..f257d80390f 100644 --- a/spec/javascripts/fixtures/boards.rb +++ b/spec/frontend/fixtures/boards.rb @@ -23,6 +23,6 @@ describe Projects::BoardsController, '(JavaScript fixtures)', type: :controller project_id: project }) - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/branches.rb b/spec/frontend/fixtures/branches.rb index 204aa9b7c7a..197fe42c52a 100644 --- a/spec/javascripts/fixtures/branches.rb +++ b/spec/frontend/fixtures/branches.rb @@ -27,6 +27,6 @@ describe Projects::BranchesController, '(JavaScript fixtures)', type: :controlle project_id: project } - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/clusters.rb b/spec/frontend/fixtures/clusters.rb index 1076404e0e3..f15ef010807 100644 --- a/spec/javascripts/fixtures/clusters.rb +++ b/spec/frontend/fixtures/clusters.rb @@ -29,6 +29,6 @@ describe Projects::ClustersController, '(JavaScript fixtures)', type: :controlle id: cluster } - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/commit.rb b/spec/frontend/fixtures/commit.rb index ff9a4bc1adc..a328c455356 100644 --- a/spec/javascripts/fixtures/commit.rb +++ b/spec/frontend/fixtures/commit.rb @@ -28,6 +28,6 @@ describe Projects::CommitController, '(JavaScript fixtures)', type: :controller get :show, params: params - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/deploy_keys.rb b/spec/frontend/fixtures/deploy_keys.rb index 38eab853da2..fca233c6f59 100644 --- a/spec/javascripts/fixtures/deploy_keys.rb +++ b/spec/frontend/fixtures/deploy_keys.rb @@ -38,6 +38,6 @@ describe Projects::DeployKeysController, '(JavaScript fixtures)', type: :control project_id: project }, format: :json - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/groups.rb b/spec/frontend/fixtures/groups.rb index 4d0afc3ce1a..c1bb2d43332 100644 --- a/spec/javascripts/fixtures/groups.rb +++ b/spec/frontend/fixtures/groups.rb @@ -21,7 +21,7 @@ describe 'Groups (JavaScript fixtures)', type: :controller do it 'groups/edit.html' do get :edit, params: { id: group } - expect(response).to be_success + expect(response).to be_successful end end @@ -29,7 +29,7 @@ describe 'Groups (JavaScript fixtures)', type: :controller do it 'groups/ci_cd_settings.html' do get :show, params: { group_id: group } - expect(response).to be_success + expect(response).to be_successful end end end diff --git a/spec/javascripts/fixtures/issues.rb b/spec/frontend/fixtures/issues.rb index d8d77f767de..b5eb38e0023 100644 --- a/spec/javascripts/fixtures/issues.rb +++ b/spec/frontend/fixtures/issues.rb @@ -48,7 +48,7 @@ describe Projects::IssuesController, '(JavaScript fixtures)', type: :controller project_id: project } - expect(response).to be_success + expect(response).to be_successful end private @@ -60,7 +60,7 @@ describe Projects::IssuesController, '(JavaScript fixtures)', type: :controller id: issue.to_param } - expect(response).to be_success + expect(response).to be_successful end end @@ -117,6 +117,6 @@ describe API::Issues, '(JavaScript fixtures)', type: :request do get_related_merge_requests(project.id, issue.iid, user) - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/jobs.rb b/spec/frontend/fixtures/jobs.rb index 46ccd6f8c8a..a3a7759c85b 100644 --- a/spec/javascripts/fixtures/jobs.rb +++ b/spec/frontend/fixtures/jobs.rb @@ -39,7 +39,7 @@ describe Projects::JobsController, '(JavaScript fixtures)', type: :controller do id: build_with_artifacts.to_param } - expect(response).to be_success + expect(response).to be_successful end it 'jobs/delayed.json' do @@ -49,6 +49,6 @@ describe Projects::JobsController, '(JavaScript fixtures)', type: :controller do id: delayed_job.to_param }, format: :json - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/labels.rb b/spec/frontend/fixtures/labels.rb index 4d1b7317274..a312287970f 100644 --- a/spec/javascripts/fixtures/labels.rb +++ b/spec/frontend/fixtures/labels.rb @@ -35,7 +35,7 @@ describe 'Labels (JavaScript fixtures)' do group_id: group }, format: 'json' - expect(response).to be_success + expect(response).to be_successful end end @@ -52,7 +52,7 @@ describe 'Labels (JavaScript fixtures)' do project_id: project }, format: 'json' - expect(response).to be_success + expect(response).to be_successful end end end diff --git a/spec/javascripts/fixtures/merge_requests.rb b/spec/frontend/fixtures/merge_requests.rb index 05860be2291..88706e96676 100644 --- a/spec/javascripts/fixtures/merge_requests.rb +++ b/spec/frontend/fixtures/merge_requests.rb @@ -129,6 +129,6 @@ describe Projects::MergeRequestsController, '(JavaScript fixtures)', type: :cont id: merge_request.to_param }, format: :html - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/merge_requests_diffs.rb b/spec/frontend/fixtures/merge_requests_diffs.rb index 03b9b713fd8..b633a0495a6 100644 --- a/spec/javascripts/fixtures/merge_requests_diffs.rb +++ b/spec/frontend/fixtures/merge_requests_diffs.rb @@ -1,4 +1,3 @@ - require 'spec_helper' describe Projects::MergeRequests::DiffsController, '(JavaScript fixtures)', type: :controller do @@ -65,6 +64,6 @@ describe Projects::MergeRequests::DiffsController, '(JavaScript fixtures)', type **extra_params }, format: :json - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/pipeline_schedules.rb b/spec/frontend/fixtures/pipeline_schedules.rb index aecd56e6198..a70091a3919 100644 --- a/spec/javascripts/fixtures/pipeline_schedules.rb +++ b/spec/frontend/fixtures/pipeline_schedules.rb @@ -28,7 +28,7 @@ describe Projects::PipelineSchedulesController, '(JavaScript fixtures)', type: : id: pipeline_schedule.id } - expect(response).to be_success + expect(response).to be_successful end it 'pipeline_schedules/edit_with_variables.html' do @@ -38,6 +38,6 @@ describe Projects::PipelineSchedulesController, '(JavaScript fixtures)', type: : id: pipeline_schedule_populated.id } - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/pipelines.rb b/spec/frontend/fixtures/pipelines.rb index 6b6b0eefab9..ed57eb0aa80 100644 --- a/spec/javascripts/fixtures/pipelines.rb +++ b/spec/frontend/fixtures/pipelines.rb @@ -29,6 +29,6 @@ describe Projects::PipelinesController, '(JavaScript fixtures)', type: :controll project_id: project }, format: :json - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/projects.rb b/spec/frontend/fixtures/projects.rb index 94c59207898..b6c29003e57 100644 --- a/spec/javascripts/fixtures/projects.rb +++ b/spec/frontend/fixtures/projects.rb @@ -18,6 +18,8 @@ describe 'Projects (JavaScript fixtures)', type: :controller do end before do + stub_licensed_features(variable_environment_scope: true) + project.add_maintainer(admin) sign_in(admin) allow(SecureRandom).to receive(:hex).and_return('securerandomhex:thereisnospoon') @@ -34,7 +36,7 @@ describe 'Projects (JavaScript fixtures)', type: :controller do id: project } - expect(response).to be_success + expect(response).to be_successful end it 'projects/overview.html' do @@ -43,7 +45,7 @@ describe 'Projects (JavaScript fixtures)', type: :controller do id: project_with_repo } - expect(response).to be_success + expect(response).to be_successful end it 'projects/edit.html' do @@ -52,7 +54,7 @@ describe 'Projects (JavaScript fixtures)', type: :controller do id: project } - expect(response).to be_success + expect(response).to be_successful end end @@ -63,7 +65,7 @@ describe 'Projects (JavaScript fixtures)', type: :controller do project_id: project } - expect(response).to be_success + expect(response).to be_successful end it 'projects/ci_cd_settings_with_variables.html' do @@ -75,7 +77,7 @@ describe 'Projects (JavaScript fixtures)', type: :controller do project_id: project_variable_populated } - expect(response).to be_success + expect(response).to be_successful end end end diff --git a/spec/javascripts/fixtures/prometheus_service.rb b/spec/frontend/fixtures/prometheus_service.rb index f3171fdd97b..93ee81120d7 100644 --- a/spec/javascripts/fixtures/prometheus_service.rb +++ b/spec/frontend/fixtures/prometheus_service.rb @@ -29,6 +29,6 @@ describe Projects::ServicesController, '(JavaScript fixtures)', type: :controlle id: service.to_param } - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/raw.rb b/spec/frontend/fixtures/raw.rb index 801c80a0112..801c80a0112 100644 --- a/spec/javascripts/fixtures/raw.rb +++ b/spec/frontend/fixtures/raw.rb diff --git a/spec/javascripts/fixtures/search.rb b/spec/frontend/fixtures/search.rb index 22fc546d761..c26c6998ae9 100644 --- a/spec/javascripts/fixtures/search.rb +++ b/spec/frontend/fixtures/search.rb @@ -12,6 +12,6 @@ describe SearchController, '(JavaScript fixtures)', type: :controller do it 'search/show.html' do get :show - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/services.rb b/spec/frontend/fixtures/services.rb index 2237702ccca..ee1e088f158 100644 --- a/spec/javascripts/fixtures/services.rb +++ b/spec/frontend/fixtures/services.rb @@ -29,6 +29,6 @@ describe Projects::ServicesController, '(JavaScript fixtures)', type: :controlle id: service.to_param } - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/javascripts/fixtures/sessions.rb b/spec/frontend/fixtures/sessions.rb index 92b74c01c89..18574ea06b5 100644 --- a/spec/javascripts/fixtures/sessions.rb +++ b/spec/frontend/fixtures/sessions.rb @@ -19,7 +19,7 @@ describe 'Sessions (JavaScript fixtures)' do it 'sessions/new.html' do get :new - expect(response).to be_success + expect(response).to be_successful end end end diff --git a/spec/javascripts/fixtures/snippet.rb b/spec/frontend/fixtures/snippet.rb index ace84b14eb7..23bcdb47ac6 100644 --- a/spec/javascripts/fixtures/snippet.rb +++ b/spec/frontend/fixtures/snippet.rb @@ -28,6 +28,6 @@ describe SnippetsController, '(JavaScript fixtures)', type: :controller do get(:show, params: { id: snippet.to_param }) - expect(response).to be_success + expect(response).to be_successful end end diff --git a/spec/frontend/fixtures/static/README.md b/spec/frontend/fixtures/static/README.md new file mode 100644 index 00000000000..011601d0df8 --- /dev/null +++ b/spec/frontend/fixtures/static/README.md @@ -0,0 +1,3 @@ +# Please do not add new files here! + +Instead use a Ruby file in the fixtures root directory (`spec/frontend/fixtures/`). diff --git a/spec/javascripts/fixtures/static/ajax_loading_spinner.html b/spec/frontend/fixtures/static/ajax_loading_spinner.html index 0e1ebb32b1c..0e1ebb32b1c 100644 --- a/spec/javascripts/fixtures/static/ajax_loading_spinner.html +++ b/spec/frontend/fixtures/static/ajax_loading_spinner.html diff --git a/spec/javascripts/fixtures/static/balsamiq_viewer.html b/spec/frontend/fixtures/static/balsamiq_viewer.html index cdd723d1a84..cdd723d1a84 100644 --- a/spec/javascripts/fixtures/static/balsamiq_viewer.html +++ b/spec/frontend/fixtures/static/balsamiq_viewer.html diff --git a/spec/javascripts/fixtures/static/create_item_dropdown.html b/spec/frontend/fixtures/static/create_item_dropdown.html index d2d38370092..d2d38370092 100644 --- a/spec/javascripts/fixtures/static/create_item_dropdown.html +++ b/spec/frontend/fixtures/static/create_item_dropdown.html diff --git a/spec/javascripts/fixtures/static/environments/table.html b/spec/frontend/fixtures/static/environments/table.html index 417af564ff1..417af564ff1 100644 --- a/spec/javascripts/fixtures/static/environments/table.html +++ b/spec/frontend/fixtures/static/environments/table.html diff --git a/spec/frontend/fixtures/static/environments_logs.html b/spec/frontend/fixtures/static/environments_logs.html new file mode 100644 index 00000000000..6179d3dbc23 --- /dev/null +++ b/spec/frontend/fixtures/static/environments_logs.html @@ -0,0 +1,29 @@ +<div class="js-kubernetes-logs" data-logs-path="/root/kubernetes-app/environments/1/logs"> + <div class="build-page"> + <div class="build-trace-container prepend-top-default"> + <div class="top-bar js-top-bar"> + <div class="truncated-info hidden-xs pull-left"></div> + <div class="dropdown prepend-left-10 js-pod-dropdown"> + <button aria-expanded="false" class="dropdown-menu-toggle" data-toggle="dropdown" type="button"> + <i class="fa fa-chevron-down"></i> + </button> + <div class="dropdown-menu dropdown-menu-selectable dropdown-menu-drop-up"></div> + </div> + <div class="controllers pull-right"> + <div class="has-tooltip controllers-buttons" data-container="body" data-placement="top" title="Scroll to top"> + <button class="js-scroll-up btn-scroll btn-transparent btn-blank" disabled type="button"></button> + </div> + <div class="has-tooltip controllers-buttons" data-container="body" data-placement="top" title="Scroll to bottom"> + <button class="js-scroll-down btn-scroll btn-transparent btn-blank" disabled type="button"></button> + </div> + <div class="refresh-control pull-right"> + <div class="has-tooltip controllers-buttons" data-container="body" data-placement="top" title="Refresh"> + <button class="js-refresh-log btn-default btn-refresh" disabled type="button"></button> + </div> + </div> + </div> + </div> + <pre class="build-trace" id="build-trace"><code class="bash js-build-output"><div class="build-loader-animation js-build-refresh"></div></code></pre> + </div> + </div> +</div> diff --git a/spec/javascripts/fixtures/static/event_filter.html b/spec/frontend/fixtures/static/event_filter.html index 8e9b6fb1b5c..8e9b6fb1b5c 100644 --- a/spec/javascripts/fixtures/static/event_filter.html +++ b/spec/frontend/fixtures/static/event_filter.html diff --git a/spec/javascripts/fixtures/static/gl_dropdown.html b/spec/frontend/fixtures/static/gl_dropdown.html index 08f6738414e..08f6738414e 100644 --- a/spec/javascripts/fixtures/static/gl_dropdown.html +++ b/spec/frontend/fixtures/static/gl_dropdown.html diff --git a/spec/javascripts/fixtures/static/gl_field_errors.html b/spec/frontend/fixtures/static/gl_field_errors.html index f8470e02b7c..f8470e02b7c 100644 --- a/spec/javascripts/fixtures/static/gl_field_errors.html +++ b/spec/frontend/fixtures/static/gl_field_errors.html diff --git a/spec/javascripts/fixtures/static/images/green_box.png b/spec/frontend/fixtures/static/images/green_box.png Binary files differindex cd1ff9f9ade..cd1ff9f9ade 100644 --- a/spec/javascripts/fixtures/static/images/green_box.png +++ b/spec/frontend/fixtures/static/images/green_box.png diff --git a/spec/javascripts/fixtures/static/images/one_white_pixel.png b/spec/frontend/fixtures/static/images/one_white_pixel.png Binary files differindex 073fcf40a18..073fcf40a18 100644 --- a/spec/javascripts/fixtures/static/images/one_white_pixel.png +++ b/spec/frontend/fixtures/static/images/one_white_pixel.png diff --git a/spec/javascripts/fixtures/static/images/red_box.png b/spec/frontend/fixtures/static/images/red_box.png Binary files differindex 73b2927da0f..73b2927da0f 100644 --- a/spec/javascripts/fixtures/static/images/red_box.png +++ b/spec/frontend/fixtures/static/images/red_box.png diff --git a/spec/javascripts/fixtures/static/issuable_filter.html b/spec/frontend/fixtures/static/issuable_filter.html index 06b70fb43f1..06b70fb43f1 100644 --- a/spec/javascripts/fixtures/static/issuable_filter.html +++ b/spec/frontend/fixtures/static/issuable_filter.html diff --git a/spec/javascripts/fixtures/static/issue_sidebar_label.html b/spec/frontend/fixtures/static/issue_sidebar_label.html index ec8fb30f219..ec8fb30f219 100644 --- a/spec/javascripts/fixtures/static/issue_sidebar_label.html +++ b/spec/frontend/fixtures/static/issue_sidebar_label.html diff --git a/spec/javascripts/fixtures/static/line_highlighter.html b/spec/frontend/fixtures/static/line_highlighter.html index 897a25d6760..897a25d6760 100644 --- a/spec/javascripts/fixtures/static/line_highlighter.html +++ b/spec/frontend/fixtures/static/line_highlighter.html diff --git a/spec/javascripts/fixtures/static/linked_tabs.html b/spec/frontend/fixtures/static/linked_tabs.html index c25463bf1db..c25463bf1db 100644 --- a/spec/javascripts/fixtures/static/linked_tabs.html +++ b/spec/frontend/fixtures/static/linked_tabs.html diff --git a/spec/javascripts/fixtures/static/merge_requests_show.html b/spec/frontend/fixtures/static/merge_requests_show.html index 87e36c9f315..87e36c9f315 100644 --- a/spec/javascripts/fixtures/static/merge_requests_show.html +++ b/spec/frontend/fixtures/static/merge_requests_show.html diff --git a/spec/javascripts/fixtures/static/mini_dropdown_graph.html b/spec/frontend/fixtures/static/mini_dropdown_graph.html index cd0b8dec3fc..cd0b8dec3fc 100644 --- a/spec/javascripts/fixtures/static/mini_dropdown_graph.html +++ b/spec/frontend/fixtures/static/mini_dropdown_graph.html diff --git a/spec/javascripts/fixtures/static/notebook_viewer.html b/spec/frontend/fixtures/static/notebook_viewer.html index 4bbb7bf1094..4bbb7bf1094 100644 --- a/spec/javascripts/fixtures/static/notebook_viewer.html +++ b/spec/frontend/fixtures/static/notebook_viewer.html diff --git a/spec/javascripts/fixtures/static/oauth_remember_me.html b/spec/frontend/fixtures/static/oauth_remember_me.html index 9ba1ffc72fe..9ba1ffc72fe 100644 --- a/spec/javascripts/fixtures/static/oauth_remember_me.html +++ b/spec/frontend/fixtures/static/oauth_remember_me.html diff --git a/spec/javascripts/fixtures/static/pdf_viewer.html b/spec/frontend/fixtures/static/pdf_viewer.html index 350d35a262f..350d35a262f 100644 --- a/spec/javascripts/fixtures/static/pdf_viewer.html +++ b/spec/frontend/fixtures/static/pdf_viewer.html diff --git a/spec/javascripts/fixtures/static/pipeline_graph.html b/spec/frontend/fixtures/static/pipeline_graph.html index 422372bb7d5..422372bb7d5 100644 --- a/spec/javascripts/fixtures/static/pipeline_graph.html +++ b/spec/frontend/fixtures/static/pipeline_graph.html diff --git a/spec/javascripts/fixtures/static/pipelines.html b/spec/frontend/fixtures/static/pipelines.html index 42333f94f2f..42333f94f2f 100644 --- a/spec/javascripts/fixtures/static/pipelines.html +++ b/spec/frontend/fixtures/static/pipelines.html diff --git a/spec/javascripts/fixtures/static/project_select_combo_button.html b/spec/frontend/fixtures/static/project_select_combo_button.html index 50c826051c0..50c826051c0 100644 --- a/spec/javascripts/fixtures/static/project_select_combo_button.html +++ b/spec/frontend/fixtures/static/project_select_combo_button.html diff --git a/spec/javascripts/fixtures/static/projects.json b/spec/frontend/fixtures/static/projects.json index 68a150f602a..68a150f602a 100644 --- a/spec/javascripts/fixtures/static/projects.json +++ b/spec/frontend/fixtures/static/projects.json diff --git a/spec/javascripts/fixtures/static/search_autocomplete.html b/spec/frontend/fixtures/static/search_autocomplete.html index 29db9020424..29db9020424 100644 --- a/spec/javascripts/fixtures/static/search_autocomplete.html +++ b/spec/frontend/fixtures/static/search_autocomplete.html diff --git a/spec/javascripts/fixtures/static/signin_tabs.html b/spec/frontend/fixtures/static/signin_tabs.html index 7e66ab9394b..7e66ab9394b 100644 --- a/spec/javascripts/fixtures/static/signin_tabs.html +++ b/spec/frontend/fixtures/static/signin_tabs.html diff --git a/spec/javascripts/fixtures/static/sketch_viewer.html b/spec/frontend/fixtures/static/sketch_viewer.html index e25e554e568..e25e554e568 100644 --- a/spec/javascripts/fixtures/static/sketch_viewer.html +++ b/spec/frontend/fixtures/static/sketch_viewer.html diff --git a/spec/javascripts/fixtures/todos.rb b/spec/frontend/fixtures/todos.rb index d0c8a6eca01..a7c183d2414 100644 --- a/spec/javascripts/fixtures/todos.rb +++ b/spec/frontend/fixtures/todos.rb @@ -29,7 +29,7 @@ describe 'Todos (JavaScript fixtures)' do it 'todos/todos.html' do get :index - expect(response).to be_success + expect(response).to be_successful end end @@ -48,7 +48,7 @@ describe 'Todos (JavaScript fixtures)' do issuable_id: issue_2.id }, format: 'json' - expect(response).to be_success + expect(response).to be_successful end end end diff --git a/spec/javascripts/fixtures/u2f.rb b/spec/frontend/fixtures/u2f.rb index f52832b6efb..8ecbc0390cd 100644 --- a/spec/javascripts/fixtures/u2f.rb +++ b/spec/frontend/fixtures/u2f.rb @@ -23,7 +23,7 @@ context 'U2F' do post :create, params: { user: { login: user.username, password: user.password } } - expect(response).to be_success + expect(response).to be_successful end end @@ -38,7 +38,7 @@ context 'U2F' do it 'u2f/register.html' do get :show - expect(response).to be_success + expect(response).to be_successful end end end diff --git a/spec/frontend/helpers/fixtures.js b/spec/frontend/helpers/fixtures.js index b77bcd6266e..778196843db 100644 --- a/spec/frontend/helpers/fixtures.js +++ b/spec/frontend/helpers/fixtures.js @@ -4,12 +4,15 @@ import path from 'path'; import { ErrorWithStack } from 'jest-util'; export function getFixture(relativePath) { - const absolutePath = path.join(global.fixturesBasePath, relativePath); + const basePath = relativePath.startsWith('static/') + ? global.staticFixturesBasePath + : global.fixturesBasePath; + const absolutePath = path.join(basePath, relativePath); if (!fs.existsSync(absolutePath)) { throw new ErrorWithStack( `Fixture file ${relativePath} does not exist. -Did you run bin/rake karma:fixtures?`, +Did you run bin/rake frontend:fixtures?`, getFixture, ); } diff --git a/spec/frontend/ide/services/index_spec.js b/spec/frontend/ide/services/index_spec.js index 499fa8fc012..3d5ed4b5c0c 100644 --- a/spec/frontend/ide/services/index_spec.js +++ b/spec/frontend/ide/services/index_spec.js @@ -16,40 +16,16 @@ describe('IDE services', () => { branch: TEST_BRANCH, commit_message: 'Hello world', actions: [], - start_sha: undefined, + start_sha: TEST_COMMIT_SHA, }; - Api.createBranch.mockReturnValue(Promise.resolve()); Api.commitMultiple.mockReturnValue(Promise.resolve()); }); - describe.each` - startSha | shouldCreateBranch - ${undefined} | ${false} - ${TEST_COMMIT_SHA} | ${true} - `('when start_sha is $startSha', ({ startSha, shouldCreateBranch }) => { - beforeEach(() => { - payload.start_sha = startSha; + it('should commit', () => { + services.commit(TEST_PROJECT_ID, payload); - return services.commit(TEST_PROJECT_ID, payload); - }); - - if (shouldCreateBranch) { - it('should create branch', () => { - expect(Api.createBranch).toHaveBeenCalledWith(TEST_PROJECT_ID, { - ref: TEST_COMMIT_SHA, - branch: TEST_BRANCH, - }); - }); - } else { - it('should not create branch', () => { - expect(Api.createBranch).not.toHaveBeenCalled(); - }); - } - - it('should commit', () => { - expect(Api.commitMultiple).toHaveBeenCalledWith(TEST_PROJECT_ID, payload); - }); + expect(Api.commitMultiple).toHaveBeenCalledWith(TEST_PROJECT_ID, payload); }); }); }); diff --git a/spec/frontend/notes/components/discussion_jump_to_next_button_spec.js b/spec/frontend/notes/components/discussion_jump_to_next_button_spec.js index 989b0458481..fd439ba46bd 100644 --- a/spec/frontend/notes/components/discussion_jump_to_next_button_spec.js +++ b/spec/frontend/notes/components/discussion_jump_to_next_button_spec.js @@ -23,8 +23,7 @@ describe('JumpToNextDiscussionButton', () => { button.trigger('click'); - expect(wrapper.emitted()).toEqual({ - onClick: [[]], - }); + expect(wrapper.emitted().onClick).toBeTruthy(); + expect(wrapper.emitted().onClick.length).toBe(1); }); }); diff --git a/spec/frontend/vue_shared/components/markdown/suggestion_diff_header_spec.js b/spec/frontend/vue_shared/components/markdown/suggestion_diff_header_spec.js index 3b6f67457ad..d69b4c7c162 100644 --- a/spec/frontend/vue_shared/components/markdown/suggestion_diff_header_spec.js +++ b/spec/frontend/vue_shared/components/markdown/suggestion_diff_header_spec.js @@ -73,7 +73,10 @@ describe('Suggestion Diff component', () => { }); it('emits apply', () => { - expect(wrapper.emittedByOrder()).toEqual([{ name: 'apply', args: [expect.any(Function)] }]); + expect(wrapper.emittedByOrder()).toContainEqual({ + name: 'apply', + args: [expect.any(Function)], + }); }); it('hides apply button', () => { diff --git a/spec/javascripts/boards/board_blank_state_spec.js b/spec/javascripts/boards/board_blank_state_spec.js index 8e2a947b0dd..b64859ec32a 100644 --- a/spec/javascripts/boards/board_blank_state_spec.js +++ b/spec/javascripts/boards/board_blank_state_spec.js @@ -1,7 +1,6 @@ import Vue from 'vue'; import boardsStore from '~/boards/stores/boards_store'; import BoardBlankState from '~/boards/components/board_blank_state.vue'; -import { mockBoardService } from './mock_data'; describe('Boards blank state', () => { let vm; @@ -11,9 +10,10 @@ describe('Boards blank state', () => { const Comp = Vue.extend(BoardBlankState); boardsStore.create(); - gl.boardService = mockBoardService(); - spyOn(gl.boardService, 'generateDefaultLists').and.callFake( + spyOn(boardsStore, 'addList').and.stub(); + spyOn(boardsStore, 'removeList').and.stub(); + spyOn(boardsStore, 'generateDefaultLists').and.callFake( () => new Promise((resolve, reject) => { if (fail) { @@ -71,9 +71,14 @@ describe('Boards blank state', () => { vm.$el.querySelector('.btn-success').click(); setTimeout(() => { - expect(boardsStore.state.lists.length).toBe(2); - expect(boardsStore.state.lists[0].title).toEqual('To Do'); - expect(boardsStore.state.lists[1].title).toEqual('Doing'); + expect(boardsStore.addList).toHaveBeenCalledTimes(2); + expect(boardsStore.addList).toHaveBeenCalledWith( + jasmine.objectContaining({ title: 'To Do' }), + ); + + expect(boardsStore.addList).toHaveBeenCalledWith( + jasmine.objectContaining({ title: 'Doing' }), + ); done(); }); @@ -86,7 +91,7 @@ describe('Boards blank state', () => { setTimeout(() => { expect(boardsStore.welcomeIsHidden()).toBeFalsy(); - expect(boardsStore.state.lists.length).toBe(1); + expect(boardsStore.removeList).toHaveBeenCalledWith(undefined, 'label'); done(); }); diff --git a/spec/javascripts/fixtures/.gitignore b/spec/javascripts/fixtures/.gitignore index bed020f5b0a..d6b7ef32c84 100644 --- a/spec/javascripts/fixtures/.gitignore +++ b/spec/javascripts/fixtures/.gitignore @@ -1,5 +1,2 @@ -*.html.raw -*.html -*.json -*.pdf -*.bmpr +* +!.gitignore diff --git a/spec/javascripts/fixtures/static/README.md b/spec/javascripts/fixtures/static/README.md deleted file mode 100644 index b5c2f8233bf..00000000000 --- a/spec/javascripts/fixtures/static/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Please do not add new files here! - -Instead use a Ruby file in the fixtures root directory (`spec/javascripts/fixtures/`). diff --git a/spec/javascripts/ide/stores/modules/commit/actions_spec.js b/spec/javascripts/ide/stores/modules/commit/actions_spec.js index 8a3c132972e..14d861f21d2 100644 --- a/spec/javascripts/ide/stores/modules/commit/actions_spec.js +++ b/spec/javascripts/ide/stores/modules/commit/actions_spec.js @@ -245,7 +245,7 @@ describe('IDE commit module actions', () => { master: { workingReference: '1', commit: { - short_id: TEST_COMMIT_SHA, + id: TEST_COMMIT_SHA, }, }, }, @@ -411,7 +411,7 @@ describe('IDE commit module actions', () => { expect(visitUrl).toHaveBeenCalledWith( `webUrl/merge_requests/new?merge_request[source_branch]=${ store.getters['commit/placeholderBranchName'] - }&merge_request[target_branch]=master`, + }&merge_request[target_branch]=master&nav_source=webide`, ); done(); diff --git a/spec/javascripts/ide/stores/mutations/file_spec.js b/spec/javascripts/ide/stores/mutations/file_spec.js index 7714f66c9a4..1b41c3c2548 100644 --- a/spec/javascripts/ide/stores/mutations/file_spec.js +++ b/spec/javascripts/ide/stores/mutations/file_spec.js @@ -103,6 +103,43 @@ describe('IDE store file mutations', () => { expect(localState.openFiles[0].rawPath).toEqual(rawPath); expect(localFile.rawPath).toEqual(rawPath); }); + + it('does not mutate certain props on the file', () => { + const path = 'New Path'; + const name = 'New Name'; + localFile.path = path; + localFile.name = name; + + localState.stagedFiles = [localFile]; + localState.changedFiles = [localFile]; + localState.openFiles = [localFile]; + + mutations.SET_FILE_DATA(localState, { + data: { + path: 'Old Path', + name: 'Old Name', + raw: 'Old Raw', + base_raw: 'Old Base Raw', + }, + file: localFile, + }); + + [ + localState.stagedFiles[0], + localState.changedFiles[0], + localState.openFiles[0], + localFile, + ].forEach(f => { + expect(f).toEqual( + jasmine.objectContaining({ + path, + name, + raw: null, + baseRaw: null, + }), + ); + }); + }); }); describe('SET_FILE_RAW_DATA', () => { diff --git a/spec/javascripts/notes/components/note_actions/reply_button_spec.js b/spec/javascripts/notes/components/note_actions/reply_button_spec.js index 11fb89808d9..003773d07ea 100644 --- a/spec/javascripts/notes/components/note_actions/reply_button_spec.js +++ b/spec/javascripts/notes/components/note_actions/reply_button_spec.js @@ -25,8 +25,7 @@ describe('ReplyButton', () => { button.trigger('click'); - expect(wrapper.emitted()).toEqual({ - startReplying: [[]], - }); + expect(wrapper.emitted().startReplying).toBeTruthy(); + expect(wrapper.emitted().startReplying.length).toBe(1); }); }); diff --git a/spec/javascripts/test_constants.js b/spec/javascripts/test_constants.js index 77c206585fe..c97d47a6406 100644 --- a/spec/javascripts/test_constants.js +++ b/spec/javascripts/test_constants.js @@ -1,6 +1,4 @@ -export const FIXTURES_PATH = `/base/${ - process.env.IS_GITLAB_EE ? 'ee/' : '' -}spec/javascripts/fixtures`; +export const FIXTURES_PATH = `/fixtures`; export const TEST_HOST = 'http://test.host'; export const DUMMY_IMAGE_URL = `${FIXTURES_PATH}/static/images/one_white_pixel.png`; diff --git a/spec/lib/gitlab/danger/helper_spec.rb b/spec/lib/gitlab/danger/helper_spec.rb index 92d90ac2fef..f11f68ab3c2 100644 --- a/spec/lib/gitlab/danger/helper_spec.rb +++ b/spec/lib/gitlab/danger/helper_spec.rb @@ -85,6 +85,20 @@ describe Gitlab::Danger::Helper do end end + describe '#markdown_list' do + it 'creates a markdown list of items' do + items = %w[a b] + + expect(helper.markdown_list(items)).to eq("* `a`\n* `b`") + end + + it 'wraps items in <details> when there are more than 10 items' do + items = ('a'..'k').to_a + + expect(helper.markdown_list(items)).to match(%r{<details>[^<]+</details>}) + end + end + describe '#changes_by_category' do it 'categorizes changed files' do expect(fake_git).to receive(:added_files) { %w[foo foo.md foo.rb foo.js db/foo lib/gitlab/database/foo.rb qa/foo ee/changelogs/foo.yml] } @@ -224,4 +238,20 @@ describe Gitlab::Danger::Helper do expect(teammates.map(&:username)).to eq(usernames) end end + + describe '#missing_database_labels' do + subject { helper.missing_database_labels(current_mr_labels) } + + context 'when current merge request has ~database::review pending' do + let(:current_mr_labels) { ['database::review pending', 'feature'] } + + it { is_expected.to match_array(['database']) } + end + + context 'when current merge request does not have ~database::review pending' do + let(:current_mr_labels) { ['feature'] } + + it { is_expected.to match_array(['database', 'database::review pending']) } + end + end end diff --git a/spec/lib/gitlab/database/migration_helpers_spec.rb b/spec/lib/gitlab/database/migration_helpers_spec.rb index 7409572288c..dd0033bbc14 100644 --- a/spec/lib/gitlab/database/migration_helpers_spec.rb +++ b/spec/lib/gitlab/database/migration_helpers_spec.rb @@ -9,9 +9,27 @@ describe Gitlab::Database::MigrationHelpers do allow(model).to receive(:puts) end + describe '#remove_timestamps' do + it 'can remove the default timestamps' do + Gitlab::Database::MigrationHelpers::DEFAULT_TIMESTAMP_COLUMNS.each do |column_name| + expect(model).to receive(:remove_column).with(:foo, column_name) + end + + model.remove_timestamps(:foo) + end + + it 'can remove custom timestamps' do + expect(model).to receive(:remove_column).with(:foo, :bar) + + model.remove_timestamps(:foo, columns: [:bar]) + end + end + describe '#add_timestamps_with_timezone' do + let(:in_transaction) { false } + before do - allow(model).to receive(:transaction_open?).and_return(false) + allow(model).to receive(:transaction_open?).and_return(in_transaction) end context 'using PostgreSQL' do @@ -21,11 +39,64 @@ describe Gitlab::Database::MigrationHelpers do end it 'adds "created_at" and "updated_at" fields with the "datetime_with_timezone" data type' do - expect(model).to receive(:add_column).with(:foo, :created_at, :datetime_with_timezone, { null: false }) - expect(model).to receive(:add_column).with(:foo, :updated_at, :datetime_with_timezone, { null: false }) + Gitlab::Database::MigrationHelpers::DEFAULT_TIMESTAMP_COLUMNS.each do |column_name| + expect(model).to receive(:add_column).with(:foo, column_name, :datetime_with_timezone, { null: false }) + end model.add_timestamps_with_timezone(:foo) end + + it 'can disable the NOT NULL constraint' do + Gitlab::Database::MigrationHelpers::DEFAULT_TIMESTAMP_COLUMNS.each do |column_name| + expect(model).to receive(:add_column).with(:foo, column_name, :datetime_with_timezone, { null: true }) + end + + model.add_timestamps_with_timezone(:foo, null: true) + end + + it 'can add just one column' do + expect(model).to receive(:add_column).with(:foo, :created_at, :datetime_with_timezone, anything) + expect(model).not_to receive(:add_column).with(:foo, :updated_at, :datetime_with_timezone, anything) + + model.add_timestamps_with_timezone(:foo, columns: [:created_at]) + end + + it 'can add choice of acceptable columns' do + expect(model).to receive(:add_column).with(:foo, :created_at, :datetime_with_timezone, anything) + expect(model).to receive(:add_column).with(:foo, :deleted_at, :datetime_with_timezone, anything) + expect(model).not_to receive(:add_column).with(:foo, :updated_at, :datetime_with_timezone, anything) + + model.add_timestamps_with_timezone(:foo, columns: [:created_at, :deleted_at]) + end + + it 'cannot add unacceptable column names' do + expect do + model.add_timestamps_with_timezone(:foo, columns: [:bar]) + end.to raise_error %r/Illegal timestamp column name/ + end + + context 'in a transaction' do + let(:in_transaction) { true } + + before do + allow(model).to receive(:add_column).with(any_args).and_call_original + allow(model).to receive(:add_column) + .with(:foo, anything, :datetime_with_timezone, anything) + .and_return(nil) + end + + it 'cannot add a default value' do + expect do + model.add_timestamps_with_timezone(:foo, default: :i_cause_an_error) + end.to raise_error %r/add_timestamps_with_timezone/ + end + + it 'can add columns without defaults' do + expect do + model.add_timestamps_with_timezone(:foo) + end.not_to raise_error + end + end end context 'using MySQL' do diff --git a/spec/lib/gitlab/git_spec.rb b/spec/lib/gitlab/git_spec.rb index ce15057dd7d..6515be85ae3 100644 --- a/spec/lib/gitlab/git_spec.rb +++ b/spec/lib/gitlab/git_spec.rb @@ -39,6 +39,26 @@ describe Gitlab::Git do end end + describe '.commit_id?' do + using RSpec::Parameterized::TableSyntax + + where(:sha, :result) do + '' | false + 'foobar' | false + '4b825dc' | false + 'zzz25dc642cb6eb9a060e54bf8d69288fbee4904' | false + + '4b825dc642cb6eb9a060e54bf8d69288fbee4904' | true + Gitlab::Git::BLANK_SHA | true + end + + with_them do + it 'returns the expected result' do + expect(described_class.commit_id?(sha)).to eq(result) + end + end + end + describe '.shas_eql?' do using RSpec::Parameterized::TableSyntax diff --git a/spec/lib/gitlab/gitaly_client_spec.rb b/spec/lib/gitlab/gitaly_client_spec.rb index b8debee3b58..e1d24ae8977 100644 --- a/spec/lib/gitlab/gitaly_client_spec.rb +++ b/spec/lib/gitlab/gitaly_client_spec.rb @@ -119,6 +119,19 @@ describe Gitlab::GitalyClient do end end + describe '.can_use_disk?' do + it 'properly caches a false result' do + # spec_helper stubs this globally + allow(described_class).to receive(:can_use_disk?).and_call_original + expect(described_class).to receive(:filesystem_id).once + expect(described_class).to receive(:filesystem_id_from_disk).once + + 2.times do + described_class.can_use_disk?('unknown') + end + end + end + describe '.connection_data' do it 'returns connection data' do address = 'tcp://localhost:9876' diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb index e6ce3f1bcea..3b7de185cf1 100644 --- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb @@ -496,6 +496,18 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do end end + context 'with restricted internal visibility' do + describe 'internal project' do + let(:visibility) { Gitlab::VisibilityLevel::INTERNAL } + + it 'uses private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + + expect(restorer.restored_project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end + end + end + context 'with group visibility' do before do group = create(:group, visibility_level: group_visibility) @@ -528,6 +540,14 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do it 'uses the group visibility' do expect(restorer.restored_project.visibility_level).to eq(group_visibility) end + + context 'with restricted internal visibility' do + it 'sets private visibility' do + stub_application_setting(restricted_visibility_levels: [Gitlab::VisibilityLevel::INTERNAL]) + + expect(restorer.restored_project.visibility_level).to eq(Gitlab::VisibilityLevel::PRIVATE) + end + end end end end diff --git a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb index 6795c1ab56b..e04056b3450 100644 --- a/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb +++ b/spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb @@ -201,7 +201,15 @@ describe Gitlab::Metrics::Subscribers::RailsCache do it 'observes cache metric' do expect(subscriber.send(:metric_cache_operation_duration_seconds)) .to receive(:observe) - .with(transaction.labels.merge(operation: :delete), event.duration / 1000.0) + .with({ operation: :delete }, event.duration / 1000.0) + + subscriber.observe(:delete, event.duration) + end + + it 'increments the operations total' do + expect(subscriber.send(:metric_cache_operations_total)) + .to receive(:increment) + .with(transaction.labels.merge(operation: :delete)) subscriber.observe(:delete, event.duration) end diff --git a/spec/lib/gitlab/rugged_instrumentation_spec.rb b/spec/lib/gitlab/rugged_instrumentation_spec.rb new file mode 100644 index 00000000000..4dcc8ae514a --- /dev/null +++ b/spec/lib/gitlab/rugged_instrumentation_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::RuggedInstrumentation, :request_store do + subject { described_class } + + describe '.query_time' do + it 'increments query times' do + subject.query_time += 0.451 + subject.query_time += 0.322 + + expect(subject.query_time).to be_within(0.001).of(0.773) + expect(subject.query_time_ms).to eq(773.0) + end + end + + context '.increment_query_count' do + it 'tracks query counts' do + expect(subject.query_count).to eq(0) + + 2.times { subject.increment_query_count } + + expect(subject.query_count).to eq(2) + end + end +end diff --git a/spec/lib/gitlab/slug/environment_spec.rb b/spec/lib/gitlab/slug/environment_spec.rb new file mode 100644 index 00000000000..7dc583a94b8 --- /dev/null +++ b/spec/lib/gitlab/slug/environment_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Slug::Environment do + describe '#generate' do + { + "staging-12345678901234567" => "staging-123456789-q517sa", + "9-staging-123456789012345" => "env-9-staging-123-q517sa", + "staging-1234567890123456" => "staging-1234567890123456", + "staging-1234567890123456-" => "staging-123456789-q517sa", + "production" => "production", + "PRODUCTION" => "production-q517sa", + "review/1-foo" => "review-1-foo-q517sa", + "1-foo" => "env-1-foo-q517sa", + "1/foo" => "env-1-foo-q517sa", + "foo-" => "foo", + "foo--bar" => "foo-bar-q517sa", + "foo**bar" => "foo-bar-q517sa", + "*-foo" => "env-foo-q517sa", + "staging-12345678-" => "staging-12345678", + "staging-12345678-01234567" => "staging-12345678-q517sa", + "" => "env-q517sa", + nil => "env-q517sa" + }.each do |name, matcher| + before do + # ('a' * 64).to_i(16).to_s(36).last(6) gives 'q517sa' + allow(Digest::SHA2).to receive(:hexdigest).with(name).and_return('a' * 64) + end + + it "returns a slug matching #{matcher}, given #{name}" do + slug = described_class.new(name).generate + + expect(slug).to match(/\A#{matcher}\z/) + end + end + end +end diff --git a/spec/lib/gitlab/usage_data_counters/redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/redis_counter_spec.rb new file mode 100644 index 00000000000..c34ac7867ab --- /dev/null +++ b/spec/lib/gitlab/usage_data_counters/redis_counter_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::UsageDataCounters::RedisCounter, :clean_gitlab_redis_shared_state do + let(:redis_key) { 'foobar' } + + subject { Class.new.extend(described_class) } + + before do + stub_application_setting(usage_ping_enabled: setting_value) + end + + context 'when usage_ping is disabled' do + let(:setting_value) { false } + + it 'counter is not increased' do + expect do + subject.increment(redis_key) + end.not_to change { subject.total_count(redis_key) } + end + end + + context 'when usage_ping is enabled' do + let(:setting_value) { true } + + it 'counter is increased' do + expect do + subject.increment(redis_key) + end.to change { subject.total_count(redis_key) }.by(1) + end + end +end diff --git a/spec/lib/gitlab/usage_data_counters/web_ide_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/web_ide_counter_spec.rb index fa0cf15e1b2..7a01f7d1de8 100644 --- a/spec/lib/gitlab/usage_data_counters/web_ide_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/web_ide_counter_spec.rb @@ -3,19 +3,54 @@ require 'spec_helper' describe Gitlab::UsageDataCounters::WebIdeCounter, :clean_gitlab_redis_shared_state do - describe '.increment_commits_count' do - it 'increments the web ide commits counter by 1' do - expect do - described_class.increment_commits_count - end.to change { described_class.total_commits_count }.by(1) + shared_examples 'counter examples' do + it 'increments counter and return the total count' do + expect(described_class.public_send(total_counter_method)).to eq(0) + + 2.times { described_class.public_send(increment_counter_method) } + + expect(described_class.public_send(total_counter_method)).to eq(2) end end - describe '.total_commits_count' do - it 'returns the total amount of web ide commits' do - 2.times { described_class.increment_commits_count } + describe 'commits counter' do + let(:increment_counter_method) { :increment_commits_count } + let(:total_counter_method) { :total_commits_count } + + it_behaves_like 'counter examples' + end + + describe 'merge requests counter' do + let(:increment_counter_method) { :increment_merge_requests_count } + let(:total_counter_method) { :total_merge_requests_count } + + it_behaves_like 'counter examples' + end + + describe 'views counter' do + let(:increment_counter_method) { :increment_views_count } + let(:total_counter_method) { :total_views_count } + + it_behaves_like 'counter examples' + end + + describe '.totals' do + commits = 5 + merge_requests = 3 + views = 2 + + before do + commits.times { described_class.increment_commits_count } + merge_requests.times { described_class.increment_merge_requests_count } + views.times { described_class.increment_views_count } + end - expect(described_class.total_commits_count).to eq(2) + it 'can report all totals' do + expect(described_class.totals).to include( + web_ide_commits: commits, + web_ide_views: views, + web_ide_merge_requests: merge_requests + ) end end end diff --git a/spec/lib/gitlab/usage_data_counters/wiki_page_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/wiki_page_counter_spec.rb new file mode 100644 index 00000000000..41afbbb191c --- /dev/null +++ b/spec/lib/gitlab/usage_data_counters/wiki_page_counter_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::UsageDataCounters::WikiPageCounter, :clean_gitlab_redis_shared_state do + shared_examples :wiki_page_event do |event| + describe ".count(#{event})" do + it "increments the wiki page #{event} counter by 1" do + expect do + described_class.count(event) + end.to change { described_class.read(event) }.by 1 + end + end + + describe ".read(#{event})" do + event_count = 5 + + it "returns the total number of #{event} events" do + event_count.times do + described_class.count(event) + end + + expect(described_class.read(event)).to eq(event_count) + end + end + end + + include_examples :wiki_page_event, :create + include_examples :wiki_page_event, :update + include_examples :wiki_page_event, :delete + + describe 'totals' do + creations = 5 + edits = 3 + deletions = 2 + + before do + creations.times do + described_class.count(:create) + end + edits.times do + described_class.count(:update) + end + deletions.times do + described_class.count(:delete) + end + end + + it 'can report all totals' do + expect(described_class.totals).to include( + wiki_pages_update: edits, + wiki_pages_create: creations, + wiki_pages_delete: deletions + ) + end + end + + describe 'unknown events' do + error = described_class::UnknownEvent + + it 'cannot increment' do + expect { described_class.count(:wibble) }.to raise_error error + end + + it 'cannot read' do + expect { described_class.read(:wibble) }.to raise_error error + end + end +end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 90a534de202..2289d906944 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -57,10 +57,18 @@ describe Gitlab::UsageData do gitaly database avg_cycle_analytics - web_ide_commits influxdb_metrics_enabled prometheus_metrics_enabled )) + + expect(subject).to include( + wiki_pages_create: a_kind_of(Integer), + wiki_pages_update: a_kind_of(Integer), + wiki_pages_delete: a_kind_of(Integer), + web_ide_views: a_kind_of(Integer), + web_ide_commits: a_kind_of(Integer), + web_ide_merge_requests: a_kind_of(Integer) + ) end it "gathers usage counts" do @@ -182,6 +190,28 @@ describe Gitlab::UsageData do end end + describe '#usage_data_counters' do + subject { described_class.usage_data_counters } + + it { is_expected.to all(respond_to :totals) } + + describe 'the results of calling #totals on all objects in the array' do + subject { described_class.usage_data_counters.map(&:totals) } + + it do + is_expected + .to all(be_a Hash) + .and all(have_attributes(keys: all(be_a Symbol), values: all(be_a Integer))) + end + end + + it 'does not have any conflicts' do + all_keys = subject.flat_map { |counter| counter.totals.keys } + + expect(all_keys.size).to eq all_keys.to_set.size + end + end + describe '#features_usage_data_ce' do subject { described_class.features_usage_data_ce } diff --git a/spec/lib/peek/views/redis_detailed_spec.rb b/spec/lib/peek/views/redis_detailed_spec.rb index da13b6df53b..61096e6c69e 100644 --- a/spec/lib/peek/views/redis_detailed_spec.rb +++ b/spec/lib/peek/views/redis_detailed_spec.rb @@ -2,14 +2,8 @@ require 'spec_helper' -describe Peek::Views::RedisDetailed do - let(:redis_detailed_class) do - Class.new do - include Peek::Views::RedisDetailed - end - end - - subject { redis_detailed_class.new } +describe Peek::Views::RedisDetailed, :request_store do + subject { described_class.new } using RSpec::Parameterized::TableSyntax @@ -22,15 +16,24 @@ describe Peek::Views::RedisDetailed do end with_them do - it 'scrubs Redis commands', :request_store do + it 'scrubs Redis commands' do subject.detail_store << { cmd: cmd, duration: 1.second } - expect(subject.details.count).to eq(1) - expect(subject.details.first) + expect(subject.results[:details].count).to eq(1) + expect(subject.results[:details].first) .to eq({ cmd: expected, duration: 1000 }) end end + + it 'returns aggregated results' do + subject.detail_store << { cmd: [:get, 'test'], duration: 0.001 } + subject.detail_store << { cmd: [:get, 'test'], duration: 1.second } + + expect(subject.results[:calls]).to eq(2) + expect(subject.results[:duration]).to eq('1001.00ms') + expect(subject.results[:details].count).to eq(2) + end end diff --git a/spec/lib/prometheus/pid_provider_spec.rb b/spec/lib/prometheus/pid_provider_spec.rb new file mode 100644 index 00000000000..e7d500612b1 --- /dev/null +++ b/spec/lib/prometheus/pid_provider_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +describe Prometheus::PidProvider do + describe '.worker_id' do + subject { described_class.worker_id } + + let(:sidekiq_module) { Module.new } + + before do + allow(sidekiq_module).to receive(:server?).and_return(false) + stub_const('Sidekiq', sidekiq_module) + end + + context 'when running in Sidekiq server mode' do + before do + expect(Sidekiq).to receive(:server?).and_return(true) + end + + it { is_expected.to eq 'sidekiq' } + end + + context 'when running in Unicorn mode' do + before do + stub_const('Unicorn::Worker', Class.new) + hide_const('Puma') + end + + context 'when `Prometheus::Client::Support::Unicorn` provides worker_id' do + before do + expect(::Prometheus::Client::Support::Unicorn).to receive(:worker_id).and_return(1) + end + + it { is_expected.to eq 'unicorn_1' } + end + + context 'when no worker_id is provided from `Prometheus::Client::Support::Unicorn`' do + before do + expect(::Prometheus::Client::Support::Unicorn).to receive(:worker_id).and_return(nil) + end + + it { is_expected.to eq 'unicorn_master' } + end + end + + context 'when running in Puma mode' do + before do + stub_const('Puma', Module.new) + hide_const('Unicorn::Worker') + end + + context 'when cluster worker id is specified in process name' do + before do + expect(described_class).to receive(:process_name).and_return('puma: cluster worker 1: 17483 [gitlab-puma-worker]') + end + + it { is_expected.to eq 'puma_1' } + end + + context 'when no worker id is specified in process name' do + before do + expect(described_class).to receive(:process_name).and_return('bin/puma') + end + + it { is_expected.to eq 'puma_master' } + end + end + + context 'when running in unknown mode' do + before do + hide_const('Puma') + hide_const('Unicorn::Worker') + end + + it { is_expected.to eq "process_#{Process.pid}" } + end + end +end diff --git a/spec/models/concerns/issuable_spec.rb b/spec/models/concerns/issuable_spec.rb index 68224a56515..e19da41c3fe 100644 --- a/spec/models/concerns/issuable_spec.rb +++ b/spec/models/concerns/issuable_spec.rb @@ -774,4 +774,25 @@ describe Issuable do end end end + + describe '#supports_milestone?' do + let(:group) { create(:group) } + let(:project) { create(:project, group: group) } + + context "for issues" do + let(:issue) { build(:issue, project: project) } + + it 'returns true' do + expect(issue.supports_milestone?).to be_truthy + end + end + + context "for merge requests" do + let(:merge_request) { build(:merge_request, target_project: project, source_project: project) } + + it 'returns true' do + expect(merge_request.supports_milestone?).to be_truthy + end + end + end end diff --git a/spec/models/concerns/project_api_compatibility_spec.rb b/spec/models/concerns/project_api_compatibility_spec.rb index 8cecd4fe7bc..f5722f88aac 100644 --- a/spec/models/concerns/project_api_compatibility_spec.rb +++ b/spec/models/concerns/project_api_compatibility_spec.rb @@ -16,23 +16,44 @@ describe ProjectAPICompatibility do expect(project.build_allow_git_fetch).to eq(false) end - # auto_devops_enabled - it "converts auto_devops_enabled=false to auto_devops_enabled?=false" do - expect(project.auto_devops_enabled?).to eq(true) - project.update!(auto_devops_enabled: false) - expect(project.auto_devops_enabled?).to eq(false) - end + describe '#auto_devops_enabled' do + where( + initial: [:missing, nil, false, true], + final: [nil, false, true] + ) + + with_them do + before do + project.build_auto_devops(enabled: initial) unless initial == :missing + end + + # Implicit auto devops when enabled is nil + let(:expected) { final.nil? ? true : final } + + it 'sets the correct value' do + project.update!(auto_devops_enabled: final) - it "converts auto_devops_enabled=true to auto_devops_enabled?=true" do - expect(project.auto_devops_enabled?).to eq(true) - project.update!(auto_devops_enabled: true) - expect(project.auto_devops_enabled?).to eq(true) + expect(project.auto_devops_enabled?).to eq(expected) + end + end end - # auto_devops_deploy_strategy - it "converts auto_devops_deploy_strategy=timed_incremental to auto_devops.deploy_strategy=timed_incremental" do - expect(project.auto_devops).to be_nil - project.update!(auto_devops_deploy_strategy: 'timed_incremental') - expect(project.auto_devops.deploy_strategy).to eq('timed_incremental') + describe '#auto_devops_deploy_strategy' do + where( + initial: [:missing, *ProjectAutoDevops.deploy_strategies.keys], + final: ProjectAutoDevops.deploy_strategies.keys + ) + + with_them do + before do + project.build_auto_devops(deploy_strategy: initial) unless initial == :missing + end + + it 'sets the correct value' do + project.update!(auto_devops_deploy_strategy: final) + + expect(project.auto_devops.deploy_strategy).to eq(final) + end + end end end diff --git a/spec/models/environment_spec.rb b/spec/models/environment_spec.rb index fe4d64818b4..ce0681c4331 100644 --- a/spec/models/environment_spec.rb +++ b/spec/models/environment_spec.rb @@ -762,32 +762,6 @@ describe Environment, :use_clean_rails_memory_store_caching do end end - describe '#generate_slug' do - SUFFIX = "-[a-z0-9]{6}".freeze - { - "staging-12345678901234567" => "staging-123456789" + SUFFIX, - "9-staging-123456789012345" => "env-9-staging-123" + SUFFIX, - "staging-1234567890123456" => "staging-1234567890123456", - "production" => "production", - "PRODUCTION" => "production" + SUFFIX, - "review/1-foo" => "review-1-foo" + SUFFIX, - "1-foo" => "env-1-foo" + SUFFIX, - "1/foo" => "env-1-foo" + SUFFIX, - "foo-" => "foo" + SUFFIX, - "foo--bar" => "foo-bar" + SUFFIX, - "foo**bar" => "foo-bar" + SUFFIX, - "*-foo" => "env-foo" + SUFFIX, - "staging-12345678-" => "staging-12345678" + SUFFIX, - "staging-12345678-01234567" => "staging-12345678" + SUFFIX - }.each do |name, matcher| - it "returns a slug matching #{matcher}, given #{name}" do - slug = described_class.new(name: name).generate_slug - - expect(slug).to match(/\A#{matcher}\z/) - end - end - end - describe '#ref_path' do subject(:environment) do create(:environment, name: 'staging / review-1') diff --git a/spec/models/project_services/jira_service_spec.rb b/spec/models/project_services/jira_service_spec.rb index 235cf314af5..02060699e9a 100644 --- a/spec/models/project_services/jira_service_spec.rb +++ b/spec/models/project_services/jira_service_spec.rb @@ -236,7 +236,7 @@ describe JiraService do allow(JIRA::Resource::Remotelink).to receive(:all).and_return(nil) expect { @jira_service.close_issue(resource, ExternalIssue.new('JIRA-123', project)) } - .not_to raise_error(NoMethodError) + .not_to raise_error end # Check https://developer.atlassian.com/jiradev/jira-platform/guides/other/guide-jira-remote-issue-links/fields-in-remote-issue-links @@ -606,6 +606,12 @@ describe JiraService do expect(service.properties['api_url']).to eq('http://jira.sample/api') end end + + it 'removes trailing slashes from url' do + service = described_class.new(url: 'http://jira.test.com/path/') + + expect(service.url).to eq('http://jira.test.com/path') + end end describe 'favicon urls', :request_store do @@ -621,4 +627,20 @@ describe JiraService do expect(props[:object][:icon][:url16x16]).to match %r{^http://localhost/uploads/-/system/appearance/favicon/\d+/dk.png$} end end + + context 'generating external URLs' do + let(:service) { described_class.new(url: 'http://jira.test.com/path/') } + + describe '#issues_url' do + it 'handles trailing slashes' do + expect(service.issues_url).to eq('http://jira.test.com/path/browse/:id') + end + end + + describe '#new_issue_url' do + it 'handles trailing slashes' do + expect(service.new_issue_url).to eq('http://jira.test.com/path/secure/CreateIssue.jspa') + end + end + end end diff --git a/spec/requests/api/commits_spec.rb b/spec/requests/api/commits_spec.rb index 45da5391bc8..e8e17228523 100644 --- a/spec/requests/api/commits_spec.rb +++ b/spec/requests/api/commits_spec.rb @@ -320,67 +320,132 @@ describe API::Commits do end end - context 'when the API user is a guest' do + context 'when committing to a new branch' do def last_commit_id(project, branch_name) project.repository.find_branch(branch_name)&.dereferenced_target&.id end - let(:public_project) { create(:project, :public, :repository) } - let!(:url) { "/projects/#{public_project.id}/repository/commits" } - let(:guest) { create(:user).tap { |u| public_project.add_guest(u) } } + before do + valid_c_params[:start_branch] = 'master' + valid_c_params[:branch] = 'patch' + end - it 'returns a 403' do - post api(url, guest), params: valid_c_params + context 'when the API user is a guest' do + let(:public_project) { create(:project, :public, :repository) } + let(:url) { "/projects/#{public_project.id}/repository/commits" } + let(:guest) { create(:user).tap { |u| public_project.add_guest(u) } } - expect(response).to have_gitlab_http_status(403) - end + it 'returns a 403' do + post api(url, guest), params: valid_c_params - context 'when start_project is provided' do - context 'when posting to a forked project the user owns' do - let!(:forked_project) { fork_project(public_project, guest, namespace: guest.namespace, repository: true) } - let!(:url) { "/projects/#{forked_project.id}/repository/commits" } + expect(response).to have_gitlab_http_status(403) + end - before do - valid_c_params[:start_branch] = "master" - valid_c_params[:branch] = "patch" - end + context 'when start_project is provided' do + context 'when posting to a forked project the user owns' do + let(:forked_project) { fork_project(public_project, guest, namespace: guest.namespace, repository: true) } + let(:url) { "/projects/#{forked_project.id}/repository/commits" } + + context 'identified by Integer (id)' do + before do + valid_c_params[:start_project] = public_project.id + end + + it 'adds a new commit to forked_project and returns a 201' do + expect_request_with_status(201) { post api(url, guest), params: valid_c_params } + .to change { last_commit_id(forked_project, valid_c_params[:branch]) } + .and not_change { last_commit_id(public_project, valid_c_params[:start_branch]) } + end + end - context 'identified by Integer (id)' do - before do - valid_c_params[:start_project] = public_project.id + context 'identified by String (full_path)' do + before do + valid_c_params[:start_project] = public_project.full_path + end + + it 'adds a new commit to forked_project and returns a 201' do + expect_request_with_status(201) { post api(url, guest), params: valid_c_params } + .to change { last_commit_id(forked_project, valid_c_params[:branch]) } + .and not_change { last_commit_id(public_project, valid_c_params[:start_branch]) } + end end - it 'adds a new commit to forked_project and returns a 201' do - expect { post api(url, guest), params: valid_c_params } - .to change { last_commit_id(forked_project, valid_c_params[:branch]) } - .and not_change { last_commit_id(public_project, valid_c_params[:start_branch]) } + context 'when branch already exists' do + before do + valid_c_params.delete(:start_branch) + valid_c_params[:branch] = 'master' + valid_c_params[:start_project] = public_project.id + end + + it 'returns a 400' do + post api(url, guest), params: valid_c_params + + expect(response).to have_gitlab_http_status(400) + expect(json_response['message']).to eq("A branch called 'master' already exists. Switch to that branch in order to make changes") + end + + context 'when force is set to true' do + before do + valid_c_params[:force] = true + end + + it 'adds a new commit to forked_project and returns a 201' do + expect_request_with_status(201) { post api(url, guest), params: valid_c_params } + .to change { last_commit_id(forked_project, valid_c_params[:branch]) } + .and not_change { last_commit_id(public_project, valid_c_params[:branch]) } + end + end + end + + context 'when start_sha is also provided' do + let(:forked_project) { fork_project(public_project, guest, namespace: guest.namespace, repository: false) } + let(:start_sha) { public_project.repository.commit.parent.sha } + + before do + # initialize an empty repository to force fetching from the original project + forked_project.repository.create_if_not_exists - expect(response).to have_gitlab_http_status(201) + valid_c_params[:start_project] = public_project.id + valid_c_params[:start_sha] = start_sha + valid_c_params.delete(:start_branch) + end + + it 'fetches the start_sha from the original project to use as parent commit and returns a 201' do + expect_request_with_status(201) { post api(url, guest), params: valid_c_params } + .to change { last_commit_id(forked_project, valid_c_params[:branch]) } + .and not_change { last_commit_id(forked_project, 'master') } + + last_commit = forked_project.repository.find_branch(valid_c_params[:branch]).dereferenced_target + expect(last_commit.parent_id).to eq(start_sha) + end end end - context 'identified by String (full_path)' do + context 'when the target project is not part of the fork network of start_project' do + let(:unrelated_project) { create(:project, :public, :repository, creator: guest) } + let(:url) { "/projects/#{unrelated_project.id}/repository/commits" } + before do - valid_c_params[:start_project] = public_project.full_path + valid_c_params[:start_branch] = 'master' + valid_c_params[:branch] = 'patch' + valid_c_params[:start_project] = public_project.id end - it 'adds a new commit to forked_project and returns a 201' do - expect { post api(url, guest), params: valid_c_params } - .to change { last_commit_id(forked_project, valid_c_params[:branch]) } - .and not_change { last_commit_id(public_project, valid_c_params[:start_branch]) } + it 'returns a 403' do + post api(url, guest), params: valid_c_params - expect(response).to have_gitlab_http_status(201) + expect(response).to have_gitlab_http_status(403) end end end - context 'when the target project is not part of the fork network of start_project' do - let(:unrelated_project) { create(:project, :public, :repository, creator: guest) } - let!(:url) { "/projects/#{unrelated_project.id}/repository/commits" } + context 'when posting to a forked project the user does not have write access' do + let(:forked_project) { fork_project(public_project, user, namespace: user.namespace, repository: true) } + let(:url) { "/projects/#{forked_project.id}/repository/commits" } before do - valid_c_params[:start_branch] = "master" - valid_c_params[:branch] = "patch" + valid_c_params[:start_branch] = 'master' + valid_c_params[:branch] = 'patch' valid_c_params[:start_project] = public_project.id end @@ -392,20 +457,68 @@ describe API::Commits do end end - context 'when posting to a forked project the user does not have write access' do - let!(:forked_project) { fork_project(public_project, user, namespace: user.namespace, repository: true) } - let!(:url) { "/projects/#{forked_project.id}/repository/commits" } + context 'when start_sha is provided' do + let(:start_sha) { project.repository.commit.parent.sha } before do - valid_c_params[:start_branch] = "master" - valid_c_params[:branch] = "patch" - valid_c_params[:start_project] = public_project.id + valid_c_params[:start_sha] = start_sha + valid_c_params.delete(:start_branch) end - it 'returns a 403' do - post api(url, guest), params: valid_c_params + it 'returns a 400 if start_branch is also provided' do + valid_c_params[:start_branch] = 'master' + post api(url, user), params: valid_c_params - expect(response).to have_gitlab_http_status(403) + expect(response).to have_gitlab_http_status(400) + expect(json_response['error']).to eq('start_branch, start_sha are mutually exclusive') + end + + it 'returns a 400 if branch already exists' do + valid_c_params[:branch] = 'master' + post api(url, user), params: valid_c_params + + expect(response).to have_gitlab_http_status(400) + expect(json_response['message']).to eq("A branch called 'master' already exists. Switch to that branch in order to make changes") + end + + it 'returns a 400 if start_sha does not exist' do + valid_c_params[:start_sha] = '1' * 40 + post api(url, user), params: valid_c_params + + expect(response).to have_gitlab_http_status(400) + expect(json_response['message']).to eq("Cannot find start_sha '#{valid_c_params[:start_sha]}'") + end + + it 'returns a 400 if start_sha is not a full SHA' do + valid_c_params[:start_sha] = start_sha.slice(0, 7) + post api(url, user), params: valid_c_params + + expect(response).to have_gitlab_http_status(400) + expect(json_response['message']).to eq("Invalid start_sha '#{valid_c_params[:start_sha]}'") + end + + it 'uses the start_sha as parent commit and returns a 201' do + expect_request_with_status(201) { post api(url, user), params: valid_c_params } + .to change { last_commit_id(project, valid_c_params[:branch]) } + .and not_change { last_commit_id(project, 'master') } + + last_commit = project.repository.find_branch(valid_c_params[:branch]).dereferenced_target + expect(last_commit.parent_id).to eq(start_sha) + end + + context 'when force is set to true and branch already exists' do + before do + valid_c_params[:force] = true + valid_c_params[:branch] = 'master' + end + + it 'uses the start_sha as parent commit and returns a 201' do + expect_request_with_status(201) { post api(url, user), params: valid_c_params } + .to change { last_commit_id(project, valid_c_params[:branch]) } + + last_commit = project.repository.find_branch(valid_c_params[:branch]).dereferenced_target + expect(last_commit.parent_id).to eq(start_sha) + end end end end diff --git a/spec/requests/request_profiler_spec.rb b/spec/requests/request_profiler_spec.rb index 75b22b1879b..851affbcf88 100644 --- a/spec/requests/request_profiler_spec.rb +++ b/spec/requests/request_profiler_spec.rb @@ -3,13 +3,18 @@ require 'spec_helper' describe 'Request Profiler' do let(:user) { create(:user) } - shared_examples 'profiling a request' do + shared_examples 'profiling a request' do |profile_type, extension| before do allow(Rails).to receive(:cache).and_return(ActiveSupport::Cache::MemoryStore.new) allow(RubyProf::Profile).to receive(:profile) do |&blk| blk.call RubyProf::Profile.new end + allow(MemoryProfiler).to receive(:report) do |&blk| + blk.call + MemoryProfiler.start + MemoryProfiler.stop + end end it 'creates a profile of the request' do @@ -18,10 +23,11 @@ describe 'Request Profiler' do path = "/#{project.full_path}" Timecop.freeze(time) do - get path, params: {}, headers: { 'X-Profile-Token' => Gitlab::RequestProfiler.profile_token } + get path, params: {}, headers: { 'X-Profile-Token' => Gitlab::RequestProfiler.profile_token, 'X-Profile-Mode' => profile_type } end - profile_path = "#{Gitlab.config.shared.path}/tmp/requests_profiles/#{path.tr('/', '|')}_#{time.to_i}.html" + profile_type = 'execution' if profile_type.nil? + profile_path = "#{Gitlab.config.shared.path}/tmp/requests_profiles/#{path.tr('/', '|')}_#{time.to_i}_#{profile_type}.#{extension}" expect(File.exist?(profile_path)).to be true end @@ -35,10 +41,14 @@ describe 'Request Profiler' do login_as(user) end - include_examples 'profiling a request' + include_examples 'profiling a request', 'execution', 'html' + include_examples 'profiling a request', nil, 'html' + include_examples 'profiling a request', 'memory', 'txt' end context "when user is not logged-in" do - include_examples 'profiling a request' + include_examples 'profiling a request', 'execution', 'html' + include_examples 'profiling a request', nil, 'html' + include_examples 'profiling a request', 'memory', 'txt' end end diff --git a/spec/services/submodules/update_service_spec.rb b/spec/services/submodules/update_service_spec.rb index cf92350c1b2..47b31d4bcbf 100644 --- a/spec/services/submodules/update_service_spec.rb +++ b/spec/services/submodules/update_service_spec.rb @@ -142,7 +142,7 @@ describe Submodules::UpdateService do let(:branch_name) { nil } it_behaves_like 'returns error result' do - let(:error_message) { 'You can only create or edit files when you are on a branch' } + let(:error_message) { 'Invalid parameters' } end end diff --git a/spec/services/wiki_pages/base_service_spec.rb b/spec/services/wiki_pages/base_service_spec.rb new file mode 100644 index 00000000000..2e70246c6f2 --- /dev/null +++ b/spec/services/wiki_pages/base_service_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe WikiPages::BaseService do + let(:project) { double('project') } + let(:user) { double('user') } + + subject(:service) { described_class.new(project, user, {}) } + + describe '#increment_usage' do + counter = Gitlab::UsageDataCounters::WikiPageCounter + error = counter::UnknownEvent + + it 'raises an error on unknown events' do + expect { subject.send(:increment_usage, :bad_event) }.to raise_error error + end + + context 'the event is valid' do + counter::KNOWN_EVENTS.each do |e| + it "updates the #{e} counter" do + expect { subject.send(:increment_usage, e) }.to change { counter.read(e) } + end + end + end + end +end diff --git a/spec/services/wiki_pages/create_service_spec.rb b/spec/services/wiki_pages/create_service_spec.rb index 84510dcf700..ef03a2e9788 100644 --- a/spec/services/wiki_pages/create_service_spec.rb +++ b/spec/services/wiki_pages/create_service_spec.rb @@ -14,6 +14,10 @@ describe WikiPages::CreateService do } end + let(:bad_opts) do + { title: '' } + end + subject(:service) { described_class.new(project, user, opts) } before do @@ -36,5 +40,26 @@ describe WikiPages::CreateService do service.execute end + + it 'counts wiki page creation' do + counter = Gitlab::UsageDataCounters::WikiPageCounter + + expect { service.execute }.to change { counter.read(:create) }.by 1 + end + + context 'when the options are bad' do + subject(:service) { described_class.new(project, user, bad_opts) } + + it 'does not count a creation event' do + counter = Gitlab::UsageDataCounters::WikiPageCounter + + expect { service.execute }.not_to change { counter.read(:create) } + end + + it 'reports the error' do + expect(service.execute).to be_invalid + .and have_attributes(errors: be_present) + end + end end end diff --git a/spec/services/wiki_pages/destroy_service_spec.rb b/spec/services/wiki_pages/destroy_service_spec.rb index c74eac4dad6..350a7eb123b 100644 --- a/spec/services/wiki_pages/destroy_service_spec.rb +++ b/spec/services/wiki_pages/destroy_service_spec.rb @@ -20,5 +20,17 @@ describe WikiPages::DestroyService do service.execute(page) end + + it 'increments the delete count' do + counter = Gitlab::UsageDataCounters::WikiPageCounter + + expect { service.execute(page) }.to change { counter.read(:delete) }.by 1 + end + + it 'does not increment the delete count if the deletion failed' do + counter = Gitlab::UsageDataCounters::WikiPageCounter + + expect { service.execute(nil) }.not_to change { counter.read(:delete) } + end end end diff --git a/spec/services/wiki_pages/update_service_spec.rb b/spec/services/wiki_pages/update_service_spec.rb index 19866bd3bfc..d5f46e7b2db 100644 --- a/spec/services/wiki_pages/update_service_spec.rb +++ b/spec/services/wiki_pages/update_service_spec.rb @@ -16,6 +16,10 @@ describe WikiPages::UpdateService do } end + let(:bad_opts) do + { title: '' } + end + subject(:service) { described_class.new(project, user, opts) } before do @@ -39,5 +43,26 @@ describe WikiPages::UpdateService do service.execute(page) end + + it 'counts edit events' do + counter = Gitlab::UsageDataCounters::WikiPageCounter + + expect { service.execute page }.to change { counter.read(:update) }.by 1 + end + + context 'when the options are bad' do + subject(:service) { described_class.new(project, user, bad_opts) } + + it 'does not count an edit event' do + counter = Gitlab::UsageDataCounters::WikiPageCounter + + expect { service.execute page }.not_to change { counter.read(:update) } + end + + it 'reports the error' do + expect(service.execute page).to be_invalid + .and have_attributes(errors: be_present) + end + end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 978158192d9..a6fb172e79b 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -104,6 +104,7 @@ RSpec.configure do |config| config.include Rails.application.routes.url_helpers, type: :routing config.include PolicyHelpers, type: :policy config.include MemoryUsageHelper + config.include ExpectRequestWithStatus, type: :request if ENV['CI'] # This includes the first try, i.e. tests will be run 4 times before failing. diff --git a/spec/support/helpers/expect_request_with_status.rb b/spec/support/helpers/expect_request_with_status.rb new file mode 100644 index 00000000000..0469a94e336 --- /dev/null +++ b/spec/support/helpers/expect_request_with_status.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module ExpectRequestWithStatus + def expect_request_with_status(status) + expect do + yield + + expect(response).to have_gitlab_http_status(status) + end + end +end diff --git a/spec/support/helpers/javascript_fixtures_helpers.rb b/spec/support/helpers/javascript_fixtures_helpers.rb index cdd7724cc13..e9129bd263e 100644 --- a/spec/support/helpers/javascript_fixtures_helpers.rb +++ b/spec/support/helpers/javascript_fixtures_helpers.rb @@ -19,7 +19,7 @@ module JavaScriptFixturesHelpers end def fixture_root_path - (Gitlab.ee? ? 'ee/' : '') + 'spec/javascripts/fixtures' + 'tmp/tests/frontend/fixtures' + (Gitlab.ee? ? '-ee' : '') end # Public: Removes all fixture files from given directory diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index 893b10ea752..b062631b995 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -130,7 +130,7 @@ module TestEnv # Keeps gitlab-shell and gitlab-test def clean_test_path Dir[TMP_TEST_PATH].each do |entry| - unless File.basename(entry) =~ /\A(gitaly|gitlab-(shell|test|test_bare|test-fork|test-fork_bare))\z/ + unless test_dirs.include?(File.basename(entry)) FileUtils.rm_rf(entry) end end @@ -141,14 +141,6 @@ module TestEnv FileUtils.mkdir_p(artifacts_path) end - def clean_gitlab_test_path - Dir[TMP_TEST_PATH].each do |entry| - unless test_dirs.include?(File.basename(entry)) - FileUtils.rm_rf(entry) - end - end - end - def setup_gitlab_shell component_timed_setup('GitLab Shell', install_dir: Gitlab.config.gitlab_shell.path, @@ -324,6 +316,7 @@ module TestEnv # These are directories that should be preserved at cleanup time def test_dirs @test_dirs ||= %w[ + frontend gitaly gitlab-shell gitlab-test @@ -368,10 +361,7 @@ module TestEnv # Try to reset without fetching to avoid using the network. unless reset.call raise 'Could not fetch test seed repository.' unless system(*%W(#{Gitlab.config.git.bin_path} -C #{repo_path} fetch origin)) - - # Before we used Git clone's --mirror option, bare repos could end up - # with missing refs, clearing them and retrying should fix the issue. - clean_gitlab_test_path && init unless reset.call + raise "Could not update test seed repository, please delete #{repo_path} and try again" unless reset.call end end diff --git a/yarn.lock b/yarn.lock index a0fbdfd4541..c0c1887a9d1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -705,21 +705,21 @@ resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-1.67.0.tgz#c7b94eca13b99fd3aaa737fb6dcc0abc41d3c579" integrity sha512-hJOmWEs6RkjzyKkb1vc9wwKGZIBIP0coHkxu/KgOoxhBVudpGk4CH7xJ6UuB2TKpb0SEh5CC1CzRZfBYaFhsaA== -"@gitlab/ui@^5.6.0": - version "5.6.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.6.0.tgz#6b5408050229e2135359f3fce5d0de7718326a0d" - integrity sha512-EohpACc5OCK8pOWgwB7/QZUcg3tA9k863ku6Ik9NxaRSKt/JIpQ8RI4wCr4UmqhejZLQMD9VZHLUmc9Sf3Mk9w== +"@gitlab/ui@^5.7.0": + version "5.7.0" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-5.7.0.tgz#2ce6b431de262f09b2e1dbe77b047d6e8e1ca41d" + integrity sha512-zOPFNrCGyZrgqa8OXNhtfRg4aQ6pRCpIV2+alq3/4jllYb3HHHH+jIk/ejLrvNe8+fk7LRNSvJEvuInRLqBbEg== dependencies: "@babel/standalone" "^7.0.0" "@gitlab/vue-toasted" "^1.2.1" bootstrap "4.3.1" - bootstrap-vue "^2.0.0-rc.24" + bootstrap-vue "^2.0.0-rc.26" copy-to-clipboard "^3.0.8" core-js "^2.6.9" echarts "^4.2.0-rc.2" highlight.js "^9.13.1" js-beautify "^1.8.8" - lodash "^4.17.11" + lodash "^4.17.14" url-search-params-polyfill "^5.0.0" vue "^2.6.10" vue-loader "^15.4.2" @@ -1921,14 +1921,13 @@ bonjour@^3.5.0: multicast-dns "^6.0.1" multicast-dns-service-types "^1.1.0" -bootstrap-vue@^2.0.0-rc.24: - version "2.0.0-rc.24" - resolved "https://registry.yarnpkg.com/bootstrap-vue/-/bootstrap-vue-2.0.0-rc.24.tgz#8ea5bbcd19e0f9b4f87ed4d9ba72abaa35231f32" - integrity sha512-8rA/I9tOvpNVIuMKD3rdlrUqgVdPEw4vPI0X8OeFJcG2hHvCHeZDF7FmWqxSeehIrUHGDV17HlTGSuP/v1Sp5g== +bootstrap-vue@^2.0.0-rc.26: + version "2.0.0-rc.26" + resolved "https://registry.yarnpkg.com/bootstrap-vue/-/bootstrap-vue-2.0.0-rc.26.tgz#2b8b9116a452584ae20ba8310d143635bc9e1375" + integrity sha512-AzN+IRTmfR9rLFWNGt+v2XPmQjZiAlH4x5z2kStA3UoJ5LR91K34iZ8apaa6isDo+DfWDcwH4q7OwHM+VNxWwg== dependencies: "@nuxt/opencollective" "^0.2.2" bootstrap "^4.3.1" - core-js ">=2.6.5 <3.0.0" popper.js "^1.15.0" portal-vue "^2.1.5" vue-functional-data-merge "^3.1.0" @@ -2827,7 +2826,7 @@ core-js@3.0.1: resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.0.1.tgz#1343182634298f7f38622f95e73f54e48ddf4738" integrity sha512-sco40rF+2KlE0ROMvydjkrVMMG1vYilP2ALoRXcYR4obqbYIuV3Bg+51GEDW+HF8n7NRA+iaA4qD0nD9lo9mew== -"core-js@>=2.6.5 <3.0.0", core-js@^2.2.0, core-js@^2.6.9: +core-js@^2.2.0, core-js@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.9.tgz#6b4b214620c834152e179323727fc19741b084f2" integrity sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A== @@ -7016,7 +7015,7 @@ lodash.upperfirst@4.3.1: resolved "https://registry.yarnpkg.com/lodash.upperfirst/-/lodash.upperfirst-4.3.1.tgz#1365edf431480481ef0d1c68957a5ed99d49f7ce" integrity sha1-E2Xt9DFIBIHvDRxolXpe2Z1J984= -lodash@^4.0.0, lodash@^4.13.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.5.0, lodash@~4.17.10: +lodash@^4.0.0, lodash@^4.13.1, lodash@^4.17.10, lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.5.0, lodash@~4.17.10: version "4.17.14" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba" integrity sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw== |
