diff options
118 files changed, 2584 insertions, 1350 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index dc8123a5888..76e2ed66cfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ documentation](doc/development/changelog.md) for instructions on adding your own entry. +## 11.9.5 (2019-04-03) + +### Fixed (3 changes) + +- Force to recreate all MR diffs on import. !26480 +- Fix API /project/:id/branches not returning correct merge status. !26785 +- Avoid excessive recursive calls with Rugged TreeEntries. !26813 + +### Performance (1 change) + +- Force a full GC after importing a project. !26803 + + ## 11.9.3 (2019-03-27) ### Security (8 changes) diff --git a/Dangerfile b/Dangerfile index 32f4b4d23c3..95dd48aae9e 100644 --- a/Dangerfile +++ b/Dangerfile @@ -13,3 +13,4 @@ danger.import_dangerfile(path: 'danger/prettier') danger.import_dangerfile(path: 'danger/eslint') danger.import_dangerfile(path: 'danger/roulette') danger.import_dangerfile(path: 'danger/single_codebase') +danger.import_dangerfile(path: 'danger/gitlab_ui_wg') diff --git a/app/assets/javascripts/ci_variable_list/ci_variable_list.js b/app/assets/javascripts/ci_variable_list/ci_variable_list.js index 5b20fa141cd..da3100b9386 100644 --- a/app/assets/javascripts/ci_variable_list/ci_variable_list.js +++ b/app/assets/javascripts/ci_variable_list/ci_variable_list.js @@ -40,6 +40,12 @@ export default class VariableList { // converted. we need the value as a string. default: $('.js-ci-variable-input-protected').attr('data-default'), }, + masked: { + selector: '.js-ci-variable-input-masked', + // use `attr` instead of `data` as we don't want the value to be + // converted. we need the value as a string. + default: $('.js-ci-variable-input-masked').attr('data-default'), + }, environment_scope: { // We can't use a `.js-` class here because // gl_dropdown replaces the <input> and doesn't copy over the class @@ -88,13 +94,16 @@ export default class VariableList { } }); - // Always make sure there is an empty last row - this.$container.on('input trigger-change', inputSelector, () => { + this.$container.on('input trigger-change', inputSelector, e => { + // Always make sure there is an empty last row const $lastRow = this.$container.find('.js-row').last(); if (this.checkIfRowTouched($lastRow)) { this.insertRow($lastRow); } + + // If masked, validate value against regex + this.validateMaskability($(e.currentTarget).closest('.js-row')); }); } @@ -171,12 +180,33 @@ export default class VariableList { checkIfRowTouched($row) { return Object.keys(this.inputMap).some(name => { + // Row should not qualify as touched if only switches have been touched + if (['protected', 'masked'].includes(name)) return false; + const entry = this.inputMap[name]; const $el = $row.find(entry.selector); return $el.length && $el.val() !== entry.default; }); } + validateMaskability($row) { + const invalidInputClass = 'gl-field-error-outline'; + + const maskableRegex = /^\w{8,}$/; // Eight or more alphanumeric characters plus underscores + const variableValue = $row.find(this.inputMap.secret_value.selector).val(); + const isValueMaskable = maskableRegex.test(variableValue) || variableValue === ''; + const isMaskedChecked = $row.find(this.inputMap.masked.selector).val() === 'true'; + + // Show a validation error if the user wants to mask an unmaskable variable value + $row + .find(this.inputMap.secret_value.selector) + .toggleClass(invalidInputClass, isMaskedChecked && !isValueMaskable); + $row + .find('.js-secret-value-placeholder') + .toggleClass(invalidInputClass, isMaskedChecked && !isValueMaskable); + $row.find('.masking-validation-error').toggle(isMaskedChecked && !isValueMaskable); + } + toggleEnableRow(isEnabled = true) { this.$container.find(this.inputMap.key.selector).attr('disabled', !isEnabled); this.$container.find('.js-row-remove-button').attr('disabled', !isEnabled); diff --git a/app/assets/javascripts/diffs/components/diff_file.vue b/app/assets/javascripts/diffs/components/diff_file.vue index 58a9605c181..f5876a73eff 100644 --- a/app/assets/javascripts/diffs/components/diff_file.vue +++ b/app/assets/javascripts/diffs/components/diff_file.vue @@ -188,10 +188,6 @@ export default { /> </div> </template> - <div v-if="isFileTooLarge" class="nothing-here-block diff-collapsed js-too-large-diff"> - {{ __('This source diff could not be displayed because it is too large.') }} - <span v-html="viewBlobLink"></span> - </div> </div> </template> diff --git a/app/assets/javascripts/ide/lib/files.js b/app/assets/javascripts/ide/lib/files.js index 5dfba8fe531..df100f753d7 100644 --- a/app/assets/javascripts/ide/lib/files.js +++ b/app/assets/javascripts/ide/lib/files.js @@ -1,6 +1,8 @@ import { viewerInformationForPath } from '~/vue_shared/components/content_viewer/lib/viewer_utils'; import { decorateData, sortTree } from '../stores/utils'; +export const escapeFileUrl = fileUrl => encodeURIComponent(fileUrl).replace(/%2F/g, '/'); + export const splitParent = path => { const idx = path.lastIndexOf('/'); @@ -45,7 +47,7 @@ export const decorateFiles = ({ id: path, name, path, - url: `/${projectId}/tree/${branchId}/-/${path}/`, + url: `/${projectId}/tree/${branchId}/-/${escapeFileUrl(path)}/`, type: 'tree', parentTreeUrl: parentFolder ? parentFolder.url : `/${projectId}/tree/${branchId}/`, tempFile, @@ -81,7 +83,7 @@ export const decorateFiles = ({ id: path, name, path, - url: `/${projectId}/blob/${branchId}/-/${path}`, + url: `/${projectId}/blob/${branchId}/-/${escapeFileUrl(path)}`, type: 'blob', parentTreeUrl: fileFolder ? fileFolder.url : `/${projectId}/blob/${branchId}`, tempFile, diff --git a/app/assets/javascripts/lib/utils/datetime_utility.js b/app/assets/javascripts/lib/utils/datetime_utility.js index d3fe8f77bd4..4d6327840db 100644 --- a/app/assets/javascripts/lib/utils/datetime_utility.js +++ b/app/assets/javascripts/lib/utils/datetime_utility.js @@ -92,7 +92,7 @@ export const getTimeago = () => { const timeAgoLocaleRemaining = [ () => [s__('Timeago|just now'), s__('Timeago|right now')], - () => [s__('Timeago|%s seconds ago'), s__('Timeago|%s seconds remaining')], + () => [s__('Timeago|just now'), s__('Timeago|%s seconds remaining')], () => [s__('Timeago|1 minute ago'), s__('Timeago|1 minute remaining')], () => [s__('Timeago|%s minutes ago'), s__('Timeago|%s minutes remaining')], () => [s__('Timeago|1 hour ago'), s__('Timeago|1 hour remaining')], @@ -121,7 +121,7 @@ export const getTimeago = () => { const timeAgoLocale = [ () => [s__('Timeago|just now'), s__('Timeago|right now')], - () => [s__('Timeago|%s seconds ago'), s__('Timeago|in %s seconds')], + () => [s__('Timeago|just now'), s__('Timeago|in %s seconds')], () => [s__('Timeago|1 minute ago'), s__('Timeago|in 1 minute')], () => [s__('Timeago|%s minutes ago'), s__('Timeago|in %s minutes')], () => [s__('Timeago|1 hour ago'), s__('Timeago|in 1 hour')], @@ -160,7 +160,11 @@ export const getTimeago = () => { * @param {Boolean} setTimeago */ export const localTimeAgo = ($timeagoEls, setTimeago = true) => { - getTimeago().render($timeagoEls, timeagoLanguageCode); + getTimeago(); + + $timeagoEls.each((i, el) => { + $(el).text(timeagoInstance.format($(el).attr('datetime'), timeagoLanguageCode)); + }); if (!setTimeago) { return; diff --git a/app/assets/javascripts/notes/components/note_actions.vue b/app/assets/javascripts/notes/components/note_actions.vue index fc73726857d..aabb77f6a85 100644 --- a/app/assets/javascripts/notes/components/note_actions.vue +++ b/app/assets/javascripts/notes/components/note_actions.vue @@ -86,9 +86,6 @@ export default { }, computed: { ...mapGetters(['getUserDataByProp']), - showReplyButton() { - return gon.features && gon.features.replyToIndividualNotes && this.showReply; - }, shouldShowActionsDropdown() { return this.currentUserId && (this.canEdit || this.canReportAsAbuse); }, @@ -167,7 +164,7 @@ export default { </a> </div> <reply-button - v-if="showReplyButton" + v-if="showReply" ref="replyButton" class="js-reply-button" @startReplying="$emit('startReplying')" diff --git a/app/assets/javascripts/notes/components/noteable_note.vue b/app/assets/javascripts/notes/components/noteable_note.vue index 5fa0ab3de98..d2cfeff53e8 100644 --- a/app/assets/javascripts/notes/components/noteable_note.vue +++ b/app/assets/javascripts/notes/components/noteable_note.vue @@ -96,7 +96,7 @@ export default { return ''; } - // We need to do this to ensure we have the currect sentence order + // We need to do this to ensure we have the correct sentence order // when translating this as the sentence order may change from one // language to the next. See: // https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/24427#note_133713771 diff --git a/app/assets/javascripts/notifications_dropdown.js b/app/assets/javascripts/notifications_dropdown.js index 6aed2492084..08545dcea46 100644 --- a/app/assets/javascripts/notifications_dropdown.js +++ b/app/assets/javascripts/notifications_dropdown.js @@ -1,5 +1,6 @@ import $ from 'jquery'; import Flash from './flash'; +import { __ } from '~/locale'; export default function notificationsDropdown() { $(document).on('click', '.update-notification', function updateNotificationCallback(e) { @@ -27,7 +28,7 @@ export default function notificationsDropdown() { .closest('.js-notification-dropdown') .replaceWith(data.html); } else { - Flash('Failed to save new settings', 'alert'); + Flash(__('Failed to save new settings'), 'alert'); } }); } diff --git a/app/assets/javascripts/pages/projects/pipeline_schedules/shared/components/timezone_dropdown.js b/app/assets/javascripts/pages/projects/pipeline_schedules/shared/components/timezone_dropdown.js index 95b57d5e048..c1f6edf2f27 100644 --- a/app/assets/javascripts/pages/projects/pipeline_schedules/shared/components/timezone_dropdown.js +++ b/app/assets/javascripts/pages/projects/pipeline_schedules/shared/components/timezone_dropdown.js @@ -1,17 +1,33 @@ -/* eslint-disable class-methods-use-this */ +const defaultTimezone = 'UTC'; + +export const formatUtcOffset = offset => { + const parsed = parseInt(offset, 10); + if (Number.isNaN(parsed) || parsed === 0) { + return `0`; + } + const prefix = offset > 0 ? '+' : '-'; + return `${prefix} ${Math.abs(offset / 3600)}`; +}; -import $ from 'jquery'; +export const formatTimezone = item => `[UTC ${formatUtcOffset(item.offset)}] ${item.name}`; -const defaultTimezone = 'UTC'; +const defaults = { + $inputEl: null, + $dropdownEl: null, + onSelectTimezone: null, +}; export default class TimezoneDropdown { - constructor() { - this.$dropdown = $('.js-timezone-dropdown'); + constructor({ $dropdownEl, $inputEl, onSelectTimezone } = defaults) { + this.$dropdown = $dropdownEl; this.$dropdownToggle = this.$dropdown.find('.dropdown-toggle-text'); - this.$input = $('#schedule_cron_timezone'); + this.$input = $inputEl; this.timezoneData = this.$dropdown.data('data'); + this.initDefaultTimezone(); this.initDropdown(); + + this.onSelectTimezone = onSelectTimezone; } initDropdown() { @@ -24,28 +40,12 @@ export default class TimezoneDropdown { fields: ['name'], }, clicked: cfg => this.updateInputValue(cfg), - text: item => this.formatTimezone(item), + text: item => formatTimezone(item), }); this.setDropdownToggle(); } - formatUtcOffset(offset) { - let prefix = ''; - - if (offset > 0) { - prefix = '+'; - } else if (offset < 0) { - prefix = '-'; - } - - return `${prefix} ${Math.abs(offset / 3600)}`; - } - - formatTimezone(item) { - return `[UTC ${this.formatUtcOffset(item.offset)}] ${item.name}`; - } - initDefaultTimezone() { const initialValue = this.$input.val(); @@ -56,13 +56,14 @@ export default class TimezoneDropdown { setDropdownToggle() { const initialValue = this.$input.val(); - this.$dropdownToggle.text(initialValue); } updateInputValue({ selectedObj, e }) { e.preventDefault(); this.$input.val(selectedObj.identifier); - gl.pipelineScheduleFieldErrors.updateFormValidityState(); + if (this.onSelectTimezone) { + this.onSelectTimezone({ selectedObj, e }); + } } } diff --git a/app/assets/javascripts/pages/projects/pipeline_schedules/shared/init_form.js b/app/assets/javascripts/pages/projects/pipeline_schedules/shared/init_form.js index 4d494efef6c..dc6df27f1c7 100644 --- a/app/assets/javascripts/pages/projects/pipeline_schedules/shared/init_form.js +++ b/app/assets/javascripts/pages/projects/pipeline_schedules/shared/init_form.js @@ -41,7 +41,13 @@ export default () => { const formElement = document.getElementById('new-pipeline-schedule-form'); - gl.timezoneDropdown = new TimezoneDropdown(); + gl.timezoneDropdown = new TimezoneDropdown({ + $dropdownEl: $('.js-timezone-dropdown'), + $inputEl: $('#schedule_cron_timezone'), + onSelectTimezone: () => { + gl.pipelineScheduleFieldErrors.updateFormValidityState(); + }, + }); gl.targetBranchDropdown = new TargetBranchDropdown(); gl.pipelineScheduleFieldErrors = new GlFieldErrors(formElement); diff --git a/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/image_diff/onion_skin_viewer.vue b/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/image_diff/onion_skin_viewer.vue index f085ef35ccc..b25aebd7c98 100644 --- a/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/image_diff/onion_skin_viewer.vue +++ b/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/image_diff/onion_skin_viewer.vue @@ -102,7 +102,7 @@ export default { :style="{ width: onionMaxPixelWidth, height: onionMaxPixelHeight, - 'user-select': dragging === true ? 'none' : '', + 'user-select': dragging ? 'none' : null, }" class="onion-skin-frame" > diff --git a/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/image_diff/swipe_viewer.vue b/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/image_diff/swipe_viewer.vue index 1c970b72a66..eddafc759a2 100644 --- a/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/image_diff/swipe_viewer.vue +++ b/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/image_diff/swipe_viewer.vue @@ -68,12 +68,10 @@ export default { }, startDrag() { this.dragging = true; - document.body.style.userSelect = 'none'; document.body.addEventListener('mousemove', this.dragMove); }, stopDrag() { this.dragging = false; - document.body.style.userSelect = ''; document.body.removeEventListener('mousemove', this.dragMove); }, prepareSwipe() { @@ -104,7 +102,13 @@ export default { <template> <div class="swipe view"> - <div ref="swipeFrame" class="swipe-frame"> + <div + ref="swipeFrame" + :style="{ + 'user-select': dragging ? 'none' : null, + }" + class="swipe-frame" + > <image-viewer key="swipeOldImg" ref="swipeOldImg" diff --git a/app/assets/stylesheets/framework/ci_variable_list.scss b/app/assets/stylesheets/framework/ci_variable_list.scss index 7207e5119ce..d9b0e4558ad 100644 --- a/app/assets/stylesheets/framework/ci_variable_list.scss +++ b/app/assets/stylesheets/framework/ci_variable_list.scss @@ -66,6 +66,7 @@ } } +.ci-variable-masked-item, .ci-variable-protected-item { flex: 0 1 auto; display: flex; diff --git a/app/assets/stylesheets/page_bundles/ide.scss b/app/assets/stylesheets/page_bundles/ide.scss index a80158943c6..0c1067bfacc 100644 --- a/app/assets/stylesheets/page_bundles/ide.scss +++ b/app/assets/stylesheets/page_bundles/ide.scss @@ -179,6 +179,14 @@ $ide-commit-header-height: 48px; display: none; } + .monaco-editor .selected-text { + z-index: 1; + } + + .monaco-editor .view-lines { + z-index: 2; + } + .is-readonly, .editor.original { .view-lines { diff --git a/app/controllers/concerns/issuable_actions.rb b/app/controllers/concerns/issuable_actions.rb index 05d88429cfe..8ef3b6502df 100644 --- a/app/controllers/concerns/issuable_actions.rb +++ b/app/controllers/concerns/issuable_actions.rb @@ -7,9 +7,6 @@ module IssuableActions included do before_action :authorize_destroy_issuable!, only: :destroy before_action :authorize_admin_issuable!, only: :bulk_update - before_action only: :show do - push_frontend_feature_flag(:reply_to_individual_notes, default_enabled: true) - end end def permitted_keys diff --git a/app/controllers/groups/variables_controller.rb b/app/controllers/groups/variables_controller.rb index 4f641de0357..b44e3b0fff4 100644 --- a/app/controllers/groups/variables_controller.rb +++ b/app/controllers/groups/variables_controller.rb @@ -41,7 +41,7 @@ module Groups end def variable_params_attributes - %i[id key secret_value protected _destroy] + %i[id key secret_value protected masked _destroy] end def authorize_admin_build! diff --git a/app/controllers/projects/variables_controller.rb b/app/controllers/projects/variables_controller.rb index bb658bfcc19..05a79d59ffd 100644 --- a/app/controllers/projects/variables_controller.rb +++ b/app/controllers/projects/variables_controller.rb @@ -38,6 +38,6 @@ class Projects::VariablesController < Projects::ApplicationController end def variable_params_attributes - %i[id key secret_value protected _destroy] + %i[id key secret_value protected masked _destroy] end end diff --git a/app/helpers/blob_helper.rb b/app/helpers/blob_helper.rb index 3e1bb9af5cc..d6fff1c36da 100644 --- a/app/helpers/blob_helper.rb +++ b/app/helpers/blob_helper.rb @@ -19,10 +19,14 @@ module BlobHelper def ide_edit_path(project = @project, ref = @ref, path = @path, options = {}) segments = [ide_path, 'project', project.full_path, 'edit', ref] - segments.concat(['-', path]) if path.present? + segments.concat(['-', encode_ide_path(path)]) if path.present? File.join(segments) end + def encode_ide_path(path) + url_encode(path).gsub('%2F', '/') + end + def edit_blob_button(project = @project, ref = @ref, path = @path, options = {}) return unless blob = readable_blob(options, path, project, ref) diff --git a/app/helpers/ci_variables_helper.rb b/app/helpers/ci_variables_helper.rb index e3728804c2a..88ce311a1d4 100644 --- a/app/helpers/ci_variables_helper.rb +++ b/app/helpers/ci_variables_helper.rb @@ -12,4 +12,12 @@ module CiVariablesHelper ci_variable_protected_by_default? end end + + def ci_variable_masked?(variable, only_key_value) + if variable && !only_key_value + variable.masked + else + true + end + end end diff --git a/app/models/individual_note_discussion.rb b/app/models/individual_note_discussion.rb index 3b6b68a9c5f..d926e39f96e 100644 --- a/app/models/individual_note_discussion.rb +++ b/app/models/individual_note_discussion.rb @@ -14,7 +14,7 @@ class IndividualNoteDiscussion < Discussion end def can_convert_to_discussion? - noteable.supports_replying_to_individual_notes? && Feature.enabled?(:reply_to_individual_notes, default_enabled: true) + noteable.supports_replying_to_individual_notes? end def convert_to_discussion!(save: false) diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index b780b67492f..458c57c1dc6 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -858,15 +858,6 @@ class MergeRequest < ApplicationRecord end def related_notes - # Fetch comments only from last 100 commits - commits_for_notes_limit = 100 - commit_ids = commit_shas.take(commits_for_notes_limit) - - commit_notes = Note - .except(:order) - .where(project_id: [source_project_id, target_project_id]) - .for_commit_id(commit_ids) - # We're using a UNION ALL here since this results in better performance # compared to using OR statements. We're using UNION ALL since the queries # used won't produce any duplicates (e.g. a note for a commit can't also be @@ -878,6 +869,16 @@ class MergeRequest < ApplicationRecord alias_method :discussion_notes, :related_notes + def commit_notes + # Fetch comments only from last 100 commits + commit_ids = commit_shas.take(100) + + Note + .user + .where(project_id: [source_project_id, target_project_id]) + .for_commit_id(commit_ids) + end + def mergeable_discussions_state? return true unless project.only_allow_merge_if_all_discussions_are_resolved? diff --git a/app/serializers/group_variable_entity.rb b/app/serializers/group_variable_entity.rb index 0edab4a3092..19c5fa26f34 100644 --- a/app/serializers/group_variable_entity.rb +++ b/app/serializers/group_variable_entity.rb @@ -6,4 +6,5 @@ class GroupVariableEntity < Grape::Entity expose :value expose :protected?, as: :protected + expose :masked?, as: :masked end diff --git a/app/serializers/variable_entity.rb b/app/serializers/variable_entity.rb index 85cf367fe51..4d48e13cfca 100644 --- a/app/serializers/variable_entity.rb +++ b/app/serializers/variable_entity.rb @@ -6,4 +6,5 @@ class VariableEntity < Grape::Entity expose :value expose :protected?, as: :protected + expose :masked?, as: :masked end diff --git a/app/services/git/tag_push_service.rb b/app/services/git/tag_push_service.rb index 318dfd4f886..9ce0fbdb206 100644 --- a/app/services/git/tag_push_service.rb +++ b/app/services/git/tag_push_service.rb @@ -13,7 +13,6 @@ module Git EventCreateService.new.push(project, current_user, push_data) Ci::CreatePipelineService.new(project, current_user, push_data).execute(:push, pipeline_options) - SystemHooksService.new.execute_hooks(build_system_push_data, :tag_push_hooks) project.execute_hooks(push_data.dup, :tag_push_hooks) project.execute_services(push_data.dup, :tag_push_hooks) @@ -50,17 +49,6 @@ module Git push_options: params[:push_options] || []) end - def build_system_push_data - Gitlab::DataBuilder::Push.build( - project, - current_user, - params[:oldrev], - params[:newrev], - params[:ref], - [], - '') - end - def pipeline_options {} # to be overridden in EE end diff --git a/app/services/quick_actions/interpret_service.rb b/app/services/quick_actions/interpret_service.rb index f463e08ee7e..8ff73522e5f 100644 --- a/app/services/quick_actions/interpret_service.rb +++ b/app/services/quick_actions/interpret_service.rb @@ -96,14 +96,27 @@ module QuickActions end def find_labels(labels_params = nil) + extract_references(labels_params, :label) | find_labels_by_name_no_tilde(labels_params) + end + + def find_labels_by_name_no_tilde(labels_params) + return Label.none if label_with_tilde?(labels_params) + finder_params = { include_ancestor_groups: true } finder_params[:project_id] = project.id if project finder_params[:group_id] = group.id if group - finder_params[:name] = labels_params.split if labels_params + finder_params[:name] = extract_label_names(labels_params) if labels_params - result = LabelsFinder.new(current_user, finder_params).execute + LabelsFinder.new(current_user, finder_params).execute + end + + def label_with_tilde?(labels_params) + labels_params&.include?('~') + end - extract_references(labels_params, :label) | result + def extract_label_names(labels_params) + # '"A" "A B C" A B' => ["A", "A B C", "A", "B"] + labels_params.scan(/"([^"]+)"|([^ ]+)/).flatten.compact end def find_label_references(labels_param) diff --git a/app/views/ci/variables/_content.html.haml b/app/views/ci/variables/_content.html.haml index 90c59bec975..d07cbe4589c 100644 --- a/app/views/ci/variables/_content.html.haml +++ b/app/views/ci/variables/_content.html.haml @@ -1,3 +1,3 @@ -= _('Environment variables are applied to environments via the runner. They can be protected by only exposing them to protected branches or tags. You can use environment variables for passwords, secret keys, or whatever you want.') += _('Environment variables are applied to environments via the runner. They can be protected by only exposing them to protected branches or tags. Additionally, they will be masked by default so they are hidden in job logs, though they must match certain regexp requirements to do so. You can use environment variables for passwords, secret keys, or whatever you want.') = _('You may also add variables that are made available to the running application by prepending the variable key with <code>K8S_SECRET_</code>.').html_safe = link_to _('More information'), help_page_path('ci/variables/README', anchor: 'variables') diff --git a/app/views/ci/variables/_header.html.haml b/app/views/ci/variables/_header.html.haml index cb7779e2175..dbfa0a9e5a1 100644 --- a/app/views/ci/variables/_header.html.haml +++ b/app/views/ci/variables/_header.html.haml @@ -1,7 +1,7 @@ - expanded = local_assigns.fetch(:expanded) %h4 - = _('Environment variables') + = _('Variables') = link_to icon('question-circle'), help_page_path('ci/variables/README', anchor: 'variables'), target: '_blank', rel: 'noopener noreferrer' %button.btn.btn-default.js-settings-toggle{ type: 'button' } diff --git a/app/views/ci/variables/_variable_row.html.haml b/app/views/ci/variables/_variable_row.html.haml index 16a7527c8ce..aecfdea10d9 100644 --- a/app/views/ci/variables/_variable_row.html.haml +++ b/app/views/ci/variables/_variable_row.html.haml @@ -7,12 +7,15 @@ - value = variable&.value - is_protected_default = ci_variable_protected_by_default? - is_protected = ci_variable_protected?(variable, only_key_value) +- is_masked_default = true +- is_masked = ci_variable_masked?(variable, only_key_value) - id_input_name = "#{form_field}[variables_attributes][][id]" - destroy_input_name = "#{form_field}[variables_attributes][][_destroy]" - key_input_name = "#{form_field}[variables_attributes][][key]" - value_input_name = "#{form_field}[variables_attributes][][secret_value]" - protected_input_name = "#{form_field}[variables_attributes][][protected]" +- masked_input_name = "#{form_field}[variables_attributes][][masked]" %li.js-row.ci-variable-row{ data: { is_persisted: "#{!id.nil?}" } } .ci-variable-row-body @@ -22,7 +25,7 @@ name: key_input_name, value: key, placeholder: s_('CiVariables|Input variable key') } - .ci-variable-body-item + .ci-variable-body-item.gl-show-field-errors .form-control.js-secret-value-placeholder.qa-ci-variable-input-value{ class: ('hide' unless id) } = '*' * 20 %textarea.js-ci-variable-input-value.js-secret-value.qa-ci-variable-input-value.form-control{ class: ('hide' if id), @@ -30,6 +33,7 @@ name: value_input_name, placeholder: s_('CiVariables|Input variable value') } = value + %p.masking-validation-error.gl-field-error.hide= s_("CiVariables|This variable will not be masked") - unless only_key_value .ci-variable-body-item.ci-variable-protected-item .append-right-default @@ -45,6 +49,20 @@ %span.toggle-icon = sprite_icon('status_success_borderless', size: 16, css_class: 'toggle-icon-svg toggle-status-checked') = sprite_icon('status_failed_borderless', size: 16, css_class: 'toggle-icon-svg toggle-status-unchecked') + .ci-variable-body-item.ci-variable-masked-item + .append-right-default + = s_("CiVariable|Masked") + %button{ type: 'button', + class: "js-project-feature-toggle project-feature-toggle #{'is-checked' if is_masked}", + "aria-label": s_("CiVariable|Toggle masked") } + %input{ type: "hidden", + class: 'js-ci-variable-input-masked js-project-feature-toggle-input', + name: masked_input_name, + value: is_masked, + data: { default: is_masked_default.to_s } } + %span.toggle-icon + = sprite_icon('status_success_borderless', size: 16, css_class: 'toggle-icon-svg toggle-status-checked') + = sprite_icon('status_failed_borderless', size: 16, css_class: 'toggle-icon-svg toggle-status-unchecked') = render_if_exists 'ci/variables/environment_scope', form_field: form_field, variable: variable %button.js-row-remove-button.ci-variable-row-remove-button{ type: 'button', 'aria-label': s_('CiVariables|Remove variable row') } = icon('minus-circle') diff --git a/changelogs/unreleased/13784-validate-variables-for-masking.yml b/changelogs/unreleased/13784-validate-variables-for-masking.yml new file mode 100644 index 00000000000..e8e97fac3d2 --- /dev/null +++ b/changelogs/unreleased/13784-validate-variables-for-masking.yml @@ -0,0 +1,5 @@ +--- +title: Add control for masking variable values in runner logs +merge_request: 26751 +author: +type: added diff --git a/changelogs/unreleased/47234-composable-auto-devops.yml b/changelogs/unreleased/47234-composable-auto-devops.yml new file mode 100644 index 00000000000..9403c5ae6d3 --- /dev/null +++ b/changelogs/unreleased/47234-composable-auto-devops.yml @@ -0,0 +1,5 @@ +--- +title: Split Auto-DevOps.gitlab-ci.yml into reusable templates +merge_request: 26520 +author: +type: changed diff --git a/changelogs/unreleased/47771-highlighting-in-diff.yml b/changelogs/unreleased/47771-highlighting-in-diff.yml new file mode 100644 index 00000000000..a8e8cbf0174 --- /dev/null +++ b/changelogs/unreleased/47771-highlighting-in-diff.yml @@ -0,0 +1,5 @@ +--- +title: Enabled text selection highlighting in diffs in Web IDE +merge_request: 26721 +author: Isaac Smith +type: fixed diff --git a/changelogs/unreleased/52560-fix-duplicate-tag-system-hooks.yml b/changelogs/unreleased/52560-fix-duplicate-tag-system-hooks.yml new file mode 100644 index 00000000000..b8d58d6bd30 --- /dev/null +++ b/changelogs/unreleased/52560-fix-duplicate-tag-system-hooks.yml @@ -0,0 +1,5 @@ +--- +title: Only execute system hooks once when pushing tags +merge_request: 26888 +author: +type: fixed diff --git a/changelogs/unreleased/53459-quick-action-adds-multiple-labels-to-issue-if-middle-words-overlap-with-existing-label.yml b/changelogs/unreleased/53459-quick-action-adds-multiple-labels-to-issue-if-middle-words-overlap-with-existing-label.yml new file mode 100644 index 00000000000..30d8c0e95d7 --- /dev/null +++ b/changelogs/unreleased/53459-quick-action-adds-multiple-labels-to-issue-if-middle-words-overlap-with-existing-label.yml @@ -0,0 +1,5 @@ +--- +title: Fix quick actions add label name middle word overlaps +merge_request: 26602 +author: Jacopo Beschi @jacopo-beschi +type: fixed diff --git a/changelogs/unreleased/55268-exclude-system-notes-from-commits-in-mr.yml b/changelogs/unreleased/55268-exclude-system-notes-from-commits-in-mr.yml new file mode 100644 index 00000000000..7af4739136b --- /dev/null +++ b/changelogs/unreleased/55268-exclude-system-notes-from-commits-in-mr.yml @@ -0,0 +1,5 @@ +--- +title: Exclude system notes from commits in merge request discussions +merge_request: 26396 +author: +type: fixed diff --git a/changelogs/unreleased/ekigbo-extend-timezone-dropdown.yml b/changelogs/unreleased/ekigbo-extend-timezone-dropdown.yml new file mode 100644 index 00000000000..42bc320a542 --- /dev/null +++ b/changelogs/unreleased/ekigbo-extend-timezone-dropdown.yml @@ -0,0 +1,5 @@ +--- +title: Extend timezone dropdown +merge_request: 26311 +author: +type: changed diff --git a/changelogs/unreleased/feature-webide_escaping.yml b/changelogs/unreleased/feature-webide_escaping.yml new file mode 100644 index 00000000000..88fa1bd948e --- /dev/null +++ b/changelogs/unreleased/feature-webide_escaping.yml @@ -0,0 +1,5 @@ +--- +title: Fixed bug with hashes in urls in WebIDE +merge_request: 54376 +author: Kieran Andrews +type: fixed diff --git a/changelogs/unreleased/fix-issues-time-counter.yml b/changelogs/unreleased/fix-issues-time-counter.yml new file mode 100644 index 00000000000..76f17063db5 --- /dev/null +++ b/changelogs/unreleased/fix-issues-time-counter.yml @@ -0,0 +1,5 @@ +--- +title: Make time counters show 'just now' for everything under one minute +merge_request: 25992 +author: Sergiu Marton +type: changed diff --git a/changelogs/unreleased/fixed-duplicated-large-text-on-diffs.yml b/changelogs/unreleased/fixed-duplicated-large-text-on-diffs.yml new file mode 100644 index 00000000000..770186a64b0 --- /dev/null +++ b/changelogs/unreleased/fixed-duplicated-large-text-on-diffs.yml @@ -0,0 +1,5 @@ +--- +title: Fixed duplicated diff too large error message +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/localize-notification-dropdown.yml b/changelogs/unreleased/localize-notification-dropdown.yml new file mode 100644 index 00000000000..9599aaf344b --- /dev/null +++ b/changelogs/unreleased/localize-notification-dropdown.yml @@ -0,0 +1,5 @@ +--- +title: Localize notifications dropdown +merge_request: 26844 +author: +type: changed diff --git a/changelogs/unreleased/recreate-all-diffs-on-import.yml b/changelogs/unreleased/recreate-all-diffs-on-import.yml deleted file mode 100644 index fd9124372f3..00000000000 --- a/changelogs/unreleased/recreate-all-diffs-on-import.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Force to recreate all MR diffs on import -merge_request: 26480 -author: -type: fixed diff --git a/changelogs/unreleased/sh-fix-project-branches-merge-status.yml b/changelogs/unreleased/sh-fix-project-branches-merge-status.yml deleted file mode 100644 index 65f41b3faf9..00000000000 --- a/changelogs/unreleased/sh-fix-project-branches-merge-status.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix API /project/:id/branches not returning correct merge status -merge_request: 26785 -author: -type: fixed diff --git a/changelogs/unreleased/sh-fix-rugged-tree-entries.yml b/changelogs/unreleased/sh-fix-rugged-tree-entries.yml deleted file mode 100644 index 97b27678905..00000000000 --- a/changelogs/unreleased/sh-fix-rugged-tree-entries.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Avoid excessive recursive calls with Rugged TreeEntries -merge_request: 26813 -author: -type: fixed diff --git a/changelogs/unreleased/sh-force-gc-after-import.yml b/changelogs/unreleased/sh-force-gc-after-import.yml deleted file mode 100644 index 755d66c1607..00000000000 --- a/changelogs/unreleased/sh-force-gc-after-import.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Force a full GC after importing a project -merge_request: 26803 -author: -type: performance diff --git a/danger/gitlab_ui_wg/Dangerfile b/danger/gitlab_ui_wg/Dangerfile new file mode 100644 index 00000000000..02d94fa5ab7 --- /dev/null +++ b/danger/gitlab_ui_wg/Dangerfile @@ -0,0 +1,55 @@ +def mention_single_codebase_approvers + frontend_maintainers = %w(@filipa @iamphill @psimyn @sarahghp @mishunov) + ux_maintainers = %w(@tauriedavis @rverissimo) + + rows = [] + users = [] + + if gitlab.mr_labels.include?('frontend') + frontend_maintainer = frontend_maintainers.sample + + rows << "| ~frontend | `#{frontend_maintainer}`" + users << frontend_maintainer + end + + if gitlab.mr_labels.include?('UX') + ux_maintainers = ux_maintainers.sample + + rows << "| ~UX | `#{ux_maintainers}`" + users << ux_maintainers + end + + if rows.empty? + backup_maintainer = frontend_maintainers.sample + + rows << "| ~frontend / ~UX | `#{backup_maintainer}`" + users << backup_maintainer + end + + markdown(<<~MARKDOWN.strip) + ## GitLab UI Working Group changes + + This merge request contains changes related to the work of [cleaning up CSS and creating + reusable components](https://gitlab.com/groups/gitlab-org/-/epics/950). + These changes will need to be reviewed and approved by the following engineers: + + | Category | Reviewer + |----------|--------- + #{rows.join("\n")} + + To make sure this happens, please follow these steps: + + 1. Add all of the mentioned users to the list of merge request approvals. + 2. Assign the merge request to the first person in the above list. + + If you are a reviewer, please follow these steps: + + 1. Review the merge request. If it is good to go, approve it. + 2. Once approved, assign to the next person in the above list. If you are + the last person in the list, merge the merge request. + MARKDOWN +end + +if gitlab.mr_labels.include?('CSS cleanup') + mention_single_codebase_approvers +end diff --git a/danger/roulette/Dangerfile b/danger/roulette/Dangerfile index 808bc96a0a0..3a4625c4eb6 100644 --- a/danger/roulette/Dangerfile +++ b/danger/roulette/Dangerfile @@ -60,7 +60,9 @@ categories = changes.keys - [:unknown] # Single codebase MRs are reviewed using a slightly different process, so we # disable the review roulette for such MRs. -if changes.any? && !gitlab.mr_labels.include?('single codebase') +# CSS Clean up MRs are reviewed using a slightly different process, so we +# disable the review roulette for such MRs. +if changes.any? && !gitlab.mr_labels.include?('single codebase') && !gitlab.mr_labels.include?('CSS cleanup') team = begin helper.project_team diff --git a/doc/api/runners.md b/doc/api/runners.md index 7d7215e6b80..0b7ef46888c 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -256,6 +256,8 @@ curl --request DELETE --header "PRIVATE-TOKEN: <your_access_token>" "https://git ## List runner's jobs +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15432) in GitLab 10.3. + List jobs that are being processed or were processed by specified Runner. ``` diff --git a/doc/ci/review_apps/index.md b/doc/ci/review_apps/index.md index 9dbcf9d1155..53651a807c2 100644 --- a/doc/ci/review_apps/index.md +++ b/doc/ci/review_apps/index.md @@ -81,7 +81,7 @@ The process of adding Review Apps in your workflow is as follows: 1. Set up the infrastructure to host and deploy the Review Apps. 1. [Install](https://docs.gitlab.com/runner/install/) and [configure](https://docs.gitlab.com/runner/commands/) a Runner to do deployment. -1. Set up a job in `.gitlab-ci.yml` that uses the predefined [predefined CI environment variable](../variables/README.md) `${CI_COMMIT_REF_NAME}` to create dynamic environments and restrict it to run only on branches. +1. Set up a job in `.gitlab-ci.yml` that uses the [predefined CI environment variable](../variables/README.md) `${CI_COMMIT_REF_NAME}` to create dynamic environments and restrict it to run only on branches. 1. Optionally, set a job that [manually stops](../environments.md#stopping-an-environment) the Review Apps. After adding Review Apps to your workflow, you follow the branched Git flow. That is: diff --git a/doc/ci/variables/README.md b/doc/ci/variables/README.md index 6532d177d2c..592fdfd2873 100644 --- a/doc/ci/variables/README.md +++ b/doc/ci/variables/README.md @@ -546,6 +546,8 @@ $'\''git'\'' "checkout" "-f" "-q" "dd648b2e48ce6518303b0bb580b2ee32fadaf045" Running on runner-8a2f473d-project-1796893-concurrent-0 via runner-8a2f473d-machine-1480971377-317a7d0f-digital-ocean-4gb... ++ export CI=true ++ CI=true +++ export CI_API_V4_API_URL=https://example.com:3000/api/v4 +++ CI_API_V4_API_URL=https://example.com:3000/api/v4 ++ export CI_DEBUG_TRACE=false ++ CI_DEBUG_TRACE=false ++ export CI_COMMIT_SHA=dd648b2e48ce6518303b0bb580b2ee32fadaf045 @@ -584,6 +586,8 @@ Running on runner-8a2f473d-project-1796893-concurrent-0 via runner-8a2f473d-mach ++ GITLAB_CI=true ++ export CI=true ++ CI=true +++ export CI_API_V4_API_URL=https://example.com:3000/api/v4 +++ CI_API_V4_API_URL=https://example.com:3000/api/v4 ++ export GITLAB_CI=true ++ GITLAB_CI=true ++ export CI_JOB_ID=7046507 diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 5690729c370..83a226d3577 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -391,6 +391,11 @@ job: The above example will run `job` for all branches on `gitlab-org/gitlab-ce`, except `master` and those with names prefixed with `release/`. +NOTE: **Note:** +Because `@` is used to denote the beginning of a ref's repository path, +matching a ref name containing the `@` character in a regular expression +requires the use of the hex character code match `\x40`. + If a job does not have an `only` rule, `only: ['branches', 'tags']` is set by default. If it doesn't have an `except` rule, it is empty. diff --git a/doc/development/testing_guide/frontend_testing.md b/doc/development/testing_guide/frontend_testing.md index 71c9637e72c..f58a8dcbcdc 100644 --- a/doc/development/testing_guide/frontend_testing.md +++ b/doc/development/testing_guide/frontend_testing.md @@ -26,6 +26,10 @@ It is not yet a requirement to use Jest. You can view the [epic](https://gitlab.com/groups/gitlab-org/-/epics/873) of issues we need to solve before being able to use Jest for all our needs. +### Debugging Jest tests + +Running `yarn jest-debug` will run Jest in debug mode, allowing you to debug/inspect as described in the [Jest docs](https://jestjs.io/docs/en/troubleshooting#tests-are-failing-and-you-don-t-know-why). + ### Timeout error The default timeout for Jest is set in diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index fd99b06b863..7693109b3c4 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -699,6 +699,21 @@ renaming `.staging` to `staging`. Then make sure to uncomment the `when` key of the `production` job to turn it into a manual action instead of deploying automatically. +### Using components of Auto-DevOps + +If you only require a subset of the features offered by Auto-DevOps, you can include +individual Auto-DevOps jobs into your own `.gitlab-ci.yml`. + +For example, to make use of [Auto Build](#auto-build), you can add the following to +your `.gitlab-ci.yml`: + +```yaml +include: + - template: Jobs/Build.gitlab-ci.yml +``` + +Consult the [Auto DevOps template] for information on available jobs. + ### PostgreSQL database support In order to support applications that require a database, diff --git a/doc/user/profile/account/two_factor_authentication.md b/doc/user/profile/account/two_factor_authentication.md index 8b3ae19b544..df413a11af0 100644 --- a/doc/user/profile/account/two_factor_authentication.md +++ b/doc/user/profile/account/two_factor_authentication.md @@ -161,8 +161,8 @@ a new set of recovery codes with SSH. 1. Run `ssh git@gitlab.example.com 2fa_recovery_codes`. 1. You are prompted to confirm that you want to generate new codes. Continuing this process invalidates previously saved codes. - ``` - bash + + ```sh $ ssh git@gitlab.example.com 2fa_recovery_codes Are you sure you want to generate new two-factor recovery codes? Any existing recovery codes you saved will be invalidated. (yes/no) @@ -207,17 +207,17 @@ Sign in and re-enable two-factor authentication as soon as possible. - You need to take special care to that 2FA keeps working after [restoring a GitLab backup](../../../raketasks/backup_restore.md). - To ensure 2FA authorizes correctly with TOTP server, you may want to ensure - your GitLab server's time is synchronized via a service like NTP. Otherwise, + your GitLab server's time is synchronized via a service like NTP. Otherwise, you may have cases where authorization always fails because of time differences. - The GitLab U2F implementation does _not_ work when the GitLab instance is accessed from multiple hostnames, or FQDNs. Each U2F registration is linked to the _current hostname_ at the time of registration, and cannot be used for other hostnames/FQDNs. - For example, if a user is trying to access a GitLab instance from `first.host.xyz` and `second.host.xyz`: + For example, if a user is trying to access a GitLab instance from `first.host.xyz` and `second.host.xyz`: - - The user logs in via `first.host.xyz` and registers their U2F key. - - The user logs out and attempts to log in via `first.host.xyz` - U2F authentication succeeds. - - The user logs out and attempts to log in via `second.host.xyz` - U2F authentication fails, because + - The user logs in via `first.host.xyz` and registers their U2F key. + - The user logs out and attempts to log in via `first.host.xyz` - U2F authentication succeeds. + - The user logs out and attempts to log in via `second.host.xyz` - U2F authentication fails, because the U2F key has only been registered on `first.host.xyz`. [Google Authenticator]: https://support.google.com/accounts/answer/1066447?hl=en diff --git a/doc/user/project/clusters/index.md b/doc/user/project/clusters/index.md index 3a9a3b4a423..c94a3f4d3b5 100644 --- a/doc/user/project/clusters/index.md +++ b/doc/user/project/clusters/index.md @@ -314,12 +314,6 @@ install it manually. ## Installing applications -NOTE: **Note:** -Before starting the installation of applications, make sure that time is synchronized -between your GitLab server and your Kubernetes cluster. Otherwise, installation could fail -and you may get errors like `Error: remote error: tls: bad certificate` -in the `stdout` of pods created by GitLab in your Kubernetes cluster. - GitLab provides a one-click install for various applications which can be added directly to your configured cluster. Those applications are needed for [Review Apps](../../../ci/review_apps/index.md) and @@ -378,6 +372,29 @@ Upgrades will reset values back to the values built into the `runner` chart plus the values set by [`values.yaml`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/vendor/runner/values.yaml) +### Troubleshooting applications + +Applications can fail with the following error: + +```text +Error: remote error: tls: bad certificate +``` + +To avoid installation errors: + +- Before starting the installation of applications, make sure that time is synchronized + between your GitLab server and your Kubernetes cluster. +- Ensure certificates are not out of sync. When installing applications, GitLab expects a new cluster with no previous installation of Tiller. + + You can confirm that the certificates match via `kubectl`: + + ```sh + kubectl get configmaps/values-content-configuration-ingress -n gitlab-managed-apps -o \ + "jsonpath={.data['cert\.pem']}" | base64 -d > a.pem + kubectl get secrets/tiller-secret -n gitlab-managed-apps -o "jsonpath={.data['ca\.crt']}" | base64 -d > b.pem + diff a.pem b.pem + ``` + ## Getting the external endpoint NOTE: **Note:** diff --git a/doc/user/project/clusters/serverless/index.md b/doc/user/project/clusters/serverless/index.md index e6804666e22..96bf455116c 100644 --- a/doc/user/project/clusters/serverless/index.md +++ b/doc/user/project/clusters/serverless/index.md @@ -214,7 +214,7 @@ The sample function can now be triggered from any HTTP client using a simple `PO --header "Content-Type: application/json" \ --request POST \ --data '{"GitLab":"FaaS"}' \ - <http://functions-echo.functions-1.functions.example.com/> + http://functions-echo.functions-1.functions.example.com ``` 2. Using a web-based tool (ie. postman, restlet, etc) diff --git a/doc/user/project/quick_actions.md b/doc/user/project/quick_actions.md index 392e72dcc5c..88f4de891a1 100644 --- a/doc/user/project/quick_actions.md +++ b/doc/user/project/quick_actions.md @@ -31,7 +31,7 @@ discussions, and descriptions: | `/reassign @user1 @user2` | Change assignee | ✓ | ✓ | | `/milestone %milestone` | Set milestone | ✓ | ✓ | | `/remove_milestone` | Remove milestone | ✓ | ✓ | -| `/label ~label1 ~label2` | Add label(s) | ✓ | ✓ | +| `/label ~label1 ~label2` | Add label(s). Label names can also start without ~ but mixed syntax is not supported. | ✓ | ✓ | | `/unlabel ~label1 ~label2` | Remove all or specific label(s)| ✓ | ✓ | | `/relabel ~label1 ~label2` | Replace label | ✓ | ✓ | | <code>/copy_metadata #issue | !merge_request</code> | Copy labels and milestone from other issue or merge request | ✓ | ✓ | diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 4533305bfd3..cc62b5a3661 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1389,8 +1389,13 @@ module API expose :name, :script, :timeout, :when, :allow_failure end + class Port < Grape::Entity + expose :number, :protocol, :name + end + class Image < Grape::Entity expose :name, :entrypoint + expose :ports, using: JobRequest::Port end class Service < Image diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index 4dd932f61d4..1d7bfba75cd 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -4,7 +4,7 @@ module Gitlab module Ci module Build class Image - attr_reader :alias, :command, :entrypoint, :name + attr_reader :alias, :command, :entrypoint, :name, :ports class << self def from_image(job) @@ -26,17 +26,25 @@ module Gitlab def initialize(image) if image.is_a?(String) @name = image + @ports = [] elsif image.is_a?(Hash) @alias = image[:alias] @command = image[:command] @entrypoint = image[:entrypoint] @name = image[:name] + @ports = build_ports(image).select(&:valid?) end end def valid? @name.present? end + + private + + def build_ports(image) + image[:ports].to_a.map { |port| ::Gitlab::Ci::Build::Port.new(port) } + end end end end diff --git a/lib/gitlab/ci/build/port.rb b/lib/gitlab/ci/build/port.rb new file mode 100644 index 00000000000..6c4656ffea2 --- /dev/null +++ b/lib/gitlab/ci/build/port.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Port + DEFAULT_PORT_NAME = 'default_port'.freeze + DEFAULT_PORT_PROTOCOL = 'http'.freeze + + attr_reader :number, :protocol, :name + + def initialize(port) + @name = DEFAULT_PORT_NAME + @protocol = DEFAULT_PORT_PROTOCOL + + case port + when Integer + @number = port + when Hash + @number = port[:number] + @protocol = port.fetch(:protocol, @protocol) + @name = port.fetch(:name, @name) + end + end + + def valid? + @number.present? + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index a13a0625e90..0beeb44c272 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -9,24 +9,24 @@ module Gitlab # class Image < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name entrypoint].freeze + ALLOWED_KEYS = %i[name entrypoint ports].freeze validations do validates :config, hash_or_string: true validates :config, allowed_keys: ALLOWED_KEYS + validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? validates :name, type: String, presence: true validates :entrypoint, array_of_strings: true, allow_nil: true end - def hash? - @config.is_a?(Hash) - end + entry :ports, Entry::Ports, + description: 'Ports used expose the image' - def string? - @config.is_a?(String) - end + attributes :ports def name value[:name] @@ -42,6 +42,14 @@ module Gitlab {} end + + def with_image_ports? + opt(:with_image_ports) + end + + def skip_config_hash_validation? + true + end end end end diff --git a/lib/gitlab/ci/config/entry/port.rb b/lib/gitlab/ci/config/entry/port.rb new file mode 100644 index 00000000000..c239b1225c5 --- /dev/null +++ b/lib/gitlab/ci/config/entry/port.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of an Image Port. + # + class Port < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_KEYS = %i[number protocol name].freeze + + validations do + validates :config, hash_or_integer: true + validates :config, allowed_keys: ALLOWED_KEYS + + validates :number, type: Integer, presence: true + validates :protocol, type: String, inclusion: { in: %w[http https], message: 'should be http or https' }, allow_blank: true + validates :name, type: String, presence: false, allow_nil: true + end + + def number + value[:number] + end + + def protocol + value[:protocol] + end + + def name + value[:name] + end + + def value + return { number: @config } if integer? + return @config if hash? + + {} + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/ports.rb b/lib/gitlab/ci/config/entry/ports.rb new file mode 100644 index 00000000000..01ffcc7dd87 --- /dev/null +++ b/lib/gitlab/ci/config/entry/ports.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a configuration of the ports of a Docker service. + # + class Ports < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: Array + validates :config, port_name_present_and_unique: true + validates :config, port_unique: true + end + + def compose!(deps = nil) + super do + @entries = [] + @config.each do |config| + @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Port) + .value(config || {}) + .with(key: "port", parent: self, description: "port definition.") # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each do |entry| + entry.compose!(deps) + end + end + end + + def value + @entries.map(&:value) + end + + def descendants + @entries + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 6df67083310..084fa4047a4 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -10,16 +10,18 @@ module Gitlab class Service < Image include ::Gitlab::Config::Entry::Validatable - ALLOWED_KEYS = %i[name entrypoint command alias].freeze + ALLOWED_KEYS = %i[name entrypoint command alias ports].freeze validations do validates :config, hash_or_string: true validates :config, allowed_keys: ALLOWED_KEYS + validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? validates :name, type: String, presence: true validates :entrypoint, array_of_strings: true, allow_nil: true validates :command, array_of_strings: true, allow_nil: true validates :alias, type: String, allow_nil: true + validates :alias, type: String, presence: true, unless: ->(record) { record.ports.blank? } end def alias diff --git a/lib/gitlab/ci/config/entry/services.rb b/lib/gitlab/ci/config/entry/services.rb index 71475f69218..83baa83711f 100644 --- a/lib/gitlab/ci/config/entry/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -12,6 +12,7 @@ module Gitlab validations do validates :config, type: Array + validates :config, services_with_ports_alias_unique: true, if: ->(record) { record.opt(:with_image_ports) } end def compose!(deps = nil) @@ -20,6 +21,7 @@ module Gitlab @config.each do |config| @entries << ::Gitlab::Config::Entry::Factory.new(Entry::Service) .value(config || {}) + .with(key: "service", parent: self, description: "service definition.") # rubocop:disable CodeReuse/ActiveRecord .create! end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index d4c70d4b9da..f786022beb0 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -72,879 +72,14 @@ stages: - performance - cleanup -build: - stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable" - services: - - docker:stable-dind - script: - - /build/build.sh - only: - - branches - - tags - -test: - services: - - postgres:latest - variables: - POSTGRES_DB: test - stage: test - image: gliderlabs/herokuish:latest - script: - - setup_test_db - - cp -R . /tmp/app - - /bin/herokuish buildpack test - only: - - branches - - tags - except: - variables: - - $TEST_DISABLED - -code_quality: - stage: test - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - code_quality - artifacts: - paths: [gl-code-quality-report.json] - only: - - branches - - tags - except: - variables: - - $CODE_QUALITY_DISABLED - -license_management: - stage: test - image: - name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" - entrypoint: [""] - allow_failure: true - script: - - license_management - artifacts: - paths: [gl-license-management-report.json] - only: - refs: - - branches - - tags - variables: - - $GITLAB_FEATURES =~ /\blicense_management\b/ - except: - variables: - - $LICENSE_MANAGEMENT_DISABLED - -performance: - stage: performance - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - performance - artifacts: - paths: - - performance.json - - sitespeed-results/ - only: - refs: - - branches - - tags - kubernetes: active - except: - variables: - - $PERFORMANCE_DISABLED - -sast: - stage: test - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - sast - artifacts: - reports: - sast: gl-sast-report.json - only: - refs: - - branches - - tags - variables: - - $GITLAB_FEATURES =~ /\bsast\b/ - except: - variables: - - $SAST_DISABLED - -dependency_scanning: - stage: test - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - dependency_scanning - artifacts: - reports: - dependency_scanning: gl-dependency-scanning-report.json - only: - refs: - - branches - - tags - variables: - - $GITLAB_FEATURES =~ /\bdependency_scanning\b/ - except: - variables: - - $DEPENDENCY_SCANNING_DISABLED - -container_scanning: - stage: test - image: docker:stable - allow_failure: true - services: - - docker:stable-dind - script: - - setup_docker - - container_scanning - artifacts: - paths: [gl-container-scanning-report.json] - only: - refs: - - branches - - tags - variables: - - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ - except: - variables: - - $CONTAINER_SCANNING_DISABLED - -dast: - stage: dast - allow_failure: true - image: registry.gitlab.com/gitlab-org/security-products/zaproxy - variables: - POSTGRES_DB: "false" - script: - - dast - artifacts: - paths: [gl-dast-report.json] - only: - refs: - - branches - - tags - kubernetes: active - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ - except: - refs: - - master - variables: - - $DAST_DISABLED - -review: - stage: review - script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - persist_environment_url - environment: - name: review/$CI_COMMIT_REF_NAME - url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN - on_stop: stop_review - artifacts: - paths: [environment_url.txt] - only: - refs: - - branches - - tags - kubernetes: active - except: - refs: - - master - variables: - - $REVIEW_DISABLED - -stop_review: - stage: cleanup - variables: - GIT_STRATEGY: none - script: - - install_dependencies - - initialize_tiller - - delete - environment: - name: review/$CI_COMMIT_REF_NAME - action: stop - when: manual - allow_failure: true - only: - refs: - - branches - - tags - kubernetes: active - except: - refs: - - master - variables: - - $REVIEW_DISABLED - -# Staging deploys are disabled by default since -# continuous deployment to production is enabled by default -# If you prefer to automatically deploy to staging and -# only manually promote to production, enable this job by setting -# STAGING_ENABLED. - -staging: - stage: staging - script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - environment: - name: staging - url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN - only: - refs: - - master - kubernetes: active - variables: - - $STAGING_ENABLED - -# Canaries are also disabled by default, but if you want them, -# and know what the downsides are, you can enable this by setting -# CANARY_ENABLED. - -canary: - stage: canary - script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy canary - environment: - name: production - url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN - when: manual - only: - refs: - - master - kubernetes: active - variables: - - $CANARY_ENABLED - -.production: &production_template - stage: production - script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy - - delete canary - - delete rollout - - persist_environment_url - environment: - name: production - url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN - artifacts: - paths: [environment_url.txt] - -production: - <<: *production_template - only: - refs: - - master - kubernetes: active - except: - variables: - - $STAGING_ENABLED - - $CANARY_ENABLED - - $INCREMENTAL_ROLLOUT_ENABLED - - $INCREMENTAL_ROLLOUT_MODE - -production_manual: - <<: *production_template - when: manual - allow_failure: false - only: - refs: - - master - kubernetes: active - variables: - - $STAGING_ENABLED - - $CANARY_ENABLED - except: - variables: - - $INCREMENTAL_ROLLOUT_ENABLED - - $INCREMENTAL_ROLLOUT_MODE - -# This job implements incremental rollout on for every push to `master`. - -.rollout: &rollout_template - script: - - check_kube_domain - - install_dependencies - - download_chart - - ensure_namespace - - initialize_tiller - - create_secret - - deploy rollout $ROLLOUT_PERCENTAGE - - scale stable $((100-ROLLOUT_PERCENTAGE)) - - delete canary - - persist_environment_url - environment: - name: production - url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN - artifacts: - paths: [environment_url.txt] - -.manual_rollout_template: &manual_rollout_template - <<: *rollout_template - stage: production - when: manual - # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4) - only: - refs: - - master - kubernetes: active - variables: - - $INCREMENTAL_ROLLOUT_MODE == "manual" - - $INCREMENTAL_ROLLOUT_ENABLED - except: - variables: - - $INCREMENTAL_ROLLOUT_MODE == "timed" - -.timed_rollout_template: &timed_rollout_template - <<: *rollout_template - when: delayed - start_in: 5 minutes - only: - refs: - - master - kubernetes: active - variables: - - $INCREMENTAL_ROLLOUT_MODE == "timed" - -timed rollout 10%: - <<: *timed_rollout_template - stage: incremental rollout 10% - variables: - ROLLOUT_PERCENTAGE: 10 - -timed rollout 25%: - <<: *timed_rollout_template - stage: incremental rollout 25% - variables: - ROLLOUT_PERCENTAGE: 25 - -timed rollout 50%: - <<: *timed_rollout_template - stage: incremental rollout 50% - variables: - ROLLOUT_PERCENTAGE: 50 - -timed rollout 100%: - <<: *timed_rollout_template - <<: *production_template - stage: incremental rollout 100% - variables: - ROLLOUT_PERCENTAGE: 100 - -rollout 10%: - <<: *manual_rollout_template - variables: - ROLLOUT_PERCENTAGE: 10 - -rollout 25%: - <<: *manual_rollout_template - variables: - ROLLOUT_PERCENTAGE: 25 - -rollout 50%: - <<: *manual_rollout_template - variables: - ROLLOUT_PERCENTAGE: 50 - -rollout 100%: - <<: *manual_rollout_template - <<: *production_template - allow_failure: false - -# --------------------------------------------------------------------------- - -.auto_devops: &auto_devops | - # Auto DevOps variables and functions - [[ "$TRACE" ]] && set -x - auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} - export DATABASE_URL=${DATABASE_URL-$auto_database_url} - if [[ -z "$CI_COMMIT_TAG" ]]; then - export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG - export CI_APPLICATION_TAG=$CI_COMMIT_SHA - else - export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE - export CI_APPLICATION_TAG=$CI_COMMIT_TAG - fi - export TILLER_NAMESPACE=$KUBE_NAMESPACE - # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - function registry_login() { - if [[ -n "$CI_REGISTRY_USER" ]]; then - echo "Logging to GitLab Container Registry with CI credentials..." - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - echo "" - fi - } - - function container_scanning() { - registry_login - - docker run -d --name db arminc/clair-db:latest - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.6 - apk add -U wget ca-certificates - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 - mv clair-scanner_linux_amd64 clair-scanner - chmod +x clair-scanner - touch clair-whitelist.yml - retries=0 - echo "Waiting for clair daemon to start" - while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done - ./clair-scanner -c http://${DOCKER_SERVICE}:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true - } - - function code_quality() { - docker run --env SOURCE_CODE="$PWD" \ - --volume "$PWD":/code \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code - } - - function license_management() { - /run.sh analyze . - } - - function sast() { - case "$CI_SERVER_VERSION" in - *-ee) - - # Deprecation notice for CONFIDENCE_LEVEL variable - if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then - SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL" - echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL" - fi - - docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code - ;; - *) - echo "GitLab EE is required" - ;; - esac - } - - function dependency_scanning() { - case "$CI_SERVER_VERSION" in - *-ee) - docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \ - --volume "$PWD:/code" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code - ;; - *) - echo "GitLab EE is required" - ;; - esac - } - - function get_replicas() { - track="${1:-stable}" - percentage="${2:-100}" - - env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) - env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) - - if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then - # for stable track get number of replicas from `PRODUCTION_REPLICAS` - eval new_replicas=\$${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - new_replicas=$REPLICAS - fi - else - # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` - eval new_replicas=\$${env_track}_${env_slug}_REPLICAS - if [[ -z "$new_replicas" ]]; then - eval new_replicas=\${env_track}_REPLICAS - fi - fi - - replicas="${new_replicas:-1}" - replicas="$(($replicas * $percentage / 100))" - - # always return at least one replicas - if [[ $replicas -gt 0 ]]; then - echo "$replicas" - else - echo 1 - fi - } - - # Extracts variables prefixed with K8S_SECRET_ - # and creates a Kubernetes secret. - # - # e.g. If we have the following environment variables: - # K8S_SECRET_A=value1 - # K8S_SECRET_B=multi\ word\ value - # - # Then we will create a secret with the following key-value pairs: - # data: - # A: dmFsdWUxCg== - # B: bXVsdGkgd29yZCB2YWx1ZQo= - function create_application_secret() { - track="${1-stable}" - export APPLICATION_SECRET_NAME=$(application_secret_name "$track") - - env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables - - kubectl create secret \ - -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ - --from-env-file k8s_prefixed_variables -o yaml --dry-run | - kubectl replace -n "$KUBE_NAMESPACE" --force -f - - - export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) - - rm k8s_prefixed_variables - } - - function deploy_name() { - name="$CI_ENVIRONMENT_SLUG" - track="${1-stable}" - - if [[ "$track" != "stable" ]]; then - name="$name-$track" - fi - - echo $name - } - - function application_secret_name() { - track="${1-stable}" - name=$(deploy_name "$track") - - echo "${name}-secret" - } - - function deploy() { - track="${1-stable}" - percentage="${2:-100}" - name=$(deploy_name "$track") - - replicas="1" - service_enabled="true" - postgres_enabled="$POSTGRES_ENABLED" - - # if track is different than stable, - # re-use all attached resources - if [[ "$track" != "stable" ]]; then - service_enabled="false" - postgres_enabled="false" - fi - - replicas=$(get_replicas "$track" "$percentage") - - if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then - secret_name='gitlab-registry' - else - secret_name='' - fi - - create_application_secret "$track" - - env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') - eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS - if [ -n "$env_ADDITIONAL_HOSTS" ]; then - additional_hosts="{$env_ADDITIONAL_HOSTS}" - elif [ -n "$ADDITIONAL_HOSTS" ]; then - additional_hosts="{$ADDITIONAL_HOSTS}" - fi - - if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then - echo "Deploying first release with database initialization..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$CI_APPLICATION_REPOSITORY" \ - --set image.tag="$CI_APPLICATION_TAG" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set postgresql.imageTag="$POSTGRES_VERSION" \ - --set application.initializeCommand="$DB_INITIALIZE" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - - echo "Deploying second release..." - helm upgrade --reuse-values \ - --wait \ - --set application.initializeCommand="" \ - --set application.migrateCommand="$DB_MIGRATE" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - else - echo "Deploying new release..." - helm upgrade --install \ - --wait \ - --set service.enabled="$service_enabled" \ - --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ - --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ - --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ - --set image.repository="$CI_APPLICATION_REPOSITORY" \ - --set image.tag="$CI_APPLICATION_TAG" \ - --set image.pullPolicy=IfNotPresent \ - --set image.secrets[0].name="$secret_name" \ - --set application.track="$track" \ - --set application.database_url="$DATABASE_URL" \ - --set application.secretName="$APPLICATION_SECRET_NAME" \ - --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ - --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ - --set service.url="$CI_ENVIRONMENT_URL" \ - --set service.additionalHosts="$additional_hosts" \ - --set replicaCount="$replicas" \ - --set postgresql.enabled="$postgres_enabled" \ - --set postgresql.nameOverride="postgres" \ - --set postgresql.postgresUser="$POSTGRES_USER" \ - --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ - --set postgresql.postgresDatabase="$POSTGRES_DB" \ - --set application.migrateCommand="$DB_MIGRATE" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - - kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" - } - - function scale() { - track="${1-stable}" - percentage="${2-100}" - name=$(deploy_name "$track") - - replicas=$(get_replicas "$track" "$percentage") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm upgrade --reuse-values \ - --wait \ - --set replicaCount="$replicas" \ - --namespace="$KUBE_NAMESPACE" \ - "$name" \ - chart/ - fi - } - - function install_dependencies() { - apk add -U openssl curl tar gzip bash ca-certificates git - curl -sSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub - curl -sSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk - apk add glibc-2.28-r0.apk - rm glibc-2.28-r0.apk - - curl -sS "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx - mv linux-amd64/helm /usr/bin/ - mv linux-amd64/tiller /usr/bin/ - helm version --client - tiller -version - - curl -sSL -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" - chmod +x /usr/bin/kubectl - kubectl version --client - } - - # With the Kubernetes executor, 'localhost' must be used instead - # https://docs.gitlab.com/runner/executors/kubernetes.html - function setup_docker() { - if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then - export DOCKER_HOST='tcp://localhost:2375' - export DOCKER_SERVICE="localhost" - else - export DOCKER_SERVICE="docker" - fi - fi - } - - function setup_test_db() { - if [ -z ${KUBERNETES_PORT+x} ]; then - DB_HOST=postgres - else - DB_HOST=localhost - fi - export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}" - } - - function download_chart() { - if [[ ! -d chart ]]; then - auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} - auto_chart_name=$(basename $auto_chart) - auto_chart_name=${auto_chart_name%.tgz} - auto_chart_name=${auto_chart_name%.tar.gz} - else - auto_chart="chart" - auto_chart_name="chart" - fi - - helm init --client-only - helm repo add gitlab ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} - if [[ ! -d "$auto_chart" ]]; then - helm fetch ${auto_chart} --untar - fi - if [ "$auto_chart_name" != "chart" ]; then - mv ${auto_chart_name} chart - fi - - helm dependency update chart/ - helm dependency build chart/ - } - - function ensure_namespace() { - kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" - } - - - # Function to ensure backwards compatibility with AUTO_DEVOPS_DOMAIN - function ensure_kube_ingress_base_domain() { - if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ] && [ -n "$AUTO_DEVOPS_DOMAIN" ] ; then - export KUBE_INGRESS_BASE_DOMAIN=$AUTO_DEVOPS_DOMAIN - fi - } - - function check_kube_domain() { - ensure_kube_ingress_base_domain - - if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then - echo "In order to deploy or use Review Apps," - echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set" - echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" - echo "or by defining a variable at group or project level." - echo "You can also manually add it in .gitlab-ci.yml" - echo "AUTO_DEVOPS_DOMAIN support will be dropped on 12.0" - false - else - true - fi - } - - function initialize_tiller() { - echo "Checking Tiller..." - - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & - echo "Tiller is listening on ${HELM_HOST}" - - if ! helm version --debug; then - echo "Failed to init Tiller." - return 1 - fi - echo "" - } - - function create_secret() { - echo "Create secret..." - if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then - return - fi - - kubectl create secret -n "$KUBE_NAMESPACE" \ - docker-registry gitlab-registry \ - --docker-server="$CI_REGISTRY" \ - --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ - --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ - --docker-email="$GITLAB_USER_EMAIL" \ - -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - - } - - function dast() { - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - - mkdir /zap/wrk/ - /zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true - cp /zap/wrk/gl-dast-report.json . - } - - function performance() { - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - - mkdir gitlab-exporter - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js - - mkdir sitespeed-results - - if [ -f .gitlab-urls.txt ] - then - sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt - else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" - fi - - mv sitespeed-results/data/performance.json performance.json - } - - function persist_environment_url() { - echo $CI_ENVIRONMENT_URL > environment_url.txt - } - - function delete() { - track="${1-stable}" - name=$(deploy_name "$track") - - if [[ -n "$(helm ls -q "^$name$")" ]]; then - helm delete --purge "$name" - fi - - secret_name=$(application_secret_name "$track") - kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" - } - -before_script: - - *auto_devops +include: + - template: Jobs/Build.gitlab-ci.yml + - template: Jobs/Test.gitlab-ci.yml + - template: Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Deploy.gitlab-ci.yml + - template: Jobs/Browser-Performance-Testing.gitlab-ci.yml + - template: Jobs/DAST.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml + - template: Security/Dependency-Scanning.gitlab-ci.yml + - template: Security/License-Management.gitlab-ci.yml + - template: Security/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml new file mode 100644 index 00000000000..546c4affb4e --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -0,0 +1,38 @@ +performance: + stage: performance + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + - mkdir gitlab-exporter + - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js + - mkdir sitespeed-results + - | + if [ -f .gitlab-urls.txt ] + then + sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt + else + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + fi + - mv sitespeed-results/data/performance.json performance.json + artifacts: + paths: + - performance.json + - sitespeed-results/ + only: + refs: + - branches + - tags + kubernetes: active + except: + variables: + - $PERFORMANCE_DISABLED diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml new file mode 100644 index 00000000000..18f7290e1d9 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -0,0 +1,18 @@ +build: + stage: build + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image/master:stable" + services: + - docker:stable-dind + script: + - | + if [[ -z "$CI_COMMIT_TAG" ]]; then + export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} + export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} + else + export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} + export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} + fi + - /build/build.sh + only: + - branches + - tags diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml new file mode 100644 index 00000000000..b09a24d8e22 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -0,0 +1,27 @@ +code_quality: + stage: test + image: docker:stable + allow_failure: true + services: + - docker:stable-dind + script: + - export CQ_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - | + docker run --env SOURCE_CODE="$PWD" \ + --volume "$PWD":/code \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/codequality:$CQ_VERSION" /code + artifacts: + paths: [gl-code-quality-report.json] + only: + - branches + - tags + except: + variables: + - $CODE_QUALITY_DISABLED diff --git a/lib/gitlab/ci/templates/Jobs/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST.gitlab-ci.yml new file mode 100644 index 00000000000..aedbbb21674 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/DAST.gitlab-ci.yml @@ -0,0 +1,54 @@ +dast: + stage: dast + image: docker:stable + variables: + DOCKER_DRIVER: overlay2 + allow_failure: true + services: + - docker:stable-dind + script: + - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} + - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} + - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - | + function dast_run() { + docker run \ + --env DAST_TARGET_AVAILABILITY_TIMEOUT \ + --volume "$PWD:/output" \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + -w /output \ + "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ + /analyze -t $DAST_WEBSITE \ + "$@" + } + - | + if [ -n "$DAST_AUTH_URL" ] + then + dast_run \ + --auth-url $DAST_AUTH_URL \ + --auth-username $DAST_USERNAME \ + --auth-password $DAST_PASSWORD \ + --auth-username-field $DAST_USERNAME_FIELD \ + --auth-password-field $DAST_PASSWORD_FIELD + else + dast_run + fi + artifacts: + reports: + dast: gl-dast-report.json + only: + refs: + - branches + - tags + variables: + - $GITLAB_FEATURES =~ /\bdast\b/ + except: + refs: + - master + variables: + - $DAST_DISABLED diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml new file mode 100644 index 00000000000..1e9591e113b --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -0,0 +1,575 @@ +review: + stage: review + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + - persist_environment_url + environment: + name: review/$CI_COMMIT_REF_NAME + url: http://$CI_PROJECT_ID-$CI_ENVIRONMENT_SLUG.$KUBE_INGRESS_BASE_DOMAIN + on_stop: stop_review + artifacts: + paths: [environment_url.txt] + only: + refs: + - branches + - tags + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +stop_review: + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - install_dependencies + - initialize_tiller + - delete + environment: + name: review/$CI_COMMIT_REF_NAME + action: stop + when: manual + allow_failure: true + only: + refs: + - branches + - tags + kubernetes: active + except: + refs: + - master + variables: + - $REVIEW_DISABLED + +# Staging deploys are disabled by default since +# continuous deployment to production is enabled by default +# If you prefer to automatically deploy to staging and +# only manually promote to production, enable this job by setting +# STAGING_ENABLED. + +staging: + stage: staging + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + environment: + name: staging + url: http://$CI_PROJECT_PATH_SLUG-staging.$KUBE_INGRESS_BASE_DOMAIN + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + +# Canaries are disabled by default, but if you want them, +# and know what the downsides are, you can enable this by setting +# CANARY_ENABLED. + +canary: + stage: canary + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy canary + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + when: manual + only: + refs: + - master + kubernetes: active + variables: + - $CANARY_ENABLED + +.production: &production_template + stage: production + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy + - delete canary + - delete rollout + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + artifacts: + paths: [environment_url.txt] + +production: + <<: *production_template + only: + refs: + - master + kubernetes: active + except: + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + - $INCREMENTAL_ROLLOUT_ENABLED + - $INCREMENTAL_ROLLOUT_MODE + +production_manual: + <<: *production_template + when: manual + allow_failure: false + only: + refs: + - master + kubernetes: active + variables: + - $STAGING_ENABLED + - $CANARY_ENABLED + except: + variables: + - $INCREMENTAL_ROLLOUT_ENABLED + - $INCREMENTAL_ROLLOUT_MODE + +# This job implements incremental rollout on for every push to `master`. + +.rollout: &rollout_template + script: + - check_kube_domain + - install_dependencies + - download_chart + - ensure_namespace + - initialize_tiller + - create_secret + - deploy rollout $ROLLOUT_PERCENTAGE + - scale stable $((100-ROLLOUT_PERCENTAGE)) + - delete canary + - persist_environment_url + environment: + name: production + url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN + artifacts: + paths: [environment_url.txt] + +.manual_rollout_template: &manual_rollout_template + <<: *rollout_template + stage: production + when: manual + # This selectors are backward compatible mode with $INCREMENTAL_ROLLOUT_ENABLED (before 11.4) + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_MODE == "manual" + - $INCREMENTAL_ROLLOUT_ENABLED + except: + variables: + - $INCREMENTAL_ROLLOUT_MODE == "timed" + +.timed_rollout_template: &timed_rollout_template + <<: *rollout_template + when: delayed + start_in: 5 minutes + only: + refs: + - master + kubernetes: active + variables: + - $INCREMENTAL_ROLLOUT_MODE == "timed" + +timed rollout 10%: + <<: *timed_rollout_template + stage: incremental rollout 10% + variables: + ROLLOUT_PERCENTAGE: 10 + +timed rollout 25%: + <<: *timed_rollout_template + stage: incremental rollout 25% + variables: + ROLLOUT_PERCENTAGE: 25 + +timed rollout 50%: + <<: *timed_rollout_template + stage: incremental rollout 50% + variables: + ROLLOUT_PERCENTAGE: 50 + +timed rollout 100%: + <<: *timed_rollout_template + <<: *production_template + stage: incremental rollout 100% + variables: + ROLLOUT_PERCENTAGE: 100 + +rollout 10%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 10 + +rollout 25%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 25 + +rollout 50%: + <<: *manual_rollout_template + variables: + ROLLOUT_PERCENTAGE: 50 + +rollout 100%: + <<: *manual_rollout_template + <<: *production_template + allow_failure: false + +.deploy_helpers: &deploy_helpers | + [[ "$TRACE" ]] && set -x + auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB} + export DATABASE_URL=${DATABASE_URL-$auto_database_url} + export TILLER_NAMESPACE=$KUBE_NAMESPACE + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products + + function get_replicas() { + track="${1:-stable}" + percentage="${2:-100}" + + env_track=$( echo $track | tr -s '[:lower:]' '[:upper:]' ) + env_slug=$( echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]' ) + + if [[ "$track" == "stable" ]] || [[ "$track" == "rollout" ]]; then + # for stable track get number of replicas from `PRODUCTION_REPLICAS` + eval new_replicas=\$${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + new_replicas=$REPLICAS + fi + else + # for all tracks get number of replicas from `CANARY_PRODUCTION_REPLICAS` + eval new_replicas=\$${env_track}_${env_slug}_REPLICAS + if [[ -z "$new_replicas" ]]; then + eval new_replicas=\${env_track}_REPLICAS + fi + fi + + replicas="${new_replicas:-1}" + replicas="$(($replicas * $percentage / 100))" + + # always return at least one replicas + if [[ $replicas -gt 0 ]]; then + echo "$replicas" + else + echo 1 + fi + } + + # Extracts variables prefixed with K8S_SECRET_ + # and creates a Kubernetes secret. + # + # e.g. If we have the following environment variables: + # K8S_SECRET_A=value1 + # K8S_SECRET_B=multi\ word\ value + # + # Then we will create a secret with the following key-value pairs: + # data: + # A: dmFsdWUxCg== + # B: bXVsdGkgd29yZCB2YWx1ZQo= + function create_application_secret() { + track="${1-stable}" + export APPLICATION_SECRET_NAME=$(application_secret_name "$track") + + env | sed -n "s/^K8S_SECRET_\(.*\)$/\1/p" > k8s_prefixed_variables + + kubectl create secret \ + -n "$KUBE_NAMESPACE" generic "$APPLICATION_SECRET_NAME" \ + --from-env-file k8s_prefixed_variables -o yaml --dry-run | + kubectl replace -n "$KUBE_NAMESPACE" --force -f - + + export APPLICATION_SECRET_CHECKSUM=$(cat k8s_prefixed_variables | sha256sum | cut -d ' ' -f 1) + + rm k8s_prefixed_variables + } + + function deploy_name() { + name="$CI_ENVIRONMENT_SLUG" + track="${1-stable}" + + if [[ "$track" != "stable" ]]; then + name="$name-$track" + fi + + echo $name + } + + function application_secret_name() { + track="${1-stable}" + name=$(deploy_name "$track") + + echo "${name}-secret" + } + + function deploy() { + track="${1-stable}" + percentage="${2:-100}" + name=$(deploy_name "$track") + + if [[ -z "$CI_COMMIT_TAG" ]]; then + image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} + image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} + else + image_repository=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} + image_tag=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} + fi + + replicas="1" + service_enabled="true" + postgres_enabled="$POSTGRES_ENABLED" + + # if track is different than stable, + # re-use all attached resources + if [[ "$track" != "stable" ]]; then + service_enabled="false" + postgres_enabled="false" + fi + + replicas=$(get_replicas "$track" "$percentage") + + if [[ "$CI_PROJECT_VISIBILITY" != "public" ]]; then + secret_name='gitlab-registry' + else + secret_name='' + fi + + create_application_secret "$track" + + env_slug=$(echo ${CI_ENVIRONMENT_SLUG//-/_} | tr -s '[:lower:]' '[:upper:]') + eval env_ADDITIONAL_HOSTS=\$${env_slug}_ADDITIONAL_HOSTS + if [ -n "$env_ADDITIONAL_HOSTS" ]; then + additional_hosts="{$env_ADDITIONAL_HOSTS}" + elif [ -n "$ADDITIONAL_HOSTS" ]; then + additional_hosts="{$ADDITIONAL_HOSTS}" + fi + + if [[ -n "$DB_INITIALIZE" && -z "$(helm ls -q "^$name$")" ]]; then + echo "Deploying first release with database initialization..." + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ + --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$image_repository" \ + --set image.tag="$image_tag" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ + --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ + --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set service.additionalHosts="$additional_hosts" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set postgresql.imageTag="$POSTGRES_VERSION" \ + --set application.initializeCommand="$DB_INITIALIZE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + + echo "Deploying second release..." + helm upgrade --reuse-values \ + --wait \ + --set application.initializeCommand="" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + else + echo "Deploying new release..." + helm upgrade --install \ + --wait \ + --set service.enabled="$service_enabled" \ + --set gitlab.app="$CI_PROJECT_PATH_SLUG" \ + --set gitlab.env="$CI_ENVIRONMENT_SLUG" \ + --set releaseOverride="$CI_ENVIRONMENT_SLUG" \ + --set image.repository="$image_repository" \ + --set image.tag="$image_tag" \ + --set image.pullPolicy=IfNotPresent \ + --set image.secrets[0].name="$secret_name" \ + --set application.track="$track" \ + --set application.database_url="$DATABASE_URL" \ + --set application.secretName="$APPLICATION_SECRET_NAME" \ + --set application.secretChecksum="$APPLICATION_SECRET_CHECKSUM" \ + --set service.commonName="le.$KUBE_INGRESS_BASE_DOMAIN" \ + --set service.url="$CI_ENVIRONMENT_URL" \ + --set service.additionalHosts="$additional_hosts" \ + --set replicaCount="$replicas" \ + --set postgresql.enabled="$postgres_enabled" \ + --set postgresql.nameOverride="postgres" \ + --set postgresql.postgresUser="$POSTGRES_USER" \ + --set postgresql.postgresPassword="$POSTGRES_PASSWORD" \ + --set postgresql.postgresDatabase="$POSTGRES_DB" \ + --set application.migrateCommand="$DB_MIGRATE" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + + kubectl rollout status -n "$KUBE_NAMESPACE" -w "$ROLLOUT_RESOURCE_TYPE/$name" + } + + function scale() { + track="${1-stable}" + percentage="${2-100}" + name=$(deploy_name "$track") + + replicas=$(get_replicas "$track" "$percentage") + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm upgrade --reuse-values \ + --wait \ + --set replicaCount="$replicas" \ + --namespace="$KUBE_NAMESPACE" \ + "$name" \ + chart/ + fi + } + + function install_dependencies() { + apk add -U openssl curl tar gzip bash ca-certificates git + curl -sSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub + curl -sSL -O https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.28-r0/glibc-2.28-r0.apk + apk add glibc-2.28-r0.apk + rm glibc-2.28-r0.apk + + curl -sS "https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz" | tar zx + mv linux-amd64/helm /usr/bin/ + mv linux-amd64/tiller /usr/bin/ + helm version --client + tiller -version + + curl -sSL -o /usr/bin/kubectl "https://storage.googleapis.com/kubernetes-release/release/v${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" + chmod +x /usr/bin/kubectl + kubectl version --client + } + + function download_chart() { + if [[ ! -d chart ]]; then + auto_chart=${AUTO_DEVOPS_CHART:-gitlab/auto-deploy-app} + auto_chart_name=$(basename $auto_chart) + auto_chart_name=${auto_chart_name%.tgz} + auto_chart_name=${auto_chart_name%.tar.gz} + else + auto_chart="chart" + auto_chart_name="chart" + fi + + helm init --client-only + helm repo add gitlab ${AUTO_DEVOPS_CHART_REPOSITORY:-https://charts.gitlab.io} + if [[ ! -d "$auto_chart" ]]; then + helm fetch ${auto_chart} --untar + fi + if [ "$auto_chart_name" != "chart" ]; then + mv ${auto_chart_name} chart + fi + + helm dependency update chart/ + helm dependency build chart/ + } + + function ensure_namespace() { + kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE" + } + + # Function to ensure backwards compatibility with AUTO_DEVOPS_DOMAIN + function ensure_kube_ingress_base_domain() { + if [ -z ${KUBE_INGRESS_BASE_DOMAIN+x} ] && [ -n "$AUTO_DEVOPS_DOMAIN" ] ; then + export KUBE_INGRESS_BASE_DOMAIN=$AUTO_DEVOPS_DOMAIN + fi + } + + function check_kube_domain() { + ensure_kube_ingress_base_domain + + if [[ -z "$KUBE_INGRESS_BASE_DOMAIN" ]]; then + echo "In order to deploy or use Review Apps," + echo "AUTO_DEVOPS_DOMAIN or KUBE_INGRESS_BASE_DOMAIN variables must be set" + echo "From 11.8, you can set KUBE_INGRESS_BASE_DOMAIN in cluster settings" + echo "or by defining a variable at group or project level." + echo "You can also manually add it in .gitlab-ci.yml" + echo "AUTO_DEVOPS_DOMAIN support will be dropped on 12.0" + false + else + true + fi + } + + function initialize_tiller() { + echo "Checking Tiller..." + + export HELM_HOST="localhost:44134" + tiller -listen ${HELM_HOST} -alsologtostderr > /dev/null 2>&1 & + echo "Tiller is listening on ${HELM_HOST}" + + if ! helm version --debug; then + echo "Failed to init Tiller." + return 1 + fi + echo "" + } + + function create_secret() { + echo "Create secret..." + if [[ "$CI_PROJECT_VISIBILITY" == "public" ]]; then + return + fi + + kubectl create secret -n "$KUBE_NAMESPACE" \ + docker-registry gitlab-registry \ + --docker-server="$CI_REGISTRY" \ + --docker-username="${CI_DEPLOY_USER:-$CI_REGISTRY_USER}" \ + --docker-password="${CI_DEPLOY_PASSWORD:-$CI_REGISTRY_PASSWORD}" \ + --docker-email="$GITLAB_USER_EMAIL" \ + -o yaml --dry-run | kubectl replace -n "$KUBE_NAMESPACE" --force -f - + } + + function persist_environment_url() { + echo $CI_ENVIRONMENT_URL > environment_url.txt + } + + function delete() { + track="${1-stable}" + name=$(deploy_name "$track") + + if [[ -n "$(helm ls -q "^$name$")" ]]; then + helm delete --purge "$name" + fi + + secret_name=$(application_secret_name "$track") + kubectl delete secret --ignore-not-found -n "$KUBE_NAMESPACE" "$secret_name" + } + +before_script: + - *deploy_helpers diff --git a/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml new file mode 100644 index 00000000000..b9fe838d1da --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Test.gitlab-ci.yml @@ -0,0 +1,23 @@ +test: + services: + - postgres:latest + variables: + POSTGRES_DB: test + stage: test + image: gliderlabs/herokuish:latest + script: + - | + if [ -z ${KUBERNETES_PORT+x} ]; then + DB_HOST=postgres + else + DB_HOST=localhost + fi + - export DATABASE_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${DB_HOST}:5432/${POSTGRES_DB}" + - cp -R . /tmp/app + - /bin/herokuish buildpack test + only: + - branches + - tags + except: + variables: + - $TEST_DISABLED diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 0b7a531682b..eef361c19e9 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -28,6 +28,12 @@ container_scanning: - docker:stable-dind script: - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi + - | + if [[ -n "$CI_REGISTRY_USER" ]]; then + echo "Logging to GitLab Container Registry with CI credentials..." + docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" + echo "" + fi - docker run -d --name db arminc/clair-db:latest - docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION} - apk add -U wget ca-certificates @@ -36,7 +42,6 @@ container_scanning: - mv clair-scanner_linux_amd64 clair-scanner - chmod +x clair-scanner - touch clair-whitelist.yml - - while( ! wget -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; done - retries=0 - echo "Waiting for clair daemon to start" - while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index ef6d7866e85..770340de16f 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -4,6 +4,9 @@ # List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables +include: + - template: Jobs/DAST.gitlab-ci.yml + variables: DAST_WEBSITE: http://example.com # Please edit to be your website to scan for vulnerabilities @@ -14,46 +17,10 @@ stages: - dast dast: - stage: dast - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - allow_failure: true - services: - - docker:stable-dind - script: - - export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - - | - function dast_run() { - docker run \ - --env DAST_TARGET_AVAILABILITY_TIMEOUT \ - --volume "$PWD:/output" \ - --volume /var/run/docker.sock:/var/run/docker.sock \ - -w /output \ - "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \ - /analyze -t $DAST_WEBSITE \ - "$@" - } - - | - if [ -n "$DAST_AUTH_URL" ] - then - dast_run \ - --auth-url $DAST_AUTH_URL \ - --auth-username $DAST_USERNAME \ - --auth-password $DAST_PASSWORD \ - --auth-username-field $DAST_USERNAME_FIELD \ - --auth-password-field $DAST_PASSWORD_FIELD - else - dast_run - fi - artifacts: - reports: - dast: gl-dast-report.json only: refs: - branches - variables: - - $GITLAB_FEATURES =~ /\bdast\b/ except: + refs: [] # Override default from template variables: - $DAST_DISABLED diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index fd666541d41..7f80a6e9285 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -15,6 +15,12 @@ dependency_scanning: script: - export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - | docker run \ --env DS_ANALYZER_IMAGES \ --env DS_ANALYZER_IMAGE_PREFIX \ diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 034fba5499c..b941e89991e 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -15,6 +15,12 @@ sast: script: - export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')} - | + if ! docker info &>/dev/null; then + if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + export DOCKER_HOST='tcp://localhost:2375' + fi + fi + - | docker run \ --env SAST_ANALYZER_IMAGES \ --env SAST_ANALYZER_IMAGE_PREFIX \ diff --git a/lib/gitlab/config/entry/configurable.rb b/lib/gitlab/config/entry/configurable.rb index 37ba16dba25..6667a5d3d33 100644 --- a/lib/gitlab/config/entry/configurable.rb +++ b/lib/gitlab/config/entry/configurable.rb @@ -21,7 +21,7 @@ module Gitlab include Validatable validations do - validates :config, type: Hash + validates :config, type: Hash, unless: :skip_config_hash_validation? end end @@ -30,6 +30,10 @@ module Gitlab return unless valid? self.class.nodes.each do |key, factory| + # If we override the config type validation + # we can end with different config types like String + next unless config.is_a?(Hash) + factory .value(config[key]) .with(key: key, parent: self) @@ -45,6 +49,10 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def skip_config_hash_validation? + false + end + class_methods do def nodes Hash[(@nodes || {}).map { |key, factory| [key, factory.dup] }] diff --git a/lib/gitlab/config/entry/factory.rb b/lib/gitlab/config/entry/factory.rb index 79f9ff32514..3c06b1e0d24 100644 --- a/lib/gitlab/config/entry/factory.rb +++ b/lib/gitlab/config/entry/factory.rb @@ -61,7 +61,7 @@ module Gitlab end def fabricate(entry, value = nil) - entry.new(value, @metadata).tap do |node| + entry.new(value, @metadata) do |node| node.key = @attributes[:key] node.parent = @attributes[:parent] node.default = @attributes[:default] diff --git a/lib/gitlab/config/entry/node.rb b/lib/gitlab/config/entry/node.rb index 9999ab4ff95..e014f15fbd8 100644 --- a/lib/gitlab/config/entry/node.rb +++ b/lib/gitlab/config/entry/node.rb @@ -17,6 +17,8 @@ module Gitlab @metadata = metadata @entries = {} + yield(self) if block_given? + self.class.aspects.to_a.each do |aspect| instance_exec(&aspect) end @@ -44,6 +46,12 @@ module Gitlab @parent ? @parent.ancestors + [@parent] : [] end + def opt(key) + opt = metadata[key] + opt = @parent.opt(key) if opt.nil? && @parent + opt + end + def valid? errors.none? end @@ -85,6 +93,18 @@ module Gitlab "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>" end + def hash? + @config.is_a?(Hash) + end + + def string? + @config.is_a?(String) + end + + def integer? + @config.is_a?(Integer) + end + def self.default(**) end diff --git a/lib/gitlab/config/entry/simplifiable.rb b/lib/gitlab/config/entry/simplifiable.rb index 5fbf7565e2a..a56a89adb35 100644 --- a/lib/gitlab/config/entry/simplifiable.rb +++ b/lib/gitlab/config/entry/simplifiable.rb @@ -19,7 +19,10 @@ module Gitlab entry = self.class.entry_class(strategy) - super(@subject = entry.new(config, metadata)) + @subject = entry.new(config, metadata) + + yield(@subject) if block_given? + super(@subject) end def self.strategy(name, **opts) diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index d348e11b753..d0ee94370ba 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -15,6 +15,17 @@ module Gitlab end end + class DisallowedKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + present_keys = value.try(:keys).to_a & options[:in] + + if present_keys.any? + record.errors.add(attribute, "contains disallowed keys: " + + present_keys.join(', ')) + end + end + end + class AllowedValuesValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) unless options[:in].include?(value.to_s) @@ -186,6 +197,97 @@ module Gitlab end end end + + class PortNamePresentAndUniqueValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + return unless value.is_a?(Array) + + ports_size = value.count + return if ports_size <= 1 + + named_ports = value.select { |e| e.is_a?(Hash) }.map { |e| e[:name] }.compact.map(&:downcase) + + if ports_size != named_ports.size + record.errors.add(attribute, 'when there is more than one port, a unique name should be added') + end + + if ports_size != named_ports.uniq.size + record.errors.add(attribute, 'each port name must be different') + end + end + end + + class PortUniqueValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + value = ports(value) + return unless value.is_a?(Array) + + ports_size = value.count + return if ports_size <= 1 + + if transform_ports(value).size != ports_size + record.errors.add(attribute, 'each port number can only be referenced once') + end + end + + private + + def ports(current_data) + current_data + end + + def transform_ports(raw_ports) + raw_ports.map do |port| + case port + when Integer + port + when Hash + port[:number] + end + end.uniq + end + end + + class JobPortUniqueValidator < PortUniqueValidator + private + + def ports(current_data) + return unless current_data.is_a?(Hash) + + (image_ports(current_data) + services_ports(current_data)).compact + end + + def image_ports(current_data) + return [] unless current_data[:image].is_a?(Hash) + + current_data.dig(:image, :ports).to_a + end + + def services_ports(current_data) + current_data.dig(:services).to_a.flat_map { |service| service.is_a?(Hash) ? service[:ports] : nil } + end + end + + class ServicesWithPortsAliasUniqueValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + current_aliases = aliases(value) + return if current_aliases.empty? + + unless aliases_unique?(current_aliases) + record.errors.add(:config, 'alias must be unique in services with ports') + end + end + + private + + def aliases(value) + value.select { |s| s.is_a?(Hash) && s[:ports] }.pluck(:alias) # rubocop:disable CodeReuse/ActiveRecord + end + + def aliases_unique?(aliases) + aliases.size == aliases.uniq.size + end + end end end end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 14126b6ec06..c237f4a7404 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -47,6 +47,19 @@ module Gitlab self.source == other.source end + # Handles regular expressions with the preferred RE2 library where possible + # via UntustedRegex. Falls back to Ruby's built-in regular expression library + # when the syntax would be invalid in RE2. + # + # One difference between these is `(?m)` multi-line mode. Ruby regex enables + # this by default, but also handles `^` and `$` differently. + # See: https://www.regular-expressions.info/modifiers.html + def self.with_fallback(pattern, multiline: false) + UntrustedRegexp.new(pattern, multiline: multiline) + rescue RegexpError + Regexp.new(pattern) + end + private attr_reader :regexp diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 7872e5b08c0..8fadadccce9 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -14,6 +14,12 @@ namespace :gitlab do rake_version = run_and_match(%w(rake --version), /[\d\.]+/).try(:to_s) # check redis version redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a + + # check for system defined proxies + if Gitlab.ee? + proxies = Gitlab::Proxy.detect_proxy.map {|k, v| "#{k}: #{v}"}.join("\n\t\t") + end + # check Git version git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a # check Go version @@ -22,6 +28,11 @@ namespace :gitlab do puts "" puts "System information".color(:yellow) puts "System:\t\t#{os_name || "unknown".color(:red)}" + + if Gitlab.ee? + puts "Proxy:\t\t#{proxies.present? ? proxies.color(:green) : "no"}" + end + puts "Current User:\t#{run_command(%w(whoami))}" puts "Using RVM:\t#{rvm_version.present? ? "yes".color(:green) : "no"}" puts "RVM Version:\t#{rvm_version}" if rvm_version.present? @@ -39,6 +50,15 @@ namespace :gitlab do http_clone_url = project.http_url_to_repo ssh_clone_url = project.ssh_url_to_repo + if Gitlab.ee? + geo_node_type = + if Gitlab::Geo.current_node + Gitlab::Geo.current_node.primary ? 'Primary' : 'Secondary' + else + 'Undefined'.color(:red) + end + end + omniauth_providers = Gitlab.config.omniauth.providers.map { |provider| provider['name'] } puts "" @@ -51,6 +71,13 @@ namespace :gitlab do puts "URL:\t\t#{Gitlab.config.gitlab.url}" puts "HTTP Clone URL:\t#{http_clone_url}" puts "SSH Clone URL:\t#{ssh_clone_url}" + + if Gitlab.ee? + puts "Elasticsearch:\t#{Gitlab::CurrentSettings.current_application_settings.elasticsearch_indexing? ? "yes".color(:green) : "no"}" + puts "Geo:\t\t#{Gitlab::Geo.enabled? ? "yes".color(:green) : "no"}" + puts "Geo node:\t#{geo_node_type}" if Gitlab::Geo.enabled? + end + puts "Using LDAP:\t#{Gitlab.config.ldap.enabled ? "yes".color(:green) : "no"}" puts "Using Omniauth:\t#{Gitlab::Auth.omniauth_enabled? ? "yes".color(:green) : "no"}" puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab::Auth.omniauth_enabled? diff --git a/locale/gitlab.pot b/locale/gitlab.pot index fcbd34a05d5..f9c411642c7 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -1620,6 +1620,9 @@ msgstr "" msgid "CiVariables|Remove variable row" msgstr "" +msgid "CiVariables|This variable will not be masked" +msgstr "" + msgid "CiVariable|* (All environments)" msgstr "" @@ -1629,9 +1632,15 @@ msgstr "" msgid "CiVariable|Error occurred while saving variables" msgstr "" +msgid "CiVariable|Masked" +msgstr "" + msgid "CiVariable|Protected" msgstr "" +msgid "CiVariable|Toggle masked" +msgstr "" + msgid "CiVariable|Toggle protected" msgstr "" @@ -3181,10 +3190,7 @@ msgstr "" msgid "Enter the merge request title" msgstr "" -msgid "Environment variables" -msgstr "" - -msgid "Environment variables are applied to environments via the runner. They can be protected by only exposing them to protected branches or tags. You can use environment variables for passwords, secret keys, or whatever you want." +msgid "Environment variables are applied to environments via the runner. They can be protected by only exposing them to protected branches or tags. Additionally, they will be masked by default so they are hidden in job logs, though they must match certain regexp requirements to do so. You can use environment variables for passwords, secret keys, or whatever you want." msgstr "" msgid "Environment variables are configured by your administrator to be %{link_start}protected%{link_end} by default" @@ -3595,6 +3601,9 @@ msgstr "" msgid "Failed to remove user key." msgstr "" +msgid "Failed to save new settings" +msgstr "" + msgid "Failed to update issues, please try again." msgstr "" @@ -8278,9 +8287,6 @@ msgstr "" msgid "This setting will update the hostname that is used to generate private commit emails. %{learn_more}" msgstr "" -msgid "This source diff could not be displayed because it is too large." -msgstr "" - msgid "This timeout will take precedence when lower than project-defined timeout and accepts a human readable time input language like \"1 hour\". Values without specification represent seconds." msgstr "" @@ -8353,9 +8359,6 @@ msgstr "" msgid "Timeago|%s months remaining" msgstr "" -msgid "Timeago|%s seconds ago" -msgstr "" - msgid "Timeago|%s seconds remaining" msgstr "" @@ -8948,6 +8951,9 @@ msgstr "" msgid "Value" msgstr "" +msgid "Variables" +msgstr "" + msgid "Various container registry settings." msgstr "" diff --git a/package.json b/package.json index 29a3b3d0459..ceb36a92cc8 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "eslint-fix": "eslint --max-warnings 0 --report-unused-disable-directives --ext .js,.vue --fix .", "eslint-report": "eslint --max-warnings 0 --ext .js,.vue --format html --output-file ./eslint-report.html --no-inline-config .", "jest": "BABEL_ENV=jest jest", + "jest-debug": "BABEL_ENV=jest node --inspect-brk node_modules/.bin/jest --runInBand", "jsdoc": "jsdoc -c config/jsdocs.config.js", "karma": "BABEL_ENV=${BABEL_ENV:=karma} karma start --single-run true config/karma.config.js", "karma-coverage": "BABEL_ENV=coverage karma start --single-run true config/karma.config.js", @@ -56,11 +57,13 @@ "d3-array": "^1.2.1", "d3-axis": "^1.0.8", "d3-brush": "^1.0.4", + "d3-ease": "^1.0.3", "d3-scale": "^1.0.7", "d3-selection": "^1.2.0", "d3-shape": "^1.2.0", "d3-time": "^1.0.8", "d3-time-format": "^2.1.1", + "d3-transition": "^1.1.1", "dateformat": "^3.0.3", "deckar01-task_list": "^2.2.0", "diff": "^3.4.0", diff --git a/qa/STYLE_GUIDE.md b/qa/STYLE_GUIDE.md index f85e7492409..900f7456e1a 100644 --- a/qa/STYLE_GUIDE.md +++ b/qa/STYLE_GUIDE.md @@ -43,4 +43,4 @@ end Notice that in the above example, before clicking the `:operations_environments_link`, another element is hovered over. -> We can create these methods as helpers to abstrac multi-step navigation.
\ No newline at end of file +> We can create these methods as helpers to abstract multi-step navigation.
\ No newline at end of file diff --git a/spec/features/group_variables_spec.rb b/spec/features/group_variables_spec.rb index 1a53e7c9512..fc5777e8c7c 100644 --- a/spec/features/group_variables_spec.rb +++ b/spec/features/group_variables_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'Group variables', :js do let(:user) { create(:user) } let(:group) { create(:group) } - let!(:variable) { create(:ci_group_variable, key: 'test_key', value: 'test_value', group: group) } + let!(:variable) { create(:ci_group_variable, key: 'test_key', value: 'test_value', masked: true, group: group) } let(:page_path) { group_settings_ci_cd_path(group) } before do diff --git a/spec/features/merge_request/user_posts_notes_spec.rb b/spec/features/merge_request/user_posts_notes_spec.rb index dc0862be6fc..e5770905dbd 100644 --- a/spec/features/merge_request/user_posts_notes_spec.rb +++ b/spec/features/merge_request/user_posts_notes_spec.rb @@ -67,18 +67,7 @@ describe 'Merge request > User posts notes', :js do end end - describe 'when reply_to_individual_notes feature flag is disabled' do - before do - stub_feature_flags(reply_to_individual_notes: false) - visit project_merge_request_path(project, merge_request) - end - - it 'does not show a reply button' do - expect(page).to have_no_selector('.js-reply-button') - end - end - - describe 'when reply_to_individual_notes feature flag is not set' do + describe 'reply button' do before do visit project_merge_request_path(project, merge_request) end diff --git a/spec/features/project_variables_spec.rb b/spec/features/project_variables_spec.rb index 6bdf5df1036..76abc640077 100644 --- a/spec/features/project_variables_spec.rb +++ b/spec/features/project_variables_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe 'Project variables', :js do let(:user) { create(:user) } let(:project) { create(:project) } - let(:variable) { create(:ci_variable, key: 'test_key', value: 'test_value') } + let(:variable) { create(:ci_variable, key: 'test_key', value: 'test_value', masked: true) } let(:page_path) { project_settings_ci_cd_path(project) } before do diff --git a/spec/fixtures/api/schemas/variable.json b/spec/fixtures/api/schemas/variable.json index 6f6b044115b..305071a6b3f 100644 --- a/spec/fixtures/api/schemas/variable.json +++ b/spec/fixtures/api/schemas/variable.json @@ -4,12 +4,14 @@ "id", "key", "value", + "masked", "protected" ], "properties": { "id": { "type": "integer" }, "key": { "type": "string" }, "value": { "type": "string" }, + "masked": { "type": "boolean" }, "protected": { "type": "boolean" }, "environment_scope": { "type": "string", "optional": true } }, diff --git a/spec/frontend/ide/lib/files_spec.js b/spec/frontend/ide/lib/files_spec.js index fe791aa2b74..aa1fa0373db 100644 --- a/spec/frontend/ide/lib/files_spec.js +++ b/spec/frontend/ide/lib/files_spec.js @@ -1,5 +1,5 @@ import { viewerInformationForPath } from '~/vue_shared/components/content_viewer/lib/viewer_utils'; -import { decorateFiles, splitParent } from '~/ide/lib/files'; +import { decorateFiles, splitParent, escapeFileUrl } from '~/ide/lib/files'; import { decorateData } from '~/ide/stores/utils'; const TEST_BRANCH_ID = 'lorem-ipsum'; @@ -20,7 +20,7 @@ const createEntries = paths => { id: path, name, path, - url: createUrl(`/${TEST_PROJECT_ID}/${type}/${TEST_BRANCH_ID}/-/${path}`), + url: createUrl(`/${TEST_PROJECT_ID}/${type}/${TEST_BRANCH_ID}/-/${escapeFileUrl(path)}`), type, previewMode: viewerInformationForPath(path), parentPath: parent, @@ -28,7 +28,7 @@ const createEntries = paths => { ? parentEntry.url : createUrl(`/${TEST_PROJECT_ID}/${type}/${TEST_BRANCH_ID}`), }), - tree: children.map(childName => jasmine.objectContaining({ name: childName })), + tree: children.map(childName => expect.objectContaining({ name: childName })), }; return acc; @@ -36,10 +36,10 @@ const createEntries = paths => { const entries = paths.reduce(createEntry, {}); - // Wrap entries in jasmine.objectContaining. + // Wrap entries in expect.objectContaining. // We couldn't do this earlier because we still need to select properties from parent entries. return Object.keys(entries).reduce((acc, key) => { - acc[key] = jasmine.objectContaining(entries[key]); + acc[key] = expect.objectContaining(entries[key]); return acc; }, {}); @@ -47,13 +47,14 @@ const createEntries = paths => { describe('IDE lib decorate files', () => { it('creates entries and treeList', () => { - const data = ['app/assets/apples/foo.js', 'app/bugs.js', 'README.md']; + const data = ['app/assets/apples/foo.js', 'app/bugs.js', 'app/#weird#file?.txt', 'README.md']; const expectedEntries = createEntries([ - { path: 'app', type: 'tree', children: ['assets', 'bugs.js'] }, + { path: 'app', type: 'tree', children: ['assets', '#weird#file?.txt', 'bugs.js'] }, { path: 'app/assets', type: 'tree', children: ['apples'] }, { path: 'app/assets/apples', type: 'tree', children: ['foo.js'] }, { path: 'app/assets/apples/foo.js', type: 'blob', children: [] }, { path: 'app/bugs.js', type: 'blob', children: [] }, + { path: 'app/#weird#file?.txt', type: 'blob', children: [] }, { path: 'README.md', type: 'blob', children: [] }, ]); @@ -64,7 +65,7 @@ describe('IDE lib decorate files', () => { }); // Here we test the keys and then each key/value individually because `expect(entries).toEqual(expectedEntries)` - // was taking a very long time for some reason. Probably due to large objects and nested `jasmine.objectContaining`. + // was taking a very long time for some reason. Probably due to large objects and nested `expect.objectContaining`. const entryKeys = Object.keys(entries); expect(entryKeys).toEqual(Object.keys(expectedEntries)); diff --git a/spec/helpers/blob_helper_spec.rb b/spec/helpers/blob_helper_spec.rb index 2bc3933809f..6808ed86c9a 100644 --- a/spec/helpers/blob_helper_spec.rb +++ b/spec/helpers/blob_helper_spec.rb @@ -230,5 +230,18 @@ describe BlobHelper do expect(helper.ide_edit_path(project, "master", "")).to eq("/gitlab/-/ide/project/#{project.namespace.path}/#{project.path}/edit/master") end + + it 'escapes special characters' do + Rails.application.routes.default_url_options[:script_name] = nil + + expect(helper.ide_edit_path(project, "testing/#hashes", "readme.md#test")).to eq("/-/ide/project/#{project.namespace.path}/#{project.path}/edit/testing/#hashes/-/readme.md%23test") + expect(helper.ide_edit_path(project, "testing/#hashes", "src#/readme.md#test")).to eq("/-/ide/project/#{project.namespace.path}/#{project.path}/edit/testing/#hashes/-/src%23/readme.md%23test") + end + + it 'does not escape "/" character' do + Rails.application.routes.default_url_options[:script_name] = nil + + expect(helper.ide_edit_path(project, "testing/slashes", "readme.md/")).to eq("/-/ide/project/#{project.namespace.path}/#{project.path}/edit/testing/slashes/-/readme.md/") + end end end diff --git a/spec/javascripts/ci_variable_list/ci_variable_list_spec.js b/spec/javascripts/ci_variable_list/ci_variable_list_spec.js index 70f49469300..394e60fc22c 100644 --- a/spec/javascripts/ci_variable_list/ci_variable_list_spec.js +++ b/spec/javascripts/ci_variable_list/ci_variable_list_spec.js @@ -127,20 +127,25 @@ describe('VariableList', () => { variableList.init(); }); - it('should add another row when editing the last rows protected checkbox', done => { + it('should not add another row when editing the last rows protected checkbox', done => { const $row = $wrapper.find('.js-row:last-child'); $row.find('.ci-variable-protected-item .js-project-feature-toggle').click(); getSetTimeoutPromise() .then(() => { - expect($wrapper.find('.js-row').length).toBe(2); + expect($wrapper.find('.js-row').length).toBe(1); + }) + .then(done) + .catch(done.fail); + }); - // Check for the correct default in the new row - const $protectedInput = $wrapper - .find('.js-row:last-child') - .find('.js-ci-variable-input-protected'); + it('should not add another row when editing the last rows masked checkbox', done => { + const $row = $wrapper.find('.js-row:last-child'); + $row.find('.ci-variable-masked-item .js-project-feature-toggle').click(); - expect($protectedInput.val()).toBe('false'); + getSetTimeoutPromise() + .then(() => { + expect($wrapper.find('.js-row').length).toBe(1); }) .then(done) .catch(done.fail); diff --git a/spec/javascripts/diffs/components/diff_file_spec.js b/spec/javascripts/diffs/components/diff_file_spec.js index d9b298e84da..ef4589ada48 100644 --- a/spec/javascripts/diffs/components/diff_file_spec.js +++ b/spec/javascripts/diffs/components/diff_file_spec.js @@ -141,18 +141,16 @@ describe('DiffFile', () => { it('should have too large warning and blob link', done => { const BLOB_LINK = '/file/view/path'; vm.file.viewer.error = diffViewerErrors.too_large; + vm.file.viewer.error_message = + 'This source diff could not be displayed because it is too large'; vm.file.view_path = BLOB_LINK; + vm.file.renderIt = true; vm.$nextTick(() => { expect(vm.$el.innerText).toContain( 'This source diff could not be displayed because it is too large', ); - expect(vm.$el.querySelector('.js-too-large-diff')).toBeDefined(); - expect( - vm.$el.querySelector('.js-too-large-diff a').href.indexOf(BLOB_LINK), - ).toBeGreaterThan(-1); - done(); }); }); diff --git a/spec/javascripts/notes/components/note_actions_spec.js b/spec/javascripts/notes/components/note_actions_spec.js index d604e90b529..0cfcc994234 100644 --- a/spec/javascripts/notes/components/note_actions_spec.js +++ b/spec/javascripts/notes/components/note_actions_spec.js @@ -128,87 +128,33 @@ describe('noteActions', () => { }); }); - describe('with feature flag replyToIndividualNotes enabled', () => { + describe('for showReply = true', () => { beforeEach(() => { - gon.features = { - replyToIndividualNotes: true, - }; - }); - - afterEach(() => { - gon.features = {}; - }); - - describe('for showReply = true', () => { - beforeEach(() => { - wrapper = shallowMountNoteActions({ - ...props, - showReply: true, - }); - }); - - it('shows a reply button', () => { - const replyButton = wrapper.find({ ref: 'replyButton' }); - - expect(replyButton.exists()).toBe(true); + wrapper = shallowMountNoteActions({ + ...props, + showReply: true, }); }); - describe('for showReply = false', () => { - beforeEach(() => { - wrapper = shallowMountNoteActions({ - ...props, - showReply: false, - }); - }); - - it('does not show a reply button', () => { - const replyButton = wrapper.find({ ref: 'replyButton' }); + it('shows a reply button', () => { + const replyButton = wrapper.find({ ref: 'replyButton' }); - expect(replyButton.exists()).toBe(false); - }); + expect(replyButton.exists()).toBe(true); }); }); - describe('with feature flag replyToIndividualNotes disabled', () => { + describe('for showReply = false', () => { beforeEach(() => { - gon.features = { - replyToIndividualNotes: false, - }; - }); - - afterEach(() => { - gon.features = {}; - }); - - describe('for showReply = true', () => { - beforeEach(() => { - wrapper = shallowMountNoteActions({ - ...props, - showReply: true, - }); - }); - - it('does not show a reply button', () => { - const replyButton = wrapper.find({ ref: 'replyButton' }); - - expect(replyButton.exists()).toBe(false); + wrapper = shallowMountNoteActions({ + ...props, + showReply: false, }); }); - describe('for showReply = false', () => { - beforeEach(() => { - wrapper = shallowMountNoteActions({ - ...props, - showReply: false, - }); - }); - - it('does not show a reply button', () => { - const replyButton = wrapper.find({ ref: 'replyButton' }); + it('does not show a reply button', () => { + const replyButton = wrapper.find({ ref: 'replyButton' }); - expect(replyButton.exists()).toBe(false); - }); + expect(replyButton.exists()).toBe(false); }); }); }); diff --git a/spec/javascripts/pages/projects/pipeline_schedules/shared/components/timezone_dropdown_spec.js b/spec/javascripts/pages/projects/pipeline_schedules/shared/components/timezone_dropdown_spec.js new file mode 100644 index 00000000000..a89952ee435 --- /dev/null +++ b/spec/javascripts/pages/projects/pipeline_schedules/shared/components/timezone_dropdown_spec.js @@ -0,0 +1,167 @@ +import $ from 'jquery'; +import GLDropdown from '~/gl_dropdown'; // eslint-disable-line no-unused-vars +import TimezoneDropdown, { + formatUtcOffset, + formatTimezone, +} from '~/pages/projects/pipeline_schedules/shared/components/timezone_dropdown'; + +describe('Timezone Dropdown', function() { + preloadFixtures('pipeline_schedules/edit.html'); + + let $inputEl = null; + let $dropdownEl = null; + let $wrapper = null; + const tzListSel = '.dropdown-content ul li a.is-active'; + + describe('Initialize', () => { + describe('with dropdown already loaded', () => { + beforeEach(() => { + loadFixtures('pipeline_schedules/edit.html'); + $wrapper = $('.dropdown'); + $inputEl = $('#schedule_cron_timezone'); + $dropdownEl = $('.js-timezone-dropdown'); + + // eslint-disable-next-line no-new + new TimezoneDropdown({ + $inputEl, + $dropdownEl, + }); + }); + + it('can take an $inputEl in the constructor', () => { + const tzStr = '[UTC + 5.5] Sri Jayawardenepura'; + const tzValue = 'Asia/Colombo'; + + expect($inputEl.val()).toBe('UTC'); + + $(`${tzListSel}:contains('${tzStr}')`, $wrapper).trigger('click'); + + const val = $inputEl.val(); + + expect(val).toBe(tzValue); + expect(val).not.toBe('UTC'); + }); + + it('will format data array of timezones into a list of offsets', () => { + const data = $dropdownEl.data('data'); + const formatted = $wrapper.find(tzListSel).text(); + + data.forEach(item => { + expect(formatted).toContain(formatTimezone(item)); + }); + }); + + it('will default the timezone to UTC', () => { + const tz = $inputEl.val(); + + expect(tz).toBe('UTC'); + }); + }); + + describe('without dropdown loaded', () => { + beforeEach(() => { + loadFixtures('pipeline_schedules/edit.html'); + $wrapper = $('.dropdown'); + $inputEl = $('#schedule_cron_timezone'); + $dropdownEl = $('.js-timezone-dropdown'); + }); + + it('will populate the list of UTC offsets after the dropdown is loaded', () => { + expect($wrapper.find(tzListSel).length).toEqual(0); + + // eslint-disable-next-line no-new + new TimezoneDropdown({ + $inputEl, + $dropdownEl, + }); + + expect($wrapper.find(tzListSel).length).toEqual($($dropdownEl).data('data').length); + }); + + it('will call a provided handler when a new timezone is selected', () => { + const onSelectTimezone = jasmine.createSpy('onSelectTimezoneMock'); + // eslint-disable-next-line no-new + new TimezoneDropdown({ + $inputEl, + $dropdownEl, + onSelectTimezone, + }); + + $wrapper + .find(tzListSel) + .first() + .trigger('click'); + + expect(onSelectTimezone).toHaveBeenCalled(); + }); + }); + }); + + describe('formatUtcOffset', () => { + it('will convert negative utc offsets in seconds to hours and minutes', () => { + expect(formatUtcOffset(-21600)).toEqual('- 6'); + }); + + it('will convert positive utc offsets in seconds to hours and minutes', () => { + expect(formatUtcOffset(25200)).toEqual('+ 7'); + expect(formatUtcOffset(49500)).toEqual('+ 13.75'); + }); + + it('will return 0 when given a string', () => { + expect(formatUtcOffset('BLAH')).toEqual('0'); + expect(formatUtcOffset('$%$%')).toEqual('0'); + }); + + it('will return 0 when given an array', () => { + expect(formatUtcOffset(['an', 'array'])).toEqual('0'); + }); + + it('will return 0 when given an object', () => { + expect(formatUtcOffset({ some: '', object: '' })).toEqual('0'); + }); + + it('will return 0 when given null', () => { + expect(formatUtcOffset(null)).toEqual('0'); + }); + + it('will return 0 when given undefined', () => { + expect(formatUtcOffset(undefined)).toEqual('0'); + }); + + it('will return 0 when given empty input', () => { + expect(formatUtcOffset('')).toEqual('0'); + }); + }); + + describe('formatTimezone', () => { + it('given name: "Chatham Is.", offset: "49500", will format for display as "[UTC + 13.75] Chatham Is."', () => { + expect( + formatTimezone({ + name: 'Chatham Is.', + offset: 49500, + identifier: 'Pacific/Chatham', + }), + ).toEqual('[UTC + 13.75] Chatham Is.'); + }); + + it('given name: "Saskatchewan", offset: "-21600", will format for display as "[UTC - 6] Saskatchewan"', () => { + expect( + formatTimezone({ + name: 'Saskatchewan', + offset: -21600, + identifier: 'America/Regina', + }), + ).toEqual('[UTC - 6] Saskatchewan'); + }); + + it('given name: "Accra", offset: "0", will format for display as "[UTC 0] Accra"', () => { + expect( + formatTimezone({ + name: 'Accra', + offset: 0, + identifier: 'Africa/Accra', + }), + ).toEqual('[UTC 0] Accra'); + }); + }); +}); diff --git a/spec/lib/api/entities/job_request/image_spec.rb b/spec/lib/api/entities/job_request/image_spec.rb new file mode 100644 index 00000000000..092c181ae9c --- /dev/null +++ b/spec/lib/api/entities/job_request/image_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe API::Entities::JobRequest::Image do + let(:ports) { [{ number: 80, protocol: 'http', name: 'name' }]} + let(:image) { double(name: 'image_name', entrypoint: ['foo'], ports: ports)} + let(:entity) { described_class.new(image) } + + subject { entity.as_json } + + it 'returns the image name' do + expect(subject[:name]).to eq 'image_name' + end + + it 'returns the entrypoint' do + expect(subject[:entrypoint]).to eq ['foo'] + end + + it 'returns the ports' do + expect(subject[:ports]).to eq ports + end + + context 'when the ports param is nil' do + let(:ports) { nil } + + it 'does not return the ports' do + expect(subject[:ports]).to be_nil + end + end +end diff --git a/spec/lib/api/entities/job_request/port_spec.rb b/spec/lib/api/entities/job_request/port_spec.rb new file mode 100644 index 00000000000..40ab4cd6231 --- /dev/null +++ b/spec/lib/api/entities/job_request/port_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe ::API::Entities::JobRequest::Port do + let(:port) { double(number: 80, protocol: 'http', name: 'name')} + let(:entity) { described_class.new(port) } + + subject { entity.as_json } + + it 'returns the port number' do + expect(subject[:number]).to eq 80 + end + + it 'returns if the port protocol' do + expect(subject[:protocol]).to eq 'http' + end + + it 'returns the port name' do + expect(subject[:name]).to eq 'name' + end +end diff --git a/spec/lib/gitlab/ci/build/image_spec.rb b/spec/lib/gitlab/ci/build/image_spec.rb index 773a52cdfbc..6e20e0ef5c3 100644 --- a/spec/lib/gitlab/ci/build/image_spec.rb +++ b/spec/lib/gitlab/ci/build/image_spec.rb @@ -18,11 +18,16 @@ describe Gitlab::Ci::Build::Image do it 'populates fabricated object with the proper name attribute' do expect(subject.name).to eq(image_name) end + + it 'does not populate the ports' do + expect(subject.ports).to be_empty + end end context 'when image is defined as hash' do let(:entrypoint) { '/bin/sh' } - let(:job) { create(:ci_build, options: { image: { name: image_name, entrypoint: entrypoint } } ) } + + let(:job) { create(:ci_build, options: { image: { name: image_name, entrypoint: entrypoint, ports: [80] } } ) } it 'fabricates an object of the proper class' do is_expected.to be_kind_of(described_class) @@ -32,6 +37,13 @@ describe Gitlab::Ci::Build::Image do expect(subject.name).to eq(image_name) expect(subject.entrypoint).to eq(entrypoint) end + + it 'populates the ports' do + port = subject.ports.first + expect(port.number).to eq 80 + expect(port.protocol).to eq 'http' + expect(port.name).to eq 'default_port' + end end context 'when image name is empty' do @@ -67,6 +79,10 @@ describe Gitlab::Ci::Build::Image do expect(subject.first).to be_kind_of(described_class) expect(subject.first.name).to eq(service_image_name) end + + it 'does not populate the ports' do + expect(subject.first.ports).to be_empty + end end context 'when service is defined as hash' do @@ -75,7 +91,7 @@ describe Gitlab::Ci::Build::Image do let(:service_command) { 'sleep 30' } let(:job) do create(:ci_build, options: { services: [{ name: service_image_name, entrypoint: service_entrypoint, - alias: service_alias, command: service_command }] }) + alias: service_alias, command: service_command, ports: [80] }] }) end it 'fabricates an non-empty array of objects' do @@ -89,6 +105,11 @@ describe Gitlab::Ci::Build::Image do expect(subject.first.entrypoint).to eq(service_entrypoint) expect(subject.first.alias).to eq(service_alias) expect(subject.first.command).to eq(service_command) + + port = subject.first.ports.first + expect(port.number).to eq 80 + expect(port.protocol).to eq 'http' + expect(port.name).to eq 'default_port' end end diff --git a/spec/lib/gitlab/ci/build/port_spec.rb b/spec/lib/gitlab/ci/build/port_spec.rb new file mode 100644 index 00000000000..1413780dfa6 --- /dev/null +++ b/spec/lib/gitlab/ci/build/port_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::Build::Port do + subject { described_class.new(port) } + + context 'when port is defined as an integer' do + let(:port) { 80 } + + it 'populates the object' do + expect(subject.number).to eq 80 + expect(subject.protocol).to eq described_class::DEFAULT_PORT_PROTOCOL + expect(subject.name).to eq described_class::DEFAULT_PORT_NAME + end + end + + context 'when port is defined as hash' do + let(:port) { { number: 80, protocol: 'https', name: 'port_name' } } + + it 'populates the object' do + expect(subject.number).to eq 80 + expect(subject.protocol).to eq 'https' + expect(subject.name).to eq 'port_name' + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/image_spec.rb b/spec/lib/gitlab/ci/config/entry/image_spec.rb index 1a4d9ed5517..1ebdda398b9 100644 --- a/spec/lib/gitlab/ci/config/entry/image_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/image_spec.rb @@ -35,6 +35,12 @@ describe Gitlab::Ci::Config::Entry::Image do expect(entry.entrypoint).to be_nil end end + + describe '#ports' do + it "returns image's ports" do + expect(entry.ports).to be_nil + end + end end context 'when configuration is a hash' do @@ -69,6 +75,38 @@ describe Gitlab::Ci::Config::Entry::Image do expect(entry.entrypoint).to eq %w(/bin/sh run) end end + + context 'when configuration has ports' do + let(:ports) { [{ number: 80, protocol: 'http', name: 'foobar' }] } + let(:config) { { name: 'ruby:2.2', entrypoint: %w(/bin/sh run), ports: ports } } + let(:entry) { described_class.new(config, { with_image_ports: image_ports }) } + let(:image_ports) { false } + + context 'when with_image_ports metadata is not enabled' do + describe '#valid?' do + it 'is not valid' do + expect(entry).not_to be_valid + expect(entry.errors).to include("image config contains disallowed keys: ports") + end + end + end + + context 'when with_image_ports metadata is enabled' do + let(:image_ports) { true } + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + + describe '#ports' do + it "returns image's ports" do + expect(entry.ports).to eq ports + end + end + end + end end context 'when entry value is not correct' do @@ -76,8 +114,8 @@ describe Gitlab::Ci::Config::Entry::Image do describe '#errors' do it 'saves errors' do - expect(entry.errors) - .to include 'image config should be a hash or a string' + expect(entry.errors.first) + .to match /config should be a hash or a string/ end end @@ -93,8 +131,8 @@ describe Gitlab::Ci::Config::Entry::Image do describe '#errors' do it 'saves errors' do - expect(entry.errors) - .to include 'image config contains unknown keys: non_existing' + expect(entry.errors.first) + .to match /config contains unknown keys: non_existing/ end end diff --git a/spec/lib/gitlab/ci/config/entry/port_spec.rb b/spec/lib/gitlab/ci/config/entry/port_spec.rb new file mode 100644 index 00000000000..5f8f294334e --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/port_spec.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::Config::Entry::Port do + let(:entry) { described_class.new(config) } + + before do + entry.compose! + end + + context 'when configuration is a string' do + let(:config) { 80 } + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + + describe '#value' do + it 'returns valid hash' do + expect(entry.value).to eq(number: 80) + end + end + + describe '#number' do + it "returns port number" do + expect(entry.number).to eq 80 + end + end + + describe '#protocol' do + it "is nil" do + expect(entry.protocol).to be_nil + end + end + + describe '#name' do + it "is nil" do + expect(entry.name).to be_nil + end + end + end + + context 'when configuration is a hash' do + context 'with the complete hash' do + let(:config) do + { number: 80, + protocol: 'http', + name: 'foobar' } + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + + describe '#value' do + it 'returns valid hash' do + expect(entry.value).to eq config + end + end + + describe '#number' do + it "returns port number" do + expect(entry.number).to eq 80 + end + end + + describe '#protocol' do + it "returns port protocol" do + expect(entry.protocol).to eq 'http' + end + end + + describe '#name' do + it "returns port name" do + expect(entry.name).to eq 'foobar' + end + end + end + + context 'with only the port number' do + let(:config) { { number: 80 } } + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + + describe '#value' do + it 'returns valid hash' do + expect(entry.value).to eq(number: 80) + end + end + + describe '#number' do + it "returns port number" do + expect(entry.number).to eq 80 + end + end + + describe '#protocol' do + it "is nil" do + expect(entry.protocol).to be_nil + end + end + + describe '#name' do + it "is nil" do + expect(entry.name).to be_nil + end + end + end + + context 'without the number' do + let(:config) { { protocol: 'http' } } + + describe '#valid?' do + it 'is not valid' do + expect(entry).not_to be_valid + end + end + end + end + + context 'when configuration is invalid' do + let(:config) { '80' } + + describe '#valid?' do + it 'is valid' do + expect(entry).not_to be_valid + end + end + end + + context 'when protocol' do + let(:config) { { number: 80, protocol: protocol, name: 'foobar' } } + + context 'is http' do + let(:protocol) { 'http' } + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'is https' do + let(:protocol) { 'https' } + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'is neither http nor https' do + let(:protocol) { 'foo' } + + describe '#valid?' do + it 'is invalid' do + expect(entry.errors).to include("port protocol should be http or https") + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/ports_spec.rb b/spec/lib/gitlab/ci/config/entry/ports_spec.rb new file mode 100644 index 00000000000..2063bd1d86c --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/ports_spec.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::Config::Entry::Ports do + let(:entry) { described_class.new(config) } + + before do + entry.compose! + end + + context 'when configuration is valid' do + let(:config) { [{ number: 80, protocol: 'http', name: 'foobar' }] } + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + + describe '#value' do + it 'returns valid array' do + expect(entry.value).to eq(config) + end + end + end + + context 'when configuration is invalid' do + let(:config) { 'postgresql:9.5' } + + describe '#valid?' do + it 'is invalid' do + expect(entry).not_to be_valid + end + end + + context 'when any of the ports' do + before do + expect(entry).not_to be_valid + expect(entry.errors.count).to eq 1 + end + + context 'have the same name' do + let(:config) do + [{ number: 80, protocol: 'http', name: 'foobar' }, + { number: 81, protocol: 'http', name: 'foobar' }] + end + + describe '#valid?' do + it 'is invalid' do + expect(entry.errors.first).to match /each port name must be different/ + end + end + end + + context 'have the same port' do + let(:config) do + [{ number: 80, protocol: 'http', name: 'foobar' }, + { number: 80, protocol: 'http', name: 'foobar1' }] + end + + describe '#valid?' do + it 'is invalid' do + expect(entry.errors.first).to match /each port number can only be referenced once/ + end + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/service_spec.rb b/spec/lib/gitlab/ci/config/entry/service_spec.rb index 9ebf947a751..d5bd139b5f1 100644 --- a/spec/lib/gitlab/ci/config/entry/service_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/service_spec.rb @@ -39,6 +39,12 @@ describe Gitlab::Ci::Config::Entry::Service do expect(entry.command).to be_nil end end + + describe '#ports' do + it "returns service's ports" do + expect(entry.ports).to be_nil + end + end end context 'when configuration is a hash' do @@ -81,6 +87,40 @@ describe Gitlab::Ci::Config::Entry::Service do expect(entry.entrypoint).to eq %w(/bin/sh run) end end + + context 'when configuration has ports' do + let(:ports) { [{ number: 80, protocol: 'http', name: 'foobar' }] } + let(:config) do + { name: 'postgresql:9.5', alias: 'db', command: %w(cmd run), entrypoint: %w(/bin/sh run), ports: ports } + end + let(:entry) { described_class.new(config, { with_image_ports: image_ports }) } + let(:image_ports) { false } + + context 'when with_image_ports metadata is not enabled' do + describe '#valid?' do + it 'is not valid' do + expect(entry).not_to be_valid + expect(entry.errors).to include("service config contains disallowed keys: ports") + end + end + end + + context 'when with_image_ports metadata is enabled' do + let(:image_ports) { true } + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + + describe '#ports' do + it "returns image's ports" do + expect(entry.ports).to eq ports + end + end + end + end end context 'when entry value is not correct' do @@ -88,8 +128,8 @@ describe Gitlab::Ci::Config::Entry::Service do describe '#errors' do it 'saves errors' do - expect(entry.errors) - .to include 'service config should be a hash or a string' + expect(entry.errors.first) + .to match /config should be a hash or a string/ end end @@ -105,8 +145,8 @@ describe Gitlab::Ci::Config::Entry::Service do describe '#errors' do it 'saves errors' do - expect(entry.errors) - .to include 'service config contains unknown keys: non_existing' + expect(entry.errors.first) + .to match /config contains unknown keys: non_existing/ end end @@ -116,4 +156,26 @@ describe Gitlab::Ci::Config::Entry::Service do end end end + + context 'when service has ports' do + let(:ports) { [{ number: 80, protocol: 'http', name: 'foobar' }] } + let(:config) do + { name: 'postgresql:9.5', command: %w(cmd run), entrypoint: %w(/bin/sh run), ports: ports } + end + + it 'alias field is mandatory' do + expect(entry).not_to be_valid + expect(entry.errors).to include("service alias can't be blank") + end + end + + context 'when service does not have ports' do + let(:config) do + { name: 'postgresql:9.5', alias: 'db', command: %w(cmd run), entrypoint: %w(/bin/sh run) } + end + + it 'alias field is optional' do + expect(entry).to be_valid + end + end end diff --git a/spec/lib/gitlab/ci/config/entry/services_spec.rb b/spec/lib/gitlab/ci/config/entry/services_spec.rb index 7c4319aee63..d5a1316f665 100644 --- a/spec/lib/gitlab/ci/config/entry/services_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/services_spec.rb @@ -32,4 +32,91 @@ describe Gitlab::Ci::Config::Entry::Services do end end end + + context 'when configuration has ports' do + let(:ports) { [{ number: 80, protocol: 'http', name: 'foobar' }] } + let(:config) { ['postgresql:9.5', { name: 'postgresql:9.1', alias: 'postgres_old', ports: ports }] } + let(:entry) { described_class.new(config, { with_image_ports: image_ports }) } + let(:image_ports) { false } + + context 'when with_image_ports metadata is not enabled' do + describe '#valid?' do + it 'is not valid' do + expect(entry).not_to be_valid + expect(entry.errors).to include("service config contains disallowed keys: ports") + end + end + end + + context 'when with_image_ports metadata is enabled' do + let(:image_ports) { true } + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + + describe '#value' do + it 'returns valid array' do + expect(entry.value).to eq([{ name: 'postgresql:9.5' }, { name: 'postgresql:9.1', alias: 'postgres_old', ports: ports }]) + end + end + + describe 'services alias' do + context 'when they are not unique' do + let(:config) do + ['postgresql:9.5', + { name: 'postgresql:9.1', alias: 'postgres_old', ports: [80] }, + { name: 'ruby', alias: 'postgres_old', ports: [81] }] + end + + describe '#valid?' do + it 'is invalid' do + expect(entry).not_to be_valid + expect(entry.errors).to include("services config alias must be unique in services with ports") + end + end + end + + context 'when they are unique' do + let(:config) do + ['postgresql:9.5', + { name: 'postgresql:9.1', alias: 'postgres_old', ports: [80] }, + { name: 'ruby', alias: 'ruby', ports: [81] }] + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'when one of the duplicated alias is in a service without ports' do + let(:config) do + ['postgresql:9.5', + { name: 'postgresql:9.1', alias: 'postgres_old', ports: [80] }, + { name: 'ruby', alias: 'postgres_old' }] + end + + it 'is valid' do + expect(entry).to be_valid + end + end + + context 'when there are not any ports' do + let(:config) do + ['postgresql:9.5', + { name: 'postgresql:9.1', alias: 'postgres_old' }, + { name: 'ruby', alias: 'postgres_old' }] + end + + it 'is valid' do + expect(entry).to be_valid + end + end + end + end + end end diff --git a/spec/lib/gitlab/ci/config_spec.rb b/spec/lib/gitlab/ci/config_spec.rb index 18f255c1ab7..00b2753c5fc 100644 --- a/spec/lib/gitlab/ci/config_spec.rb +++ b/spec/lib/gitlab/ci/config_spec.rb @@ -123,6 +123,63 @@ describe Gitlab::Ci::Config do ) end end + + context 'when ports have been set' do + context 'in the main image' do + let(:yml) do + <<-EOS + image: + name: ruby:2.2 + ports: + - 80 + EOS + end + + it 'raises an error' do + expect(config.errors).to include("image config contains disallowed keys: ports") + end + end + + context 'in the job image' do + let(:yml) do + <<-EOS + image: ruby:2.2 + + test: + script: rspec + image: + name: ruby:2.2 + ports: + - 80 + EOS + end + + it 'raises an error' do + expect(config.errors).to include("jobs:test:image config contains disallowed keys: ports") + end + end + + context 'in the services' do + let(:yml) do + <<-EOS + image: ruby:2.2 + + test: + script: rspec + image: ruby:2.2 + services: + - name: test + alias: test + ports: + - 80 + EOS + end + + it 'raises an error' do + expect(config.errors).to include("jobs:test:services:service config contains disallowed keys: ports") + end + end + end end context "when using 'include' directive" do diff --git a/spec/lib/gitlab/ci/templates/templates_spec.rb b/spec/lib/gitlab/ci/templates/templates_spec.rb index fbbd58280a9..e20a621168c 100644 --- a/spec/lib/gitlab/ci/templates/templates_spec.rb +++ b/spec/lib/gitlab/ci/templates/templates_spec.rb @@ -4,6 +4,7 @@ require 'spec_helper' describe "CI YML Templates" do ABSTRACT_TEMPLATES = %w[Serverless].freeze + PROJECT_DEPENDENT_TEMPLATES = %w[Auto-DevOps].freeze def self.concrete_templates Gitlab::Template::GitlabCiYmlTemplate.all.reject do |template| @@ -20,7 +21,10 @@ describe "CI YML Templates" do describe 'concrete templates with CI/CD jobs' do concrete_templates.each do |template| it "#{template.name} template should be valid" do - expect { Gitlab::Ci::YamlProcessor.new(template.content) } + # Trigger processing of included files + project = create(:project, :test_repo) if PROJECT_DEPENDENT_TEMPLATES.include?(template.name) + + expect { Gitlab::Ci::YamlProcessor.new(template.content, project: project) } .not_to raise_error end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 29638ef47c5..63a0d54dcfc 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -1233,7 +1233,7 @@ module Gitlab config = YAML.dump({ services: [10, "test"], rspec: { script: "test" } }) expect do Gitlab::Ci::YamlProcessor.new(config) - end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "service config should be a hash or a string") + end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "services:service config should be a hash or a string") end it "returns errors if job services parameter is not an array" do @@ -1247,7 +1247,7 @@ module Gitlab config = YAML.dump({ rspec: { script: "test", services: [10, "test"] } }) expect do Gitlab::Ci::YamlProcessor.new(config) - end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "service config should be a hash or a string") + end.to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, "jobs:rspec:services:service config should be a hash or a string") end it "returns error if job configuration is invalid" do diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 892fdc4e4e9..6f34ef9c1bc 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -805,6 +805,14 @@ describe MergeRequest do expect(merge_request.commits).not_to be_empty expect(merge_request.related_notes.count).to eq(3) end + + it "excludes system notes for commits" do + system_note = create(:note_on_commit, :system, commit_id: merge_request.commits.first.id, + project: merge_request.project) + + expect(merge_request.related_notes.count).to eq(2) + expect(merge_request.related_notes).not_to include(system_note) + end end describe '#for_fork?' do diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 3ccedd8dd06..5fdc7c64030 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -470,11 +470,11 @@ describe API::Runner, :clean_gitlab_redis_shared_state do expect(json_response['token']).to eq(job.token) expect(json_response['job_info']).to eq(expected_job_info) expect(json_response['git_info']).to eq(expected_git_info) - expect(json_response['image']).to eq({ 'name' => 'ruby:2.1', 'entrypoint' => '/bin/sh' }) + expect(json_response['image']).to eq({ 'name' => 'ruby:2.1', 'entrypoint' => '/bin/sh', 'ports' => [] }) expect(json_response['services']).to eq([{ 'name' => 'postgres', 'entrypoint' => nil, - 'alias' => nil, 'command' => nil }, + 'alias' => nil, 'command' => nil, 'ports' => [] }, { 'name' => 'docker:stable-dind', 'entrypoint' => '/bin/sh', - 'alias' => 'docker', 'command' => 'sleep 30' }]) + 'alias' => 'docker', 'command' => 'sleep 30', 'ports' => [] }]) expect(json_response['steps']).to eq(expected_steps) expect(json_response['artifacts']).to eq(expected_artifacts) expect(json_response['cache']).to eq(expected_cache) @@ -853,6 +853,56 @@ describe API::Runner, :clean_gitlab_redis_shared_state do end end + describe 'port support' do + let(:job) { create(:ci_build, pipeline: pipeline, options: options) } + + context 'when job image has ports' do + let(:options) do + { + image: { + name: 'ruby', + ports: [80] + }, + services: ['mysql'] + } + end + + it 'returns the image ports' do + request_job + + expect(response).to have_http_status(:created) + expect(json_response).to include( + 'id' => job.id, + 'image' => a_hash_including('name' => 'ruby', 'ports' => [{ 'number' => 80, 'protocol' => 'http', 'name' => 'default_port' }]), + 'services' => all(a_hash_including('name' => 'mysql'))) + end + end + + context 'when job services settings has ports' do + let(:options) do + { + image: 'ruby', + services: [ + { + name: 'tomcat', + ports: [{ number: 8081, protocol: 'http', name: 'custom_port' }] + } + ] + } + end + + it 'returns the service ports' do + request_job + + expect(response).to have_http_status(:created) + expect(json_response).to include( + 'id' => job.id, + 'image' => a_hash_including('name' => 'ruby'), + 'services' => all(a_hash_including('name' => 'tomcat', 'ports' => [{ 'number' => 8081, 'protocol' => 'http', 'name' => 'custom_port' }]))) + end + end + end + def request_job(token = runner.token, **params) new_params = params.merge(token: token, last_update: last_update) post api('/jobs/request'), params: new_params, headers: { 'User-Agent' => user_agent } diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 24707cd2d41..866d709d446 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -306,6 +306,56 @@ describe Ci::CreatePipelineService do it_behaves_like 'a failed pipeline' end + + context 'when config has ports' do + context 'in the main image' do + let(:ci_yaml) do + <<-EOS + image: + name: ruby:2.2 + ports: + - 80 + EOS + end + + it_behaves_like 'a failed pipeline' + end + + context 'in the job image' do + let(:ci_yaml) do + <<-EOS + image: ruby:2.2 + + test: + script: rspec + image: + name: ruby:2.2 + ports: + - 80 + EOS + end + + it_behaves_like 'a failed pipeline' + end + + context 'in the service' do + let(:ci_yaml) do + <<-EOS + image: ruby:2.2 + + test: + script: rspec + image: ruby:2.2 + services: + - name: test + ports: + - 80 + EOS + end + + it_behaves_like 'a failed pipeline' + end + end end context 'when commit contains a [ci skip] directive' do diff --git a/spec/services/git/tag_push_service_spec.rb b/spec/services/git/tag_push_service_spec.rb index e151db5827f..2d960fc9f08 100644 --- a/spec/services/git/tag_push_service_spec.rb +++ b/spec/services/git/tag_push_service_spec.rb @@ -31,6 +31,20 @@ describe Git::TagPushService do end end + describe 'System Hooks' do + let!(:push_data) { service.tap(&:execute).push_data } + + it "executes system hooks after pushing a tag" do + expect_next_instance_of(SystemHooksService) do |system_hooks_service| + expect(system_hooks_service) + .to receive(:execute_hooks) + .with(push_data, :tag_push_hooks) + end + + service.execute + end + end + describe "Pipelines" do subject { service.execute } diff --git a/spec/services/notes/build_service_spec.rb b/spec/services/notes/build_service_spec.rb index af4daff336b..96fff20f7fb 100644 --- a/spec/services/notes/build_service_spec.rb +++ b/spec/services/notes/build_service_spec.rb @@ -128,37 +128,19 @@ describe Notes::BuildService do subject { described_class.new(project, author, note: 'Test', in_reply_to_discussion_id: note.discussion_id).execute } - shared_examples 'an individual note reply' do - it 'builds another individual note' do - expect(subject).to be_valid - expect(subject).to be_a(Note) - expect(subject.discussion_id).not_to eq(note.discussion_id) - end + it 'sets the note up to be in reply to that note' do + expect(subject).to be_valid + expect(subject).to be_a(DiscussionNote) + expect(subject.discussion_id).to eq(note.discussion_id) end - context 'when reply_to_individual_notes is disabled' do - before do - stub_feature_flags(reply_to_individual_notes: false) - end - - it_behaves_like 'an individual note reply' - end + context 'when noteable does not support replies' do + let(:note) { create(:note_on_commit) } - context 'when reply_to_individual_notes is enabled' do - before do - stub_feature_flags(reply_to_individual_notes: true) - end - - it 'sets the note up to be in reply to that note' do + it 'builds another individual note' do expect(subject).to be_valid - expect(subject).to be_a(DiscussionNote) - expect(subject.discussion_id).to eq(note.discussion_id) - end - - context 'when noteable does not support replies' do - let(:note) { create(:note_on_commit) } - - it_behaves_like 'an individual note reply' + expect(subject).to be_a(Note) + expect(subject.discussion_id).not_to eq(note.discussion_id) end end end diff --git a/spec/services/notes/create_service_spec.rb b/spec/services/notes/create_service_spec.rb index 8d8e81173ff..bcbb8950910 100644 --- a/spec/services/notes/create_service_spec.rb +++ b/spec/services/notes/create_service_spec.rb @@ -298,41 +298,20 @@ describe Notes::CreateService do subject { described_class.new(project, user, reply_opts).execute } - context 'when reply_to_individual_notes is disabled' do - before do - stub_feature_flags(reply_to_individual_notes: false) - end - - it 'creates an individual note' do - expect(subject.type).to eq(nil) - expect(subject.discussion_id).not_to eq(existing_note.discussion_id) - end - - it 'does not convert existing note' do - expect { subject }.not_to change { existing_note.reload.type } - end + it 'creates a DiscussionNote in reply to existing note' do + expect(subject).to be_a(DiscussionNote) + expect(subject.discussion_id).to eq(existing_note.discussion_id) end - context 'when reply_to_individual_notes is enabled' do - before do - stub_feature_flags(reply_to_individual_notes: true) - end - - it 'creates a DiscussionNote in reply to existing note' do - expect(subject).to be_a(DiscussionNote) - expect(subject.discussion_id).to eq(existing_note.discussion_id) - end - - it 'converts existing note to DiscussionNote' do - expect do - existing_note + it 'converts existing note to DiscussionNote' do + expect do + existing_note - Timecop.freeze(Time.now + 1.minute) { subject } + Timecop.freeze(Time.now + 1.minute) { subject } - existing_note.reload - end.to change { existing_note.type }.from(nil).to('DiscussionNote') - .and change { existing_note.updated_at } - end + existing_note.reload + end.to change { existing_note.type }.from(nil).to('DiscussionNote') + .and change { existing_note.updated_at } end end end diff --git a/spec/services/quick_actions/interpret_service_spec.rb b/spec/services/quick_actions/interpret_service_spec.rb index 8b0f9c8ade2..c7e5cca324f 100644 --- a/spec/services/quick_actions/interpret_service_spec.rb +++ b/spec/services/quick_actions/interpret_service_spec.rb @@ -10,6 +10,7 @@ describe QuickActions::InterpretService do let(:milestone) { create(:milestone, project: project, title: '9.10') } let(:commit) { create(:commit, project: project) } let(:inprogress) { create(:label, project: project, title: 'In Progress') } + let(:helmchart) { create(:label, project: project, title: 'Helm Chart Registry') } let(:bug) { create(:label, project: project, title: 'Bug') } let(:note) { build(:note, commit_id: merge_request.diff_head_sha) } let(:service) { described_class.new(project, developer) } @@ -94,6 +95,26 @@ describe QuickActions::InterpretService do end end + shared_examples 'multiword label name starting without ~' do + it 'fetches label ids and populates add_label_ids if content contains /label' do + helmchart # populate the label + _, updates = service.execute(content, issuable) + + expect(updates).to eq(add_label_ids: [helmchart.id]) + end + end + + shared_examples 'label name is included in the middle of another label name' do + it 'ignores the sublabel when the content contains the includer label name' do + helmchart # populate the label + create(:label, project: project, title: 'Chart') + + _, updates = service.execute(content, issuable) + + expect(updates).to eq(add_label_ids: [helmchart.id]) + end + end + shared_examples 'unlabel command' do it 'fetches label ids and populates remove_label_ids if content contains /unlabel' do issuable.update!(label_ids: [inprogress.id]) # populate the label @@ -624,6 +645,26 @@ describe QuickActions::InterpretService do let(:issuable) { issue } end + it_behaves_like 'multiword label name starting without ~' do + let(:content) { %(/label "#{helmchart.title}") } + let(:issuable) { issue } + end + + it_behaves_like 'multiword label name starting without ~' do + let(:content) { %(/label "#{helmchart.title}") } + let(:issuable) { merge_request } + end + + it_behaves_like 'label name is included in the middle of another label name' do + let(:content) { %(/label ~"#{helmchart.title}") } + let(:issuable) { issue } + end + + it_behaves_like 'label name is included in the middle of another label name' do + let(:content) { %(/label ~"#{helmchart.title}") } + let(:issuable) { merge_request } + end + it_behaves_like 'unlabel command' do let(:content) { %(/unlabel ~"#{inprogress.title}") } let(:issuable) { issue } diff --git a/spec/support/features/variable_list_shared_examples.rb b/spec/support/features/variable_list_shared_examples.rb index 73156d18c1b..693b796fbdc 100644 --- a/spec/support/features/variable_list_shared_examples.rb +++ b/spec/support/features/variable_list_shared_examples.rb @@ -23,10 +23,13 @@ shared_examples 'variable list' do end end - it 'adds empty variable' do + it 'adds a new protected variable' do page.within('.js-ci-variable-list-section .js-row:last-child') do find('.js-ci-variable-input-key').set('key') - find('.js-ci-variable-input-value').set('') + find('.js-ci-variable-input-value').set('key_value') + find('.ci-variable-protected-item .js-project-feature-toggle').click + + expect(find('.js-ci-variable-input-protected', visible: false).value).to eq('true') end click_button('Save variables') @@ -37,17 +40,17 @@ shared_examples 'variable list' do # We check the first row because it re-sorts to alphabetical order on refresh page.within('.js-ci-variable-list-section .js-row:nth-child(1)') do expect(find('.js-ci-variable-input-key').value).to eq('key') - expect(find('.js-ci-variable-input-value', visible: false).value).to eq('') + expect(find('.js-ci-variable-input-value', visible: false).value).to eq('key_value') + expect(find('.js-ci-variable-input-protected', visible: false).value).to eq('true') end end - it 'adds new protected variable' do + it 'defaults to masked' do page.within('.js-ci-variable-list-section .js-row:last-child') do find('.js-ci-variable-input-key').set('key') find('.js-ci-variable-input-value').set('key_value') - find('.ci-variable-protected-item .js-project-feature-toggle').click - expect(find('.js-ci-variable-input-protected', visible: false).value).to eq('true') + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('true') end click_button('Save variables') @@ -59,7 +62,7 @@ shared_examples 'variable list' do page.within('.js-ci-variable-list-section .js-row:nth-child(1)') do expect(find('.js-ci-variable-input-key').value).to eq('key') expect(find('.js-ci-variable-input-value', visible: false).value).to eq('key_value') - expect(find('.js-ci-variable-input-protected', visible: false).value).to eq('true') + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('true') end end @@ -163,27 +166,6 @@ shared_examples 'variable list' do end end - it 'edits variable with empty value' do - page.within('.js-ci-variable-list-section') do - click_button('Reveal value') - - page.within('.js-row:nth-child(1)') do - find('.js-ci-variable-input-key').set('new_key') - find('.js-ci-variable-input-value').set('') - end - - click_button('Save variables') - wait_for_requests - - visit page_path - - page.within('.js-row:nth-child(1)') do - expect(find('.js-ci-variable-input-key').value).to eq('new_key') - expect(find('.js-ci-variable-input-value', visible: false).value).to eq('') - end - end - end - it 'edits variable to be protected' do # Create the unprotected variable page.within('.js-ci-variable-list-section .js-row:last-child') do @@ -251,6 +233,57 @@ shared_examples 'variable list' do end end + it 'edits variable to be unmasked' do + page.within('.js-ci-variable-list-section .js-row:nth-child(1)') do + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('true') + + find('.ci-variable-masked-item .js-project-feature-toggle').click + + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('false') + end + + click_button('Save variables') + wait_for_requests + + visit page_path + + page.within('.js-ci-variable-list-section .js-row:nth-child(1)') do + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('false') + end + end + + it 'edits variable to be masked' do + page.within('.js-ci-variable-list-section .js-row:nth-child(1)') do + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('true') + + find('.ci-variable-masked-item .js-project-feature-toggle').click + + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('false') + end + + click_button('Save variables') + wait_for_requests + + visit page_path + + page.within('.js-ci-variable-list-section .js-row:nth-child(1)') do + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('false') + + find('.ci-variable-masked-item .js-project-feature-toggle').click + + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('true') + end + + click_button('Save variables') + wait_for_requests + + visit page_path + + page.within('.js-ci-variable-list-section .js-row:nth-child(1)') do + expect(find('.js-ci-variable-input-masked', visible: false).value).to eq('true') + end + end + it 'handles multiple edits and deletion in the middle' do page.within('.js-ci-variable-list-section') do # Create 2 variables @@ -297,11 +330,11 @@ shared_examples 'variable list' do it 'shows validation error box about duplicate keys' do page.within('.js-ci-variable-list-section .js-row:last-child') do find('.js-ci-variable-input-key').set('samekey') - find('.js-ci-variable-input-value').set('value1') + find('.js-ci-variable-input-value').set('value123') end page.within('.js-ci-variable-list-section .js-row:last-child') do find('.js-ci-variable-input-key').set('samekey') - find('.js-ci-variable-input-value').set('value2') + find('.js-ci-variable-input-value').set('value456') end click_button('Save variables') @@ -314,4 +347,34 @@ shared_examples 'variable list' do expect(find('.js-ci-variable-error-box')).to have_content(/Validation failed Variables have duplicate values \(.+\)/) end end + + it 'shows validation error box about empty values' do + page.within('.js-ci-variable-list-section .js-row:last-child') do + find('.js-ci-variable-input-key').set('empty_value') + find('.js-ci-variable-input-value').set('') + end + + click_button('Save variables') + wait_for_requests + + page.within('.js-ci-variable-list-section') do + expect(all('.js-ci-variable-error-box ul li').count).to eq(1) + expect(find('.js-ci-variable-error-box')).to have_content(/Validation failed Variables value is invalid/) + end + end + + it 'shows validation error box about unmaskable values' do + page.within('.js-ci-variable-list-section .js-row:last-child') do + find('.js-ci-variable-input-key').set('unmaskable_value') + find('.js-ci-variable-input-value').set('???') + end + + click_button('Save variables') + wait_for_requests + + page.within('.js-ci-variable-list-section') do + expect(all('.js-ci-variable-error-box ul li').count).to eq(1) + expect(find('.js-ci-variable-error-box')).to have_content(/Validation failed Variables value is invalid/) + end + end end diff --git a/spec/support/shared_context/policies/project_policy_shared_context.rb b/spec/support/shared_context/policies/project_policy_shared_context.rb index 8bcd26ec0cd..3ad6e067674 100644 --- a/spec/support/shared_context/policies/project_policy_shared_context.rb +++ b/spec/support/shared_context/policies/project_policy_shared_context.rb @@ -15,7 +15,7 @@ RSpec.shared_context 'ProjectPolicy context' do read_project_for_iids read_issue_iid read_label read_milestone read_project_snippet read_project_member read_note create_project create_issue create_note upload_file create_merge_request_in - award_emoji read_release + award_emoji ] end @@ -24,7 +24,7 @@ RSpec.shared_context 'ProjectPolicy context' do download_code fork_project create_project_snippet update_issue admin_issue admin_label admin_list read_commit_status read_build read_container_image read_pipeline read_environment read_deployment - read_merge_request download_wiki_code read_sentry_issue + read_merge_request download_wiki_code read_sentry_issue read_release ] end diff --git a/spec/support/shared_examples/time_tracking_shared_examples.rb b/spec/support/shared_examples/time_tracking_shared_examples.rb deleted file mode 100644 index 909d4e2ee8d..00000000000 --- a/spec/support/shared_examples/time_tracking_shared_examples.rb +++ /dev/null @@ -1,85 +0,0 @@ -shared_examples 'issuable time tracker' do - it 'renders the sidebar component empty state' do - page.within '.time-tracking-no-tracking-pane' do - expect(page).to have_content 'No estimate or time spent' - end - end - - it 'updates the sidebar component when estimate is added' do - submit_time('/estimate 3w 1d 1h') - - wait_for_requests - page.within '.time-tracking-estimate-only-pane' do - expect(page).to have_content '3w 1d 1h' - end - end - - it 'updates the sidebar component when spent is added' do - submit_time('/spend 3w 1d 1h') - - wait_for_requests - page.within '.time-tracking-spend-only-pane' do - expect(page).to have_content '3w 1d 1h' - end - end - - it 'shows the comparison when estimate and spent are added' do - submit_time('/estimate 3w 1d 1h') - submit_time('/spend 3w 1d 1h') - - wait_for_requests - page.within '.time-tracking-comparison-pane' do - expect(page).to have_content '3w 1d 1h' - end - end - - it 'updates the sidebar component when estimate is removed' do - submit_time('/estimate 3w 1d 1h') - submit_time('/remove_estimate') - - page.within '.time-tracking-component-wrap' do - expect(page).to have_content 'No estimate or time spent' - end - end - - it 'updates the sidebar component when spent is removed' do - submit_time('/spend 3w 1d 1h') - submit_time('/remove_time_spent') - - page.within '.time-tracking-component-wrap' do - expect(page).to have_content 'No estimate or time spent' - end - end - - it 'shows the help state when icon is clicked' do - page.within '.time-tracking-component-wrap' do - find('.help-button').click - expect(page).to have_content 'Track time with quick actions' - expect(page).to have_content 'Learn more' - end - end - - it 'hides the help state when close icon is clicked' do - page.within '.time-tracking-component-wrap' do - find('.help-button').click - find('.close-help-button').click - - expect(page).not_to have_content 'Track time with quick actions' - expect(page).not_to have_content 'Learn more' - end - end - - it 'displays the correct help url' do - page.within '.time-tracking-component-wrap' do - find('.help-button').click - - expect(find_link('Learn more')[:href]).to have_content('/help/workflow/time_tracking.md') - end - end -end - -def submit_time(quick_action) - fill_in 'note[note]', with: quick_action - find('.js-comment-submit-button').click - wait_for_requests -end diff --git a/yarn.lock b/yarn.lock index 581d5818ccc..15fa876300b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2804,7 +2804,7 @@ d3-dsv@1, d3-dsv@1.0.8: iconv-lite "0.4" rw "1" -d3-ease@1, d3-ease@1.0.3: +d3-ease@1, d3-ease@1.0.3, d3-ease@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/d3-ease/-/d3-ease-1.0.3.tgz#68bfbc349338a380c44d8acc4fbc3304aa2d8c0e" integrity sha1-aL+8NJM4o4DETYrMT7wzBKotjA4= @@ -2920,7 +2920,7 @@ d3-timer@1, d3-timer@1.0.7: resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-1.0.7.tgz#df9650ca587f6c96607ff4e60cc38229e8dd8531" integrity sha512-vMZXR88XujmG/L5oB96NNKH5lCWwiLM/S2HyyAQLcjWJCloK5shxta4CwOFYLZoY3AWX73v8Lgv4cCAdWtRmOA== -d3-transition@1, d3-transition@1.1.1: +d3-transition@1, d3-transition@1.1.1, d3-transition@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/d3-transition/-/d3-transition-1.1.1.tgz#d8ef89c3b848735b060e54a39b32aaebaa421039" integrity sha512-xeg8oggyQ+y5eb4J13iDgKIjUcEfIOZs2BqV/eEmXm2twx80wTzJ4tB4vaZ5BKfz7XsI/DFmQL5me6O27/5ykQ== |