diff options
44 files changed, 1176 insertions, 338 deletions
diff --git a/.gitlab/issue_templates/QA failure.md b/.gitlab/issue_templates/QA failure.md index 13b5d7bf92c..e1b3eec5d29 100644 --- a/.gitlab/issue_templates/QA failure.md +++ b/.gitlab/issue_templates/QA failure.md @@ -40,7 +40,10 @@ Attach the screenshot and HTML snapshot of the page from the job's artifacts: /due in 2 weeks <!-- Base labels. --> -/label ~Quality ~QA ~bug ~S1 +/label ~Quality ~QA ~test + +<!-- Test failure type label, please use just one.--> +/label ~"failure::broken-test" ~"failure::flaky-test" ~"failure::stale-test" ~"failure::test-environment" ~"failure::investigating" <!-- Choose the stage that appears in the test path, e.g. ~"devops::create" for diff --git a/app/assets/javascripts/blob/pdf/index.js b/app/assets/javascripts/blob/pdf/index.js index 19778d07983..218987585b4 100644 --- a/app/assets/javascripts/blob/pdf/index.js +++ b/app/assets/javascripts/blob/pdf/index.js @@ -1,57 +1,17 @@ import Vue from 'vue'; -import pdfLab from '../../pdf/index.vue'; -import { GlLoadingIcon } from '@gitlab/ui'; +import PdfViewer from './pdf_viewer.vue'; export default () => { const el = document.getElementById('js-pdf-viewer'); return new Vue({ el, - components: { - pdfLab, - GlLoadingIcon, + render(createElement) { + return createElement(PdfViewer, { + props: { + pdf: el.dataset.endpoint, + }, + }); }, - data() { - return { - error: false, - loadError: false, - loading: true, - pdf: el.dataset.endpoint, - }; - }, - methods: { - onLoad() { - this.loading = false; - }, - onError(error) { - this.loading = false; - this.loadError = true; - this.error = error; - }, - }, - template: ` - <div class="js-pdf-viewer container-fluid md prepend-top-default append-bottom-default"> - <div - class="text-center loading" - v-if="loading && !error"> - <gl-loading-icon class="mt-5" size="lg"/> - </div> - <pdf-lab - v-if="!loadError" - :pdf="pdf" - @pdflabload="onLoad" - @pdflaberror="onError" /> - <p - class="text-center" - v-if="error"> - <span v-if="loadError"> - An error occurred while loading the file. Please try again later. - </span> - <span v-else> - An error occurred while decoding the file. - </span> - </p> - </div> - `, }); }; diff --git a/app/assets/javascripts/blob/pdf/pdf_viewer.vue b/app/assets/javascripts/blob/pdf/pdf_viewer.vue new file mode 100644 index 00000000000..5eaddfc099a --- /dev/null +++ b/app/assets/javascripts/blob/pdf/pdf_viewer.vue @@ -0,0 +1,49 @@ +<script> +import PdfLab from '../../pdf/index.vue'; +import { GlLoadingIcon } from '@gitlab/ui'; + +export default { + components: { + PdfLab, + GlLoadingIcon, + }, + props: { + pdf: { + type: String, + required: true, + }, + }, + data() { + return { + error: false, + loadError: false, + loading: true, + }; + }, + methods: { + onLoad() { + this.loading = false; + }, + onError(error) { + this.loading = false; + this.loadError = true; + this.error = error; + }, + }, +}; +</script> + +<template> + <div class="js-pdf-viewer container-fluid md prepend-top-default append-bottom-default"> + <div v-if="loading && !error" class="text-center loading"> + <gl-loading-icon class="mt-5" size="lg" /> + </div> + <pdf-lab v-if="!loadError" :pdf="pdf" @pdflabload="onLoad" @pdflaberror="onError" /> + <p v-if="error" class="text-center"> + <span v-if="loadError" ref="loadError"> + {{ __('An error occurred while loading the file. Please try again later.') }} + </span> + <span v-else>{{ __('An error occurred while decoding the file.') }}</span> + </p> + </div> +</template> diff --git a/app/assets/javascripts/lib/utils/common_utils.js b/app/assets/javascripts/lib/utils/common_utils.js index dd5a52fe1ce..abecfba5718 100644 --- a/app/assets/javascripts/lib/utils/common_utils.js +++ b/app/assets/javascripts/lib/utils/common_utils.js @@ -8,6 +8,7 @@ import axios from './axios_utils'; import { getLocationHash } from './url_utility'; import { convertToCamelCase, convertToSnakeCase } from './text_utility'; import { isObject } from './type_utility'; +import { isFunction } from 'lodash'; export const getPagePath = (index = 0) => { const page = $('body').attr('data-page') || ''; @@ -667,30 +668,34 @@ export const spriteIcon = (icon, className = '') => { }; /** - * This method takes in object with snake_case property names - * and returns a new object with camelCase property names - * - * Reasoning for this method is to ensure consistent property - * naming conventions across JS code. + * @callback ConversionFunction + * @param {string} prop + */ + +/** + * This function takes a conversion function as the first parameter + * and applies this function to each prop in the provided object. * * This method also supports additional params in `options` object * + * @param {ConversionFunction} conversionFunction - Function to apply to each prop of the object. * @param {Object} obj - Object to be converted. * @param {Object} options - Object containing additional options. * @param {boolean} options.deep - FLag to allow deep object converting - * @param {Array[]} dropKeys - List of properties to discard while building new object - * @param {Array[]} ignoreKeyNames - List of properties to leave intact (as snake_case) while building new object + * @param {Array[]} options.dropKeys - List of properties to discard while building new object + * @param {Array[]} options.ignoreKeyNames - List of properties to leave intact (as snake_case) while building new object */ -export const convertObjectPropsToCamelCase = (obj = {}, options = {}) => { - if (obj === null) { +export const convertObjectProps = (conversionFunction, obj = {}, options = {}) => { + if (!isFunction(conversionFunction) || obj === null) { return {}; } - const initial = Array.isArray(obj) ? [] : {}; const { deep = false, dropKeys = [], ignoreKeyNames = [] } = options; + const isObjParameterArray = Array.isArray(obj); + const initialValue = isObjParameterArray ? [] : {}; + return Object.keys(obj).reduce((acc, prop) => { - const result = acc; const val = obj[prop]; // Drop properties from new object if @@ -702,34 +707,54 @@ export const convertObjectPropsToCamelCase = (obj = {}, options = {}) => { // Skip converting properties in new object // if there are any mentioned in options if (ignoreKeyNames.indexOf(prop) > -1) { - result[prop] = obj[prop]; + acc[prop] = val; return acc; } if (deep && (isObject(val) || Array.isArray(val))) { - result[convertToCamelCase(prop)] = convertObjectPropsToCamelCase(val, options); + if (isObjParameterArray) { + acc[prop] = convertObjectProps(conversionFunction, val, options); + } else { + acc[conversionFunction(prop)] = convertObjectProps(conversionFunction, val, options); + } } else { - result[convertToCamelCase(prop)] = obj[prop]; + acc[conversionFunction(prop)] = val; } return acc; - }, initial); + }, initialValue); }; /** + * This method takes in object with snake_case property names + * and returns a new object with camelCase property names + * + * Reasoning for this method is to ensure consistent property + * naming conventions across JS code. + * + * This method also supports additional params in `options` object + * + * @param {Object} obj - Object to be converted. + * @param {Object} options - Object containing additional options. + * @param {boolean} options.deep - FLag to allow deep object converting + * @param {Array[]} options.dropKeys - List of properties to discard while building new object + * @param {Array[]} options.ignoreKeyNames - List of properties to leave intact (as snake_case) while building new object + */ +export const convertObjectPropsToCamelCase = (obj = {}, options = {}) => + convertObjectProps(convertToCamelCase, obj, options); + +/** * Converts all the object keys to snake case * - * @param {Object} obj Object to transform - * @returns {Object} + * This method also supports additional params in `options` object + * + * @param {Object} obj - Object to be converted. + * @param {Object} options - Object containing additional options. + * @param {boolean} options.deep - FLag to allow deep object converting + * @param {Array[]} options.dropKeys - List of properties to discard while building new object + * @param {Array[]} options.ignoreKeyNames - List of properties to leave intact (as snake_case) while building new object */ -// Follow up to add additional options param: -// https://gitlab.com/gitlab-org/gitlab/issues/39173 -export const convertObjectPropsToSnakeCase = (obj = {}) => - obj - ? Object.entries(obj).reduce( - (acc, [key, value]) => ({ ...acc, [convertToSnakeCase(key)]: value }), - {}, - ) - : {}; +export const convertObjectPropsToSnakeCase = (obj = {}, options = {}) => + convertObjectProps(convertToSnakeCase, obj, options); export const imagePath = imgUrl => `${gon.asset_host || ''}${gon.relative_url_root || ''}/assets/${imgUrl}`; diff --git a/app/controllers/groups/settings/ci_cd_controller.rb b/app/controllers/groups/settings/ci_cd_controller.rb index ffa3f2c3364..3d347429398 100644 --- a/app/controllers/groups/settings/ci_cd_controller.rb +++ b/app/controllers/groups/settings/ci_cd_controller.rb @@ -7,7 +7,7 @@ module Groups before_action :authorize_admin_group! before_action :authorize_update_max_artifacts_size!, only: [:update] before_action do - push_frontend_feature_flag(:new_variables_ui, @group) + push_frontend_feature_flag(:new_variables_ui, @group, default_enabled: true) end before_action :define_variables, only: [:show, :create_deploy_token] diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index bea24d2b204..af185887a8c 100644 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -21,7 +21,7 @@ class Projects::MergeRequestsController < Projects::MergeRequests::ApplicationCo before_action only: [:show] do push_frontend_feature_flag(:diffs_batch_load, @project, default_enabled: true) push_frontend_feature_flag(:deploy_from_footer, @project, default_enabled: true) - push_frontend_feature_flag(:single_mr_diff_view, @project) + push_frontend_feature_flag(:single_mr_diff_view, @project, default_enabled: true) push_frontend_feature_flag(:suggest_pipeline) if experiment_enabled?(:suggest_pipeline) end diff --git a/app/controllers/projects/settings/ci_cd_controller.rb b/app/controllers/projects/settings/ci_cd_controller.rb index aac6ecb07e4..43c798bfc6e 100644 --- a/app/controllers/projects/settings/ci_cd_controller.rb +++ b/app/controllers/projects/settings/ci_cd_controller.rb @@ -6,7 +6,7 @@ module Projects before_action :authorize_admin_pipeline! before_action :define_variables before_action do - push_frontend_feature_flag(:new_variables_ui, @project) + push_frontend_feature_flag(:new_variables_ui, @project, default_enabled: true) end def show diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index f9a5f713814..38730357593 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -28,7 +28,8 @@ module Ci license_scanning: 'gl-license-scanning-report.json', performance: 'performance.json', metrics: 'metrics.txt', - lsif: 'lsif.json' + lsif: 'lsif.json', + dotenv: '.env' }.freeze INTERNAL_TYPES = { @@ -43,6 +44,7 @@ module Ci metrics_referee: :gzip, network_referee: :gzip, lsif: :gzip, + dotenv: :gzip, # All these file formats use `raw` as we need to store them uncompressed # for Frontend to fetch the files and do analysis @@ -118,7 +120,8 @@ module Ci metrics: 12, ## EE-specific metrics_referee: 13, ## runner referees network_referee: 14, ## runner referees - lsif: 15 # LSIF data for code navigation + lsif: 15, # LSIF data for code navigation + dotenv: 16 } enum file_format: { diff --git a/app/models/ci/job_variable.rb b/app/models/ci/job_variable.rb index 862a0bc1299..f2968c037c7 100644 --- a/app/models/ci/job_variable.rb +++ b/app/models/ci/job_variable.rb @@ -4,11 +4,14 @@ module Ci class JobVariable < ApplicationRecord extend Gitlab::Ci::Model include NewHasVariable + include BulkInsertSafe belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id alias_attribute :secret_value, :value - validates :key, uniqueness: { scope: :job_id } + validates :key, uniqueness: { scope: :job_id }, unless: :dotenv_source? + + enum source: { internal: 0, dotenv: 1 }, _suffix: true end end diff --git a/app/serializers/diff_file_entity.rb b/app/serializers/diff_file_entity.rb index c3826692c52..45c16aabe9e 100644 --- a/app/serializers/diff_file_entity.rb +++ b/app/serializers/diff_file_entity.rb @@ -67,14 +67,14 @@ class DiffFileEntity < DiffFileBaseEntity private def parallel_diff_view?(options, diff_file) - return true unless Feature.enabled?(:single_mr_diff_view, diff_file.repository.project) + return true unless Feature.enabled?(:single_mr_diff_view, diff_file.repository.project, default_enabled: true) # If we're not rendering inline, we must be rendering parallel !inline_diff_view?(options, diff_file) end def inline_diff_view?(options, diff_file) - return true unless Feature.enabled?(:single_mr_diff_view, diff_file.repository.project) + return true unless Feature.enabled?(:single_mr_diff_view, diff_file.repository.project, default_enabled: true) # If nothing is present, inline will be the default. options.fetch(:diff_view, :inline).to_sym == :inline diff --git a/app/services/ci/create_job_artifacts_service.rb b/app/services/ci/create_job_artifacts_service.rb index 3aa2b16bc73..d207c215618 100644 --- a/app/services/ci/create_job_artifacts_service.rb +++ b/app/services/ci/create_job_artifacts_service.rb @@ -10,10 +10,24 @@ module Ci ].freeze def execute(job, artifacts_file, params, metadata_file: nil) + return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file) + + artifact, artifact_metadata = build_artifact(job, artifacts_file, params, metadata_file) + result = parse_artifact(job, artifact) + + return result unless result[:status] == :success + + persist_artifact(job, artifact, artifact_metadata) + end + + private + + def build_artifact(job, artifacts_file, params, metadata_file) expire_in = params['expire_in'] || Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in - job.job_artifacts.build( + artifact = Ci::JobArtifact.new( + job_id: job.id, project: job.project, file: artifacts_file, file_type: params['artifact_type'], @@ -21,34 +35,51 @@ module Ci file_sha256: artifacts_file.sha256, expire_in: expire_in) - if metadata_file - job.job_artifacts.build( - project: job.project, - file: metadata_file, - file_type: :metadata, - file_format: :gzip, - file_sha256: metadata_file.sha256, - expire_in: expire_in) + artifact_metadata = if metadata_file + Ci::JobArtifact.new( + job_id: job.id, + project: job.project, + file: metadata_file, + file_type: :metadata, + file_format: :gzip, + file_sha256: metadata_file.sha256, + expire_in: expire_in) + end + + [artifact, artifact_metadata] + end + + def parse_artifact(job, artifact) + unless Feature.enabled?(:ci_synchronous_artifact_parsing, job.project, default_enabled: true) + return success end - if job.update(artifacts_expire_in: expire_in) - success - else - error(job.errors.messages, :bad_request) + case artifact.file_type + when 'dotenv' then parse_dotenv_artifact(job, artifact) + else success end + end - rescue ActiveRecord::RecordNotUnique => error - return success if sha256_matches_existing_artifact?(job, params['artifact_type'], artifacts_file) + def persist_artifact(job, artifact, artifact_metadata) + Ci::JobArtifact.transaction do + artifact.save! + artifact_metadata&.save! + + # NOTE: The `artifacts_expire_at` column is already deprecated and to be removed in the near future. + job.update_column(:artifacts_expire_at, artifact.expire_at) + end + success + rescue ActiveRecord::RecordNotUnique => error track_exception(error, job, params) error('another artifact of the same type already exists', :bad_request) rescue *OBJECT_STORAGE_ERRORS => error track_exception(error, job, params) error(error.message, :service_unavailable) + rescue => error + error(error.message, :bad_request) end - private - def sha256_matches_existing_artifact?(job, artifact_type, artifacts_file) existing_artifact = job.job_artifacts.find_by_file_type(artifact_type) return false unless existing_artifact @@ -63,5 +94,9 @@ module Ci uploading_type: params['artifact_type'] ) end + + def parse_dotenv_artifact(job, artifact) + Ci::ParseDotenvArtifactService.new(job.project, current_user).execute(artifact) + end end end diff --git a/app/services/ci/parse_dotenv_artifact_service.rb b/app/services/ci/parse_dotenv_artifact_service.rb new file mode 100644 index 00000000000..fcbdc94c097 --- /dev/null +++ b/app/services/ci/parse_dotenv_artifact_service.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Ci + class ParseDotenvArtifactService < ::BaseService + MAX_ACCEPTABLE_DOTENV_SIZE = 5.kilobytes + MAX_ACCEPTABLE_VARIABLES_COUNT = 10 + + SizeLimitError = Class.new(StandardError) + ParserError = Class.new(StandardError) + + def execute(artifact) + validate!(artifact) + + variables = parse!(artifact) + Ci::JobVariable.bulk_insert!(variables) + + success + rescue SizeLimitError, ParserError, ActiveRecord::RecordInvalid => error + Gitlab::ErrorTracking.track_exception(error, job_id: artifact.job_id) + error(error.message, :bad_request) + end + + private + + def validate!(artifact) + unless artifact&.dotenv? + raise ArgumentError, 'Artifact is not dotenv file type' + end + + unless artifact.file.size < MAX_ACCEPTABLE_DOTENV_SIZE + raise SizeLimitError, + "Dotenv Artifact Too Big. Maximum Allowable Size: #{MAX_ACCEPTABLE_DOTENV_SIZE}" + end + end + + def parse!(artifact) + variables = [] + + artifact.each_blob do |blob| + blob.each_line do |line| + key, value = scan_line!(line) + + variables << Ci::JobVariable.new(job_id: artifact.job_id, + source: :dotenv, key: key, value: value) + end + end + + if variables.size > MAX_ACCEPTABLE_VARIABLES_COUNT + raise SizeLimitError, + "Dotenv files cannot have more than #{MAX_ACCEPTABLE_VARIABLES_COUNT} variables" + end + + variables + end + + def scan_line!(line) + result = line.scan(/^(.*)=(.*)$/).last + + raise ParserError, 'Invalid Format' if result.nil? + + result.each(&:strip!) + end + end +end diff --git a/app/views/ci/variables/_index.html.haml b/app/views/ci/variables/_index.html.haml index f11c730eba6..aadb2c62d83 100644 --- a/app/views/ci/variables/_index.html.haml +++ b/app/views/ci/variables/_index.html.haml @@ -5,7 +5,7 @@ - link_start = '<a href="%{url}">'.html_safe % { url: help_page_path('ci/variables/README', anchor: 'protected-variables') } = s_('Environment variables are configured by your administrator to be %{link_start}protected%{link_end} by default').html_safe % { link_start: link_start, link_end: '</a>'.html_safe } -- if Feature.enabled?(:new_variables_ui, @project || @group) +- if Feature.enabled?(:new_variables_ui, @project || @group, default_enabled: true) - is_group = !@group.nil? #js-ci-project-variables{ data: { endpoint: save_endpoint, project_id: @project&.id || '', group: is_group.to_s, maskable_regex: ci_variable_maskable_regex} } diff --git a/changelogs/unreleased/dotenv-report-artifact.yml b/changelogs/unreleased/dotenv-report-artifact.yml new file mode 100644 index 00000000000..54ed75bc7ab --- /dev/null +++ b/changelogs/unreleased/dotenv-report-artifact.yml @@ -0,0 +1,5 @@ +--- +title: Support DotEnv Variables through report type artifact +merge_request: 26247 +author: +type: added diff --git a/changelogs/unreleased/feature-enable-split-diffs-by-default.yml b/changelogs/unreleased/feature-enable-split-diffs-by-default.yml new file mode 100644 index 00000000000..9f703a226c6 --- /dev/null +++ b/changelogs/unreleased/feature-enable-split-diffs-by-default.yml @@ -0,0 +1,5 @@ +--- +title: Diffs load each view style separately, on demand +merge_request: 24821 +author: +type: performance diff --git a/changelogs/unreleased/turn-on-new-variables-ui-ff.yml b/changelogs/unreleased/turn-on-new-variables-ui-ff.yml new file mode 100644 index 00000000000..9cb100481db --- /dev/null +++ b/changelogs/unreleased/turn-on-new-variables-ui-ff.yml @@ -0,0 +1,5 @@ +--- +title: Update UI for project and group settings CI variables +merge_request: 26901 +author: +type: added diff --git a/db/migrate/20200310145304_add_runtime_created_to_ci_job_variables.rb b/db/migrate/20200310145304_add_runtime_created_to_ci_job_variables.rb new file mode 100644 index 00000000000..d5ec8854bfa --- /dev/null +++ b/db/migrate/20200310145304_add_runtime_created_to_ci_job_variables.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddRuntimeCreatedToCiJobVariables < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + DEFAULT_SOURCE = 0 # Equvalent to Ci::JobVariable.internal_source + + def up + add_column_with_default(:ci_job_variables, :source, :integer, limit: 2, default: DEFAULT_SOURCE, allow_null: false) + end + + def down + remove_column(:ci_job_variables, :source) + end +end diff --git a/db/schema.rb b/db/schema.rb index f57b0638e05..741bb2b9262 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -780,6 +780,7 @@ ActiveRecord::Schema.define(version: 2020_03_12_163407) do t.string "encrypted_value_iv" t.bigint "job_id", null: false t.integer "variable_type", limit: 2, default: 1, null: false + t.integer "source", limit: 2, default: 0, null: false t.index ["job_id"], name: "index_ci_job_variables_on_job_id" t.index ["key", "job_id"], name: "index_ci_job_variables_on_key_and_job_id", unique: true end diff --git a/doc/ci/environments.md b/doc/ci/environments.md index 2268ab309c3..5bb1e221781 100644 --- a/doc/ci/environments.md +++ b/doc/ci/environments.md @@ -156,6 +156,70 @@ Starting with GitLab 9.3, the environment URL is exposed to the Runner via - `.gitlab-ci.yml`. - The external URL from the environment if not defined in `.gitlab-ci.yml`. +#### Set dynamic environment URLs after a job finishes + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/17066) in GitLab 12.9. + +In a job script, you can specify a static [environment URL](#using-the-environment-url). +However, there may be times when you want a dynamic URL. For example, +if you deploy a Review App to an external hosting +service that generates a random URL per deployment, like `https://94dd65b.amazonaws.com/qa-lambda-1234567`, +you don't know the URL before the deployment script finishes. +If you want to use the environment URL in GitLab, you would have to update it manually. + +To address this problem, you can configure a deployment job to report back a set of +variables, including the URL that was dynamically-generated by the external service. +GitLab supports [dotenv](https://github.com/bkeepers/dotenv) file as the format, +and expands the `environment:url` value with variables defined in the dotenv file. + +To use this feature, specify the +[`artifacts:reports:dotenv`](yaml/README.md#artifactsreportsdotenv) keyword in `.gitlab-ci.yml`. + +##### Example of setting dynamic environment URLs + +The following example shows a Review App that creates a new environment +per merge request. The `review` job is triggered by every push, and +creates or updates an environment named `review/your-branch-name`. +The environment URL is set to `$DYNAMIC_ENVIRONMENT_URL`: + +```yaml +review: + script: + - DYNAMIC_ENVIRONMENT_URL=$(deploy-script) # In script, get the environment URL. + - echo "DYNAMIC_ENVIRONMENT_URL=$DYNAMIC_ENVIRONMENT_URL" >> deploy.env # Add the value to a dotenv file. + artifacts: + reports: + dotenv: deploy.env # Report back dotenv file to rails. + environment: + name: review/$CI_COMMIT_REF_SLUG + url: $DYNAMIC_ENVIRONMENT_URL # and set the variable produced in script to `environment:url` + on_stop: stop_review + +stop_review: + script: + - ./teardown-environment + when: manual + environment: + name: review/$CI_COMMIT_REF_SLUG + action: stop +``` + +As soon as the `review` job finishes, GitLab updates the `review/your-branch-name` +environment's URL. +It parses the report artifact `deploy.env`, registers a list of variables as runtime-created, +uses it for expanding `environment:url: $DYNAMIC_ENVIRONMENT_URL` and sets it to the environment URL. +You can also specify a static part of the URL at `environment:url:`, such as +`https://$DYNAMIC_ENVIRONMENT_URL`. If the value of `DYNAMIC_ENVIRONMENT_URL` is +`123.awesome.com`, the final result will be `https://123.awesome.com`. + +The assigned URL for the `review/your-branch-name` environment is visible in the UI. +[See where the environment URL is displayed](#using-the-environment-url). + +> **Notes:** +> +> - `stop_review` doesn't generate a dotenv report artifact, so it won't recognize the `DYNAMIC_ENVIRONMENT_URL` variable. Therefore you should not set `environment:url:` in the `stop_review` job. +> - If the environment URL is not valid (for example, the URL is malformed), the system doesn't update the environment URL. + ### Configuring manual deployments Adding `when: manual` to an automatically executed job's configuration converts it to diff --git a/doc/ci/pipelines/index.md b/doc/ci/pipelines/index.md index a823793a995..092e7729ad3 100644 --- a/doc/ci/pipelines/index.md +++ b/doc/ci/pipelines/index.md @@ -1,4 +1,5 @@ --- +disqus_identifier: 'https://docs.gitlab.com/ee/ci/pipelines.html' type: reference --- diff --git a/doc/ci/pipelines/job_artifacts.md b/doc/ci/pipelines/job_artifacts.md index ef3a33e22ea..4cc6c2aa098 100644 --- a/doc/ci/pipelines/job_artifacts.md +++ b/doc/ci/pipelines/job_artifacts.md @@ -1,4 +1,5 @@ --- +disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/job_artifacts.html' type: reference, howto --- diff --git a/doc/ci/pipelines/schedules.md b/doc/ci/pipelines/schedules.md index 92cf1985d86..b9a2972dc89 100644 --- a/doc/ci/pipelines/schedules.md +++ b/doc/ci/pipelines/schedules.md @@ -1,4 +1,5 @@ --- +disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/schedules.html' type: reference, howto --- diff --git a/doc/ci/pipelines/settings.md b/doc/ci/pipelines/settings.md index 53a529493d2..13b8f4ee307 100644 --- a/doc/ci/pipelines/settings.md +++ b/doc/ci/pipelines/settings.md @@ -1,4 +1,5 @@ --- +disqus_identifier: 'https://docs.gitlab.com/ee/user/project/pipelines/settings.html' type: reference, howto --- diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 9224d73b58b..1441526bc80 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -2264,6 +2264,25 @@ concatenated into a single file. Use a filename pattern (`junit: rspec-*.xml`), an array of filenames (`junit: [rspec-1.xml, rspec-2.xml, rspec-3.xml]`), or a combination thereof (`junit: [rspec.xml, test-results/TEST-*.xml]`). +##### `artifacts:reports:dotenv` + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/17066) in GitLab 12.9. Requires GitLab Runner 11.5 and later. + +The `dotenv` report collects a set of environment variables as artifacts. + +The collected variables are registered as runtime-created variables of the job, +which is useful to [set dynamic environment URLs after a job finishes](../environments.md#set-dynamic-environment-urls-after-a-job-finishes). +It is not available for download through the web interface. + +There are a couple of limitations on top of the [original dotenv rules](https://github.com/motdotla/dotenv#rules). + +- The variable key can contain only letters, digits and underscore ('_'). +- The size of dotenv file must be smaller than 5 kilobytes. +- The number of variables must be less than 10. +- It doesn't support variable substitution in the dotenv file itself. +- It doesn't support empty lines and comments (`#`) in dotenv file. +- It doesn't support quote escape, spaces in a quote, a new line expansion in a quote, in dotenv file. + ##### `artifacts:reports:codequality` **(STARTER)** > Introduced in GitLab 11.5. Requires GitLab Runner 11.5 and above. diff --git a/doc/user/incident_management/index.md b/doc/user/incident_management/index.md index 21dd3bf4d9a..3ddc6894653 100644 --- a/doc/user/incident_management/index.md +++ b/doc/user/incident_management/index.md @@ -39,6 +39,8 @@ To select your issue template for use within Incident Management: GitLab can react to the alerts that your applications and services may be triggering by automatically creating issues, and alerting developers via email. +The emails will be sent to [owners and maintainers](../permissions.md) of the project and will contain details on the alert as well as a link to see more information. + ### Prometheus alerts Prometheus alerts can be set up in both: diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 571e056e096..994d3799004 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -11,7 +11,10 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management license_scanning metrics lsif].freeze + ALLOWED_KEYS = + %i[junit codequality sast dependency_scanning container_scanning + dast performance license_management license_scanning metrics lsif + dotenv].freeze attributes ALLOWED_KEYS @@ -31,6 +34,7 @@ module Gitlab validates :license_scanning, array_of_strings_or_string: true validates :metrics, array_of_strings_or_string: true validates :lsif, array_of_strings_or_string: true + validates :dotenv, array_of_strings_or_string: true end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 0336e834d06..1bbb8be6af5 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -1777,6 +1777,9 @@ msgstr "" msgid "An error occurred while committing your changes." msgstr "" +msgid "An error occurred while decoding the file." +msgstr "" + msgid "An error occurred while deleting the approvers group" msgstr "" @@ -1918,6 +1921,9 @@ msgstr "" msgid "An error occurred while loading the file." msgstr "" +msgid "An error occurred while loading the file. Please try again later." +msgstr "" + msgid "An error occurred while loading the merge request changes." msgstr "" diff --git a/spec/factories/ci/job_artifacts.rb b/spec/factories/ci/job_artifacts.rb index 296215abfa0..e0942bf0ac3 100644 --- a/spec/factories/ci/job_artifacts.rb +++ b/spec/factories/ci/job_artifacts.rb @@ -149,6 +149,16 @@ FactoryBot.define do end end + trait :dotenv do + file_type { :dotenv } + file_format { :gzip } + + after(:build) do |artifact, evaluator| + artifact.file = fixture_file_upload( + Rails.root.join('spec/fixtures/build.env.gz'), 'application/x-gzip') + end + end + trait :correct_checksum do after(:build) do |artifact, evaluator| artifact.file_sha256 = Digest::SHA256.file(artifact.file.path).hexdigest diff --git a/spec/factories/ci/job_variables.rb b/spec/factories/ci/job_variables.rb index 472a89d3bef..ae2b89ad98d 100644 --- a/spec/factories/ci/job_variables.rb +++ b/spec/factories/ci/job_variables.rb @@ -6,5 +6,9 @@ FactoryBot.define do value { 'VARIABLE_VALUE' } job factory: :ci_build + + trait :dotenv_source do + source { :dotenv } + end end end diff --git a/spec/fixtures/build.env.gz b/spec/fixtures/build.env.gz Binary files differnew file mode 100644 index 00000000000..39ad1e17ffe --- /dev/null +++ b/spec/fixtures/build.env.gz diff --git a/spec/frontend/blob/pdf/pdf_viewer_spec.js b/spec/frontend/blob/pdf/pdf_viewer_spec.js new file mode 100644 index 00000000000..0eea3aea639 --- /dev/null +++ b/spec/frontend/blob/pdf/pdf_viewer_spec.js @@ -0,0 +1,67 @@ +import { shallowMount } from '@vue/test-utils'; +import { GlLoadingIcon } from '@gitlab/ui'; + +import { FIXTURES_PATH } from 'spec/test_constants'; +import component from '~/blob/pdf/pdf_viewer.vue'; +import PdfLab from '~/pdf/index.vue'; + +const testPDF = `${FIXTURES_PATH}/blob/pdf/test.pdf`; + +describe('PDF renderer', () => { + let wrapper; + + const mountComponent = () => { + wrapper = shallowMount(component, { + propsData: { + pdf: testPDF, + }, + }); + }; + + const findLoading = () => wrapper.find(GlLoadingIcon); + const findPdfLab = () => wrapper.find(PdfLab); + const findLoadError = () => wrapper.find({ ref: 'loadError' }); + + beforeEach(() => { + mountComponent(); + }); + + afterEach(() => { + wrapper.destroy(); + wrapper = null; + }); + + it('shows loading icon', () => { + expect(findLoading().exists()).toBe(true); + }); + + describe('successful response', () => { + beforeEach(() => { + findPdfLab().vm.$emit('pdflabload'); + }); + + it('does not show loading icon', () => { + expect(findLoading().exists()).toBe(false); + }); + + it('renders the PDF', () => { + expect(findPdfLab().exists()).toBe(true); + }); + }); + + describe('error getting file', () => { + beforeEach(() => { + findPdfLab().vm.$emit('pdflaberror', 'foo'); + }); + + it('does not show loading icon', () => { + expect(findLoading().exists()).toBe(false); + }); + + it('shows error message', () => { + expect(findLoadError().text()).toBe( + 'An error occurred while loading the file. Please try again later.', + ); + }); + }); +}); diff --git a/spec/frontend/fixtures/static/pdf_viewer.html b/spec/frontend/fixtures/static/pdf_viewer.html deleted file mode 100644 index 350d35a262f..00000000000 --- a/spec/frontend/fixtures/static/pdf_viewer.html +++ /dev/null @@ -1 +0,0 @@ -<div class="file-content" data-endpoint="/test" id="js-pdf-viewer"></div> diff --git a/spec/frontend/lib/utils/common_utils_spec.js b/spec/frontend/lib/utils/common_utils_spec.js index d0d45b153af..6ba8f58086a 100644 --- a/spec/frontend/lib/utils/common_utils_spec.js +++ b/spec/frontend/lib/utils/common_utils_spec.js @@ -539,193 +539,382 @@ describe('common_utils', () => { }); }); - describe('convertObjectPropsToCamelCase', () => { - it('returns new object with camelCase property names by converting object with snake_case names', () => { - const snakeRegEx = /(_\w)/g; - const mockObj = { - id: 1, - group_name: 'GitLab.org', - absolute_web_url: 'https://gitlab.com/gitlab-org/', - }; - const mappings = { - id: 'id', - groupName: 'group_name', - absoluteWebUrl: 'absolute_web_url', - }; + describe('convertObjectProps*', () => { + const mockConversionFunction = prop => `${prop}_converted`; + const isEmptyObject = obj => + typeof obj === 'object' && obj !== null && Object.keys(obj).length === 0; + + const mockObjects = { + convertObjectProps: { + obj: { + id: 1, + group_name: 'GitLab.org', + absolute_web_url: 'https://gitlab.com/gitlab-org/', + }, + objNested: { + project_name: 'GitLab CE', + group_name: 'GitLab.org', + license_type: 'MIT', + tech_stack: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, + }, + }, + convertObjectPropsToCamelCase: { + obj: { + id: 1, + group_name: 'GitLab.org', + absolute_web_url: 'https://gitlab.com/gitlab-org/', + }, + objNested: { + project_name: 'GitLab CE', + group_name: 'GitLab.org', + license_type: 'MIT', + tech_stack: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, + }, + }, + convertObjectPropsToSnakeCase: { + obj: { + id: 1, + groupName: 'GitLab.org', + absoluteWebUrl: 'https://gitlab.com/gitlab-org/', + }, + objNested: { + projectName: 'GitLab CE', + groupName: 'GitLab.org', + licenseType: 'MIT', + techStack: { + backend: 'Ruby', + frontendFramework: 'Vue', + database: 'PostgreSQL', + }, + }, + }, + }; - const convertedObj = commonUtils.convertObjectPropsToCamelCase(mockObj); + describe('convertObjectProps', () => { + it('returns an empty object if `conversionFunction` parameter is not a function', () => { + const result = commonUtils.convertObjectProps(null, mockObjects.convertObjectProps.obj); - Object.keys(convertedObj).forEach(prop => { - expect(snakeRegEx.test(prop)).toBeFalsy(); - expect(convertedObj[prop]).toBe(mockObj[mappings[prop]]); + expect(isEmptyObject(result)).toBeTruthy(); }); }); - it('return empty object if method is called with null or undefined', () => { - expect(Object.keys(commonUtils.convertObjectPropsToCamelCase(null)).length).toBe(0); - expect(Object.keys(commonUtils.convertObjectPropsToCamelCase()).length).toBe(0); - expect(Object.keys(commonUtils.convertObjectPropsToCamelCase({})).length).toBe(0); - }); - - it('does not deep-convert by default', () => { - const obj = { - snake_key: { - child_snake_key: 'value', - }, - }; - - expect(commonUtils.convertObjectPropsToCamelCase(obj)).toEqual({ - snakeKey: { - child_snake_key: 'value', - }, + describe.each` + functionName | mockObj | mockObjNested + ${'convertObjectProps'} | ${mockObjects.convertObjectProps.obj} | ${mockObjects.convertObjectProps.objNested} + ${'convertObjectPropsToCamelCase'} | ${mockObjects.convertObjectPropsToCamelCase.obj} | ${mockObjects.convertObjectPropsToCamelCase.objNested} + ${'convertObjectPropsToSnakeCase'} | ${mockObjects.convertObjectPropsToSnakeCase.obj} | ${mockObjects.convertObjectPropsToSnakeCase.objNested} + `('$functionName', ({ functionName, mockObj, mockObjNested }) => { + const testFunction = + functionName === 'convertObjectProps' + ? (obj, options = {}) => + commonUtils.convertObjectProps(mockConversionFunction, obj, options) + : commonUtils[functionName]; + + it('returns an empty object if `obj` parameter is null, undefined or an empty object', () => { + expect(isEmptyObject(testFunction(null))).toBeTruthy(); + expect(isEmptyObject(testFunction())).toBeTruthy(); + expect(isEmptyObject(testFunction({}))).toBeTruthy(); }); - }); - describe('convertObjectPropsToSnakeCase', () => { - it('converts each object key to snake case', () => { - const obj = { - some: 'some', - 'cool object': 'cool object', - likeThisLongOne: 'likeThisLongOne', + it('converts object properties', () => { + const expected = { + convertObjectProps: { + id_converted: 1, + group_name_converted: 'GitLab.org', + absolute_web_url_converted: 'https://gitlab.com/gitlab-org/', + }, + convertObjectPropsToCamelCase: { + id: 1, + groupName: 'GitLab.org', + absoluteWebUrl: 'https://gitlab.com/gitlab-org/', + }, + convertObjectPropsToSnakeCase: { + id: 1, + group_name: 'GitLab.org', + absolute_web_url: 'https://gitlab.com/gitlab-org/', + }, }; - expect(commonUtils.convertObjectPropsToSnakeCase(obj)).toEqual({ - some: 'some', - cool_object: 'cool object', - like_this_long_one: 'likeThisLongOne', - }); + expect(testFunction(mockObj)).toEqual(expected[functionName]); }); - it('returns an empty object if there are no keys', () => { - ['', {}, [], null].forEach(badObj => { - expect(commonUtils.convertObjectPropsToSnakeCase(badObj)).toEqual({}); - }); - }); - }); - - describe('with options', () => { - const objWithoutChildren = { - project_name: 'GitLab CE', - group_name: 'GitLab.org', - license_type: 'MIT', - }; + it('does not deep-convert by default', () => { + const expected = { + convertObjectProps: { + project_name_converted: 'GitLab CE', + group_name_converted: 'GitLab.org', + license_type_converted: 'MIT', + tech_stack_converted: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, + }, + convertObjectPropsToCamelCase: { + projectName: 'GitLab CE', + groupName: 'GitLab.org', + licenseType: 'MIT', + techStack: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, + }, + convertObjectPropsToSnakeCase: { + project_name: 'GitLab CE', + group_name: 'GitLab.org', + license_type: 'MIT', + tech_stack: { + backend: 'Ruby', + frontendFramework: 'Vue', + database: 'PostgreSQL', + }, + }, + }; - const objWithChildren = { - project_name: 'GitLab CE', - group_name: 'GitLab.org', - license_type: 'MIT', - tech_stack: { - backend: 'Ruby', - frontend_framework: 'Vue', - database: 'PostgreSQL', - }, - }; + expect(testFunction(mockObjNested)).toEqual(expected[functionName]); + }); - describe('when options.deep is true', () => { - it('converts object with child objects', () => { - const obj = { - snake_key: { - child_snake_key: 'value', + describe('with options', () => { + describe('when options.deep is true', () => { + const expected = { + convertObjectProps: { + project_name_converted: 'GitLab CE', + group_name_converted: 'GitLab.org', + license_type_converted: 'MIT', + tech_stack_converted: { + backend_converted: 'Ruby', + frontend_framework_converted: 'Vue', + database_converted: 'PostgreSQL', + }, + }, + convertObjectPropsToCamelCase: { + projectName: 'GitLab CE', + groupName: 'GitLab.org', + licenseType: 'MIT', + techStack: { + backend: 'Ruby', + frontendFramework: 'Vue', + database: 'PostgreSQL', + }, + }, + convertObjectPropsToSnakeCase: { + project_name: 'GitLab CE', + group_name: 'GitLab.org', + license_type: 'MIT', + tech_stack: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, }, }; - expect(commonUtils.convertObjectPropsToCamelCase(obj, { deep: true })).toEqual({ - snakeKey: { - childSnakeKey: 'value', - }, + it('converts nested objects', () => { + expect(testFunction(mockObjNested, { deep: true })).toEqual(expected[functionName]); }); - }); - it('converts array with child objects', () => { - const arr = [ - { - child_snake_key: 'value', - }, - ]; + it('converts array of nested objects', () => { + expect(testFunction([mockObjNested], { deep: true })).toEqual([expected[functionName]]); + }); - expect(commonUtils.convertObjectPropsToCamelCase(arr, { deep: true })).toEqual([ - { - childSnakeKey: 'value', - }, - ]); + it('converts array with child arrays', () => { + expect(testFunction([[mockObjNested]], { deep: true })).toEqual([ + [expected[functionName]], + ]); + }); }); - it('converts array with child arrays', () => { - const arr = [ - [ - { - child_snake_key: 'value', + describe('when options.dropKeys is provided', () => { + it('discards properties mentioned in `dropKeys` array', () => { + const expected = { + convertObjectProps: { + project_name_converted: 'GitLab CE', + license_type_converted: 'MIT', + tech_stack_converted: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, }, - ], - ]; - - expect(commonUtils.convertObjectPropsToCamelCase(arr, { deep: true })).toEqual([ - [ - { - childSnakeKey: 'value', + convertObjectPropsToCamelCase: { + projectName: 'GitLab CE', + licenseType: 'MIT', + techStack: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, }, - ], - ]); - }); - }); - - describe('when options.dropKeys is provided', () => { - it('discards properties mentioned in `dropKeys` array', () => { - expect( - commonUtils.convertObjectPropsToCamelCase(objWithoutChildren, { - dropKeys: ['group_name'], - }), - ).toEqual({ - projectName: 'GitLab CE', - licenseType: 'MIT', + convertObjectPropsToSnakeCase: { + project_name: 'GitLab CE', + license_type: 'MIT', + tech_stack: { + backend: 'Ruby', + frontendFramework: 'Vue', + database: 'PostgreSQL', + }, + }, + }; + + const dropKeys = { + convertObjectProps: ['group_name'], + convertObjectPropsToCamelCase: ['group_name'], + convertObjectPropsToSnakeCase: ['groupName'], + }; + + expect( + testFunction(mockObjNested, { + dropKeys: dropKeys[functionName], + }), + ).toEqual(expected[functionName]); }); - }); - it('discards properties mentioned in `dropKeys` array when `deep` is true', () => { - expect( - commonUtils.convertObjectPropsToCamelCase(objWithChildren, { - deep: true, - dropKeys: ['group_name', 'database'], - }), - ).toEqual({ - projectName: 'GitLab CE', - licenseType: 'MIT', - techStack: { - backend: 'Ruby', - frontendFramework: 'Vue', - }, + it('discards properties mentioned in `dropKeys` array when `deep` is true', () => { + const expected = { + convertObjectProps: { + project_name_converted: 'GitLab CE', + license_type_converted: 'MIT', + tech_stack_converted: { + backend_converted: 'Ruby', + frontend_framework_converted: 'Vue', + }, + }, + convertObjectPropsToCamelCase: { + projectName: 'GitLab CE', + licenseType: 'MIT', + techStack: { + backend: 'Ruby', + frontendFramework: 'Vue', + }, + }, + convertObjectPropsToSnakeCase: { + project_name: 'GitLab CE', + license_type: 'MIT', + tech_stack: { + backend: 'Ruby', + frontend_framework: 'Vue', + }, + }, + }; + + const dropKeys = { + convertObjectProps: ['group_name', 'database'], + convertObjectPropsToCamelCase: ['group_name', 'database'], + convertObjectPropsToSnakeCase: ['groupName', 'database'], + }; + + expect( + testFunction(mockObjNested, { + dropKeys: dropKeys[functionName], + deep: true, + }), + ).toEqual(expected[functionName]); }); }); - }); - describe('when options.ignoreKeyNames is provided', () => { - it('leaves properties mentioned in `ignoreKeyNames` array intact', () => { - expect( - commonUtils.convertObjectPropsToCamelCase(objWithoutChildren, { - ignoreKeyNames: ['group_name'], - }), - ).toEqual({ - projectName: 'GitLab CE', - licenseType: 'MIT', - group_name: 'GitLab.org', + describe('when options.ignoreKeyNames is provided', () => { + it('leaves properties mentioned in `ignoreKeyNames` array intact', () => { + const expected = { + convertObjectProps: { + project_name_converted: 'GitLab CE', + group_name: 'GitLab.org', + license_type_converted: 'MIT', + tech_stack_converted: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, + }, + convertObjectPropsToCamelCase: { + projectName: 'GitLab CE', + group_name: 'GitLab.org', + licenseType: 'MIT', + techStack: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, + }, + convertObjectPropsToSnakeCase: { + project_name: 'GitLab CE', + groupName: 'GitLab.org', + license_type: 'MIT', + tech_stack: { + backend: 'Ruby', + frontendFramework: 'Vue', + database: 'PostgreSQL', + }, + }, + }; + + const ignoreKeyNames = { + convertObjectProps: ['group_name'], + convertObjectPropsToCamelCase: ['group_name'], + convertObjectPropsToSnakeCase: ['groupName'], + }; + + expect( + testFunction(mockObjNested, { + ignoreKeyNames: ignoreKeyNames[functionName], + }), + ).toEqual(expected[functionName]); }); - }); - it('leaves properties mentioned in `ignoreKeyNames` array intact when `deep` is true', () => { - expect( - commonUtils.convertObjectPropsToCamelCase(objWithChildren, { - deep: true, - ignoreKeyNames: ['group_name', 'frontend_framework'], - }), - ).toEqual({ - projectName: 'GitLab CE', - group_name: 'GitLab.org', - licenseType: 'MIT', - techStack: { - backend: 'Ruby', - frontend_framework: 'Vue', - database: 'PostgreSQL', - }, + it('leaves properties mentioned in `ignoreKeyNames` array intact when `deep` is true', () => { + const expected = { + convertObjectProps: { + project_name_converted: 'GitLab CE', + group_name: 'GitLab.org', + license_type_converted: 'MIT', + tech_stack_converted: { + backend_converted: 'Ruby', + frontend_framework: 'Vue', + database_converted: 'PostgreSQL', + }, + }, + convertObjectPropsToCamelCase: { + projectName: 'GitLab CE', + group_name: 'GitLab.org', + licenseType: 'MIT', + techStack: { + backend: 'Ruby', + frontend_framework: 'Vue', + database: 'PostgreSQL', + }, + }, + convertObjectPropsToSnakeCase: { + project_name: 'GitLab CE', + groupName: 'GitLab.org', + license_type: 'MIT', + tech_stack: { + backend: 'Ruby', + frontendFramework: 'Vue', + database: 'PostgreSQL', + }, + }, + }; + + const ignoreKeyNames = { + convertObjectProps: ['group_name', 'frontend_framework'], + convertObjectPropsToCamelCase: ['group_name', 'frontend_framework'], + convertObjectPropsToSnakeCase: ['groupName', 'frontendFramework'], + }; + + expect( + testFunction(mockObjNested, { + deep: true, + ignoreKeyNames: ignoreKeyNames[functionName], + }), + ).toEqual(expected[functionName]); }); }); }); diff --git a/spec/javascripts/blob/pdf/index_spec.js b/spec/javascripts/blob/pdf/index_spec.js deleted file mode 100644 index 66769a8aa47..00000000000 --- a/spec/javascripts/blob/pdf/index_spec.js +++ /dev/null @@ -1,72 +0,0 @@ -import { FIXTURES_PATH } from 'spec/test_constants'; -import renderPDF from '~/blob/pdf'; - -const testPDF = `${FIXTURES_PATH}/blob/pdf/test.pdf`; - -describe('PDF renderer', () => { - let viewer; - let app; - - const checkLoaded = done => { - if (app.loading) { - setTimeout(() => { - checkLoaded(done); - }, 100); - } else { - done(); - } - }; - - preloadFixtures('static/pdf_viewer.html'); - - beforeEach(() => { - loadFixtures('static/pdf_viewer.html'); - viewer = document.getElementById('js-pdf-viewer'); - viewer.dataset.endpoint = testPDF; - }); - - it('shows loading icon', () => { - renderPDF(); - - expect(document.querySelector('.loading')).not.toBeNull(); - }); - - describe('successful response', () => { - beforeEach(done => { - app = renderPDF(); - - checkLoaded(done); - }); - - it('does not show loading icon', () => { - expect(document.querySelector('.loading')).toBeNull(); - }); - - it('renders the PDF', () => { - expect(document.querySelector('.pdf-viewer')).not.toBeNull(); - }); - - it('renders the PDF page', () => { - expect(document.querySelector('.pdf-page')).not.toBeNull(); - }); - }); - - describe('error getting file', () => { - beforeEach(done => { - viewer.dataset.endpoint = 'invalid/path/to/file.pdf'; - app = renderPDF(); - - checkLoaded(done); - }); - - it('does not show loading icon', () => { - expect(document.querySelector('.loading')).toBeNull(); - }); - - it('shows error message', () => { - expect(document.querySelector('.md').textContent.trim()).toBe( - 'An error occurred while loading the file. Please try again later.', - ); - }); - }); -}); diff --git a/spec/javascripts/pdf/index_spec.js b/spec/javascripts/pdf/index_spec.js index e14f1b27f6c..39cd4dacd70 100644 --- a/spec/javascripts/pdf/index_spec.js +++ b/spec/javascripts/pdf/index_spec.js @@ -1,13 +1,10 @@ import Vue from 'vue'; -import { GlobalWorkerOptions } from 'pdfjs-dist/build/pdf'; -import workerSrc from 'pdfjs-dist/build/pdf.worker.min'; import { FIXTURES_PATH } from 'spec/test_constants'; import PDFLab from '~/pdf/index.vue'; const pdf = `${FIXTURES_PATH}/blob/pdf/test.pdf`; -GlobalWorkerOptions.workerSrc = workerSrc; const Component = Vue.extend(PDFLab); describe('PDF component', () => { diff --git a/spec/javascripts/pdf/page_spec.js b/spec/javascripts/pdf/page_spec.js index bb2294e8d18..cc2cc204ee3 100644 --- a/spec/javascripts/pdf/page_spec.js +++ b/spec/javascripts/pdf/page_spec.js @@ -1,6 +1,5 @@ import Vue from 'vue'; -import pdfjsLib from 'pdfjs-dist/build/pdf'; -import workerSrc from 'pdfjs-dist/build/pdf.worker.min'; +import pdfjsLib from 'pdfjs-dist/webpack'; import mountComponent from 'spec/helpers/vue_mount_component_helper'; import { FIXTURES_PATH } from 'spec/test_constants'; @@ -14,7 +13,6 @@ describe('Page component', () => { let testPage; beforeEach(done => { - pdfjsLib.GlobalWorkerOptions.workerSrc = workerSrc; pdfjsLib .getDocument(testPDF) .promise.then(pdf => pdf.getPage(1)) diff --git a/spec/lib/gitlab/ci/config/entry/reports_spec.rb b/spec/lib/gitlab/ci/config/entry/reports_spec.rb index c64bb0a4cc3..31e1aaa42bf 100644 --- a/spec/lib/gitlab/ci/config/entry/reports_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/reports_spec.rb @@ -44,6 +44,7 @@ describe Gitlab::Ci::Config::Entry::Reports do :license_scanning | 'gl-license-scanning-report.json' :performance | 'performance.json' :lsif | 'lsif.json' + :dotenv | 'build.dotenv' end with_them do diff --git a/spec/models/pages_domain_spec.rb b/spec/models/pages_domain_spec.rb index 33459767302..a4ed02c3254 100644 --- a/spec/models/pages_domain_spec.rb +++ b/spec/models/pages_domain_spec.rb @@ -7,6 +7,11 @@ describe PagesDomain do subject(:pages_domain) { described_class.new } + # Locking in date due to cert expiration date https://gitlab.com/gitlab-org/gitlab/-/issues/210557#note_304749257 + around do |example| + Timecop.travel(Time.new(2020, 3, 12)) { example.run } + end + describe 'associations' do it { is_expected.to belong_to(:project) } it { is_expected.to have_many(:serverless_domain_clusters) } diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 6e5c2088ee7..5a8add1e9db 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -1937,6 +1937,49 @@ describe API::Runner, :clean_gitlab_redis_shared_state do end end end + + context 'when artifact_type is dotenv' do + context 'when artifact_format is gzip' do + let(:file_upload) { fixture_file_upload('spec/fixtures/build.env.gz') } + let(:params) { { artifact_type: :dotenv, artifact_format: :gzip } } + + it 'stores dotenv file' do + upload_artifacts(file_upload, headers_with_token, params) + + expect(response).to have_gitlab_http_status(:created) + expect(job.reload.job_artifacts_dotenv).not_to be_nil + end + + it 'parses dotenv file' do + expect do + upload_artifacts(file_upload, headers_with_token, params) + end.to change { job.job_variables.count }.from(0).to(2) + end + + context 'when parse error happens' do + let(:file_upload) { fixture_file_upload('spec/fixtures/ci_build_artifacts_metadata.gz') } + + it 'returns an error' do + upload_artifacts(file_upload, headers_with_token, params) + + expect(response).to have_gitlab_http_status(:bad_request) + expect(json_response['message']).to eq('Invalid Format') + end + end + end + + context 'when artifact_format is raw' do + let(:file_upload) { fixture_file_upload('spec/fixtures/build.env.gz') } + let(:params) { { artifact_type: :dotenv, artifact_format: :raw } } + + it 'returns an error' do + upload_artifacts(file_upload, headers_with_token, params) + + expect(response).to have_gitlab_http_status(:bad_request) + expect(job.reload.job_artifacts_dotenv).to be_nil + end + end + end end context 'when artifacts already exist for the job' do diff --git a/spec/services/ci/create_job_artifacts_service_spec.rb b/spec/services/ci/create_job_artifacts_service_spec.rb index 03106687678..fe64a66f322 100644 --- a/spec/services/ci/create_job_artifacts_service_spec.rb +++ b/spec/services/ci/create_job_artifacts_service_spec.rb @@ -121,6 +121,42 @@ describe Ci::CreateJobArtifactsService do end end + context 'when artifact type is dotenv' do + let(:artifacts_file) do + file_to_upload('spec/fixtures/build.env.gz', sha256: artifacts_sha256) + end + + let(:params) do + { + 'artifact_type' => 'dotenv', + 'artifact_format' => 'gzip' + } + end + + it 'calls parse service' do + expect_any_instance_of(Ci::ParseDotenvArtifactService) do |service| + expect(service).to receive(:execute).once.and_call_original + end + + expect(subject[:status]).to eq(:success) + expect(job.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => 'VAR1', 'source' => 'dotenv'), + hash_including('key' => 'KEY2', 'value' => 'VAR2', 'source' => 'dotenv')) + end + + context 'when ci_synchronous_artifact_parsing feature flag is disabled' do + before do + stub_feature_flags(ci_synchronous_artifact_parsing: false) + end + + it 'does not call parse service' do + expect(Ci::ParseDotenvArtifactService).not_to receive(:new) + + expect(subject[:status]).to eq(:success) + end + end + end + shared_examples 'rescues object storage error' do |klass, message, expected_message| it "handles #{klass}" do allow_next_instance_of(JobArtifactUploader) do |uploader| diff --git a/spec/services/ci/parse_dotenv_artifact_service_spec.rb b/spec/services/ci/parse_dotenv_artifact_service_spec.rb new file mode 100644 index 00000000000..fc4131d262b --- /dev/null +++ b/spec/services/ci/parse_dotenv_artifact_service_spec.rb @@ -0,0 +1,260 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Ci::ParseDotenvArtifactService do + let_it_be(:project) { create(:project) } + let_it_be(:pipeline) { create(:ci_pipeline, project: project) } + let(:build) { create(:ci_build, pipeline: pipeline, project: project) } + let(:service) { described_class.new(project, nil) } + + describe '#execute' do + subject { service.execute(artifact) } + + context 'when build has a dotenv artifact' do + let!(:artifact) { create(:ci_job_artifact, :dotenv, job: build) } + + it 'parses the artifact' do + expect(subject[:status]).to eq(:success) + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => 'VAR1'), + hash_including('key' => 'KEY2', 'value' => 'VAR2')) + end + + context 'when parse error happens' do + before do + allow(service).to receive(:scan_line!) { raise described_class::ParserError.new('Invalid Format') } + end + + it 'returns error' do + expect(Gitlab::ErrorTracking).to receive(:track_exception) + .with(described_class::ParserError, job_id: build.id) + + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq('Invalid Format') + expect(subject[:http_status]).to eq(:bad_request) + end + end + + context 'when artifact size is too big' do + before do + allow(artifact.file).to receive(:size) { 10.kilobytes } + end + + it 'returns error' do + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq("Dotenv Artifact Too Big. Maximum Allowable Size: #{described_class::MAX_ACCEPTABLE_DOTENV_SIZE}") + expect(subject[:http_status]).to eq(:bad_request) + end + end + + context 'when artifact has the specified blob' do + before do + allow(artifact).to receive(:each_blob).and_yield(blob) + end + + context 'when a white space trails the key' do + let(:blob) { 'KEY1 =VAR1' } + + it 'trims the trailing space' do + subject + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => 'VAR1')) + end + end + + context 'when multiple key/value pairs exist in one line' do + let(:blob) { 'KEY1=VAR1KEY2=VAR1' } + + it 'returns error' do + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq("Validation failed: Key can contain only letters, digits and '_'.") + expect(subject[:http_status]).to eq(:bad_request) + end + end + + context 'when key contains UNICODE' do + let(:blob) { '🛹=skateboard' } + + it 'returns error' do + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq("Validation failed: Key can contain only letters, digits and '_'.") + expect(subject[:http_status]).to eq(:bad_request) + end + end + + context 'when value contains UNICODE' do + let(:blob) { 'skateboard=🛹' } + + it 'parses the dotenv data' do + subject + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'skateboard', 'value' => '🛹')) + end + end + + context 'when key contains a space' do + let(:blob) { 'K E Y 1=VAR1' } + + it 'returns error' do + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq("Validation failed: Key can contain only letters, digits and '_'.") + expect(subject[:http_status]).to eq(:bad_request) + end + end + + context 'when value contains a space' do + let(:blob) { 'KEY1=V A R 1' } + + it 'parses the dotenv data' do + subject + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => 'V A R 1')) + end + end + + context 'when value is double quoated' do + let(:blob) { 'KEY1="VAR1"' } + + it 'parses the value as-is' do + subject + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => '"VAR1"')) + end + end + + context 'when value is single quoated' do + let(:blob) { "KEY1='VAR1'" } + + it 'parses the value as-is' do + subject + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => "'VAR1'")) + end + end + + context 'when value has white spaces in double quote' do + let(:blob) { 'KEY1=" VAR1 "' } + + it 'parses the value as-is' do + subject + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => '" VAR1 "')) + end + end + + context 'when key is missing' do + let(:blob) { '=VAR1' } + + it 'returns error' do + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to match(/Key can't be blank/) + expect(subject[:http_status]).to eq(:bad_request) + end + end + + context 'when value is missing' do + let(:blob) { 'KEY1=' } + + it 'parses the dotenv data' do + subject + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => '')) + end + end + + context 'when it is not dotenv format' do + let(:blob) { "{ 'KEY1': 'VAR1' }" } + + it 'returns error' do + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq('Invalid Format') + expect(subject[:http_status]).to eq(:bad_request) + end + end + + context 'when more than limitated variables are specified in dotenv' do + let(:blob) do + StringIO.new.tap do |s| + (described_class::MAX_ACCEPTABLE_VARIABLES_COUNT + 1).times do |i| + s << "KEY#{i}=VAR#{i}\n" + end + end.string + end + + it 'returns error' do + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq("Dotenv files cannot have more than #{described_class::MAX_ACCEPTABLE_VARIABLES_COUNT} variables") + expect(subject[:http_status]).to eq(:bad_request) + end + end + + context 'when variables are cross-referenced in dotenv' do + let(:blob) do + <<~EOS + KEY1=VAR1 + KEY2=${KEY1}_Test + EOS + end + + it 'does not support variable expansion in dotenv parser' do + subject + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => 'VAR1'), + hash_including('key' => 'KEY2', 'value' => '${KEY1}_Test')) + end + end + + context 'when there is an empty line' do + let(:blob) do + <<~EOS + KEY1=VAR1 + + KEY2=VAR2 + EOS + end + + it 'does not support empty line in dotenv parser' do + subject + + expect(subject[:status]).to eq(:error) + expect(subject[:message]).to eq('Invalid Format') + expect(subject[:http_status]).to eq(:bad_request) + end + end + + context 'when there is a comment' do + let(:blob) do + <<~EOS + KEY1=VAR1 # This is variable + EOS + end + + it 'does not support comment in dotenv parser' do + subject + + expect(build.job_variables.as_json).to contain_exactly( + hash_including('key' => 'KEY1', 'value' => 'VAR1 # This is variable')) + end + end + end + end + + context 'when build does not have a dotenv artifact' do + let!(:artifact) { } + + it 'raises an error' do + expect { subject }.to raise_error(ArgumentError) + end + end + end +end diff --git a/spec/services/ci/retry_build_service_spec.rb b/spec/services/ci/retry_build_service_spec.rb index 8a22b2c8da3..6cecab8656a 100644 --- a/spec/services/ci/retry_build_service_spec.rb +++ b/spec/services/ci/retry_build_service_spec.rb @@ -36,7 +36,7 @@ describe Ci::RetryBuildService do job_artifacts_performance job_artifacts_lsif job_artifacts_codequality job_artifacts_metrics scheduled_at job_variables waiting_for_resource_at job_artifacts_metrics_referee - job_artifacts_network_referee needs].freeze + job_artifacts_network_referee job_artifacts_dotenv needs].freeze IGNORE_ACCESSORS = %i[type lock_version target_url base_tags trace_sections diff --git a/spec/services/deployments/after_create_service_spec.rb b/spec/services/deployments/after_create_service_spec.rb index 3aa137a866e..5a69ffd8b9c 100644 --- a/spec/services/deployments/after_create_service_spec.rb +++ b/spec/services/deployments/after_create_service_spec.rb @@ -177,6 +177,26 @@ describe Deployments::AfterCreateService do it { is_expected.to eq('http://review/host') } end + context 'when job variables are generated during runtime' do + let(:job) do + create(:ci_build, + :with_deployment, + pipeline: pipeline, + environment: 'review/$CI_COMMIT_REF_NAME', + project: project, + job_variables: [job_variable], + options: { environment: { name: 'review/$CI_COMMIT_REF_NAME', url: 'http://$DYNAMIC_ENV_URL' } }) + end + + let(:job_variable) do + build(:ci_job_variable, :dotenv_source, key: 'DYNAMIC_ENV_URL', value: 'abc.test.com') + end + + it 'expands the environment URL from the dynamic variable' do + is_expected.to eq('http://abc.test.com') + end + end + context 'when yaml environment does not have url' do let(:job) { create(:ci_build, :with_deployment, pipeline: pipeline, environment: 'staging', project: project) } diff --git a/spec/workers/pages_domain_ssl_renewal_cron_worker_spec.rb b/spec/workers/pages_domain_ssl_renewal_cron_worker_spec.rb index 736acc40371..ae66122b4de 100644 --- a/spec/workers/pages_domain_ssl_renewal_cron_worker_spec.rb +++ b/spec/workers/pages_domain_ssl_renewal_cron_worker_spec.rb @@ -7,6 +7,11 @@ describe PagesDomainSslRenewalCronWorker do subject(:worker) { described_class.new } + # Locking in date due to cert expiration date https://gitlab.com/gitlab-org/gitlab/-/issues/210557#note_304749257 + around do |example| + Timecop.travel(Time.new(2020, 3, 12)) { example.run } + end + before do stub_lets_encrypt_settings end |