diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 09:09:22 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-07-20 09:09:22 +0000 |
| commit | a7608a4940a91e14754d56a7acbe496321fed99c (patch) | |
| tree | fb661eddbd2d190695050788b7f89168a6f541e3 | |
| parent | 7734690def0c885f9f79567185c3dc5df353f9a0 (diff) | |
| download | gitlab-ce-a7608a4940a91e14754d56a7acbe496321fed99c.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
56 files changed, 1878 insertions, 153 deletions
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index f508bfa1465..839a06862b2 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -125,6 +125,7 @@ .db-patterns: &db-patterns - "{,ee/}{,spec/}{db,migrations}/**/*" - "{,ee/}{,spec/}lib/{,ee/}gitlab/background_migration/**/*" + - "config/prometheus/common_metrics.yml" # Used by Gitlab::DatabaseImporters::CommonMetrics::Importer .backstage-patterns: &backstage-patterns - "Dangerfile" diff --git a/app/assets/javascripts/diffs/components/app.vue b/app/assets/javascripts/diffs/components/app.vue index 1e524882d5f..9bca555bf1d 100644 --- a/app/assets/javascripts/diffs/components/app.vue +++ b/app/assets/javascripts/diffs/components/app.vue @@ -441,7 +441,7 @@ export default { [CENTERED_LIMITED_CONTAINER_CLASSES]: isLimitedContainer, }" > - <commit-widget v-if="commit" :commit="commit" /> + <commit-widget v-if="commit" :commit="commit" :collapsible="false" /> <div v-if="isBatchLoading" class="loading"><gl-loading-icon size="lg" /></div> <template v-else-if="renderDiffFiles"> <diff-file diff --git a/app/assets/javascripts/diffs/components/commit_item.vue b/app/assets/javascripts/diffs/components/commit_item.vue index 99bc1b5c040..f579b2ae2ba 100644 --- a/app/assets/javascripts/diffs/components/commit_item.vue +++ b/app/assets/javascripts/diffs/components/commit_item.vue @@ -56,6 +56,11 @@ export default { type: Object, required: true, }, + collapsible: { + type: Boolean, + required: false, + default: true, + }, }, computed: { author() { @@ -104,7 +109,7 @@ export default { </script> <template> - <li class="commit flex-row js-toggle-container"> + <li :class="{ 'js-toggle-container': collapsible }" class="commit flex-row"> <user-avatar-link :link-href="authorUrl" :img-src="authorAvatar" @@ -123,7 +128,7 @@ export default { <span class="commit-row-message d-block d-sm-none">· {{ commit.short_id }}</span> <button - v-if="commit.description_html" + v-if="commit.description_html && collapsible" class="text-expander js-toggle-button" type="button" :aria-label="__('Toggle commit description')" @@ -144,7 +149,8 @@ export default { <pre v-if="commit.description_html" - class="commit-row-description js-toggle-content gl-mb-3" + :class="{ 'js-toggle-content': collapsible, 'd-block': !collapsible }" + class="commit-row-description gl-mb-3 text-dark" v-html="commit.description_html" ></pre> </div> diff --git a/app/assets/javascripts/diffs/components/commit_widget.vue b/app/assets/javascripts/diffs/components/commit_widget.vue index 31ed003cc0f..5c7e84bd87c 100644 --- a/app/assets/javascripts/diffs/components/commit_widget.vue +++ b/app/assets/javascripts/diffs/components/commit_widget.vue @@ -23,15 +23,20 @@ export default { type: Object, required: true, }, + collapsible: { + type: Boolean, + required: false, + default: true, + }, }, }; </script> <template> - <div class="info-well w-100"> + <div class="info-well mw-100 mx-0"> <div class="well-segment"> <ul class="blob-commit-info"> - <commit-item :commit="commit" /> + <commit-item :commit="commit" :collapsible="collapsible" /> </ul> </div> </div> diff --git a/app/assets/javascripts/vue_shared/components/rich_content_editor/services/build_html_to_markdown_renderer.js b/app/assets/javascripts/vue_shared/components/rich_content_editor/services/build_html_to_markdown_renderer.js index 3331f4fb20d..21ded83a771 100644 --- a/app/assets/javascripts/vue_shared/components/rich_content_editor/services/build_html_to_markdown_renderer.js +++ b/app/assets/javascripts/vue_shared/components/rich_content_editor/services/build_html_to_markdown_renderer.js @@ -4,6 +4,8 @@ import { defaults, repeat } from 'lodash'; const DEFAULTS = { subListIndentSpaces: 4, unorderedListBulletChar: '-', + strong: '*', + emphasis: '_', }; const countIndentSpaces = text => { @@ -13,12 +15,14 @@ const countIndentSpaces = text => { }; const buildHTMLToMarkdownRender = (baseRenderer, formattingPreferences = {}) => { - const { subListIndentSpaces, unorderedListBulletChar } = defaults( + const { subListIndentSpaces, unorderedListBulletChar, strong, emphasis } = defaults( formattingPreferences, DEFAULTS, ); const sublistNode = 'LI OL, LI UL'; const unorderedListItemNode = 'UL LI'; + const emphasisNode = 'EM, I'; + const strongNode = 'STRONG, B'; return { TEXT_NODE(node) { @@ -57,6 +61,17 @@ const buildHTMLToMarkdownRender = (baseRenderer, formattingPreferences = {}) => return baseResult.replace(/^(\s*)([*|-])/, `$1${unorderedListBulletChar}`); }, + [emphasisNode](node, subContent) { + const result = baseRenderer.convert(node, subContent); + + return result.replace(/(^[*_]{1}|[*_]{1}$)/g, emphasis); + }, + [strongNode](node, subContent) { + const result = baseRenderer.convert(node, subContent); + const strongSyntax = repeat(strong, 2); + + return result.replace(/^[*_]{2}/, strongSyntax).replace(/[*_]{2}$/, strongSyntax); + }, }; }; diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb new file mode 100644 index 00000000000..bc5b305f2dd --- /dev/null +++ b/app/models/clusters/agent.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Clusters + class Agent < ApplicationRecord + self.table_name = 'cluster_agents' + + belongs_to :project, class_name: '::Project' # Otherwise, it will load ::Clusters::Project + + has_many :agent_tokens, class_name: 'Clusters::AgentToken' + + validates :name, presence: true, length: { maximum: 255 }, uniqueness: { scope: :project_id } + end +end diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb new file mode 100644 index 00000000000..e9f1ee4e033 --- /dev/null +++ b/app/models/clusters/agent_token.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Clusters + class AgentToken < ApplicationRecord + include TokenAuthenticatable + add_authentication_token_field :token, encrypted: :required + + self.table_name = 'cluster_agent_tokens' + + belongs_to :agent, class_name: 'Clusters::Agent' + + before_save :ensure_token + end +end diff --git a/app/models/clusters/concerns/application_status.rb b/app/models/clusters/concerns/application_status.rb index 86d74ed7b1c..ee6290e613e 100644 --- a/app/models/clusters/concerns/application_status.rb +++ b/app/models/clusters/concerns/application_status.rb @@ -79,7 +79,7 @@ module Clusters transition [:scheduled] => :uninstalling end - before_transition any => [:scheduled] do |application, _| + before_transition any => [:scheduled, :installed, :uninstalled] do |application, _| application.status_reason = nil end diff --git a/app/models/concerns/ci/contextable.rb b/app/models/concerns/ci/contextable.rb index 10df5e1a8dc..fdca0ec696b 100644 --- a/app/models/concerns/ci/contextable.rb +++ b/app/models/concerns/ci/contextable.rb @@ -64,7 +64,7 @@ module Ci variables.append(key: 'CI_PIPELINE_TRIGGERED', value: 'true') if trigger_request variables.append(key: 'CI_NODE_INDEX', value: self.options[:instance].to_s) if self.options&.include?(:instance) - variables.append(key: 'CI_NODE_TOTAL', value: (self.options&.dig(:parallel) || 1).to_s) + variables.append(key: 'CI_NODE_TOTAL', value: ci_node_total_value.to_s) # legacy variables variables.append(key: 'CI_BUILD_NAME', value: name) @@ -96,5 +96,13 @@ module Ci def secret_project_variables(environment: persisted_environment) project.ci_variables_for(ref: git_ref, environment: environment) end + + private + + def ci_node_total_value + parallel = self.options&.dig(:parallel) + parallel = parallel.dig(:total) if parallel.is_a?(Hash) + parallel || 1 + end end end diff --git a/app/models/project.rb b/app/models/project.rb index 32f9b580b47..92d2c85e99a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -261,6 +261,7 @@ class Project < ApplicationRecord has_many :clusters, through: :cluster_project, class_name: 'Clusters::Cluster' has_many :kubernetes_namespaces, class_name: 'Clusters::KubernetesNamespace' has_many :management_clusters, class_name: 'Clusters::Cluster', foreign_key: :management_project_id, inverse_of: :management_project + has_many :cluster_agents, class_name: 'Clusters::Agent' has_many :prometheus_metrics has_many :prometheus_alerts, inverse_of: :project diff --git a/app/views/projects/commits/_commit.html.haml b/app/views/projects/commits/_commit.html.haml index ab1d855a6e0..33fedde0cd1 100644 --- a/app/views/projects/commits/_commit.html.haml +++ b/app/views/projects/commits/_commit.html.haml @@ -10,12 +10,13 @@ - ref = local_assigns.fetch(:ref) { merge_request&.source_branch } - commit = commit.present(current_user: current_user) - commit_status = commit.status_for(ref) +- collapsible = local_assigns.fetch(:collapsible, true) - link = commit_path(project, commit, merge_request: merge_request) - show_project_name = local_assigns.fetch(:show_project_name, false) -%li.commit.flex-row.js-toggle-container{ id: "commit-#{commit.short_id}" } +%li{ class: ["commit flex-row", ("js-toggle-container" if collapsible)], id: "commit-#{commit.short_id}" } .avatar-cell.d-none.d-sm-block = author_avatar(commit, size: 40, has_tooltip: false) @@ -29,7 +30,7 @@ %span.commit-row-message.d-inline.d-sm-none · = commit.short_id - - if commit.description? + - if commit.description? && collapsible %button.text-expander.js-toggle-button = sprite_icon('ellipsis_h', size: 12) @@ -41,7 +42,7 @@ = render_if_exists 'projects/commits/project_namespace', show_project_name: show_project_name, project: project - if commit.description? - %pre.commit-row-description.js-toggle-content.gl-mb-3 + %pre{ class: ["commit-row-description gl-mb-3", (collapsible ? "js-toggle-content" : "d-block")] } = preserve(markdown_field(commit, :description)) .commit-actions.flex-row diff --git a/app/views/projects/merge_requests/diffs/_commit_widget.html.haml b/app/views/projects/merge_requests/diffs/_commit_widget.html.haml index efc052ca791..c022d2c70d8 100644 --- a/app/views/projects/merge_requests/diffs/_commit_widget.html.haml +++ b/app/views/projects/merge_requests/diffs/_commit_widget.html.haml @@ -2,8 +2,10 @@ WARNING: Please keep changes up-to-date with the following files: - `assets/javascripts/diffs/components/commit_widget.vue` -#----------------------------------------------------------------- +- collapsible = local_assigns.fetch(:collapsible, true) + - if @commit - .info-well.d-none.d-sm-block.gl-mt-3 + .info-well.mw-100.mx-0 .well-segment %ul.blob-commit-info - = render 'projects/commits/commit', commit: @commit, merge_request: @merge_request, view_details: true + = render 'projects/commits/commit', commit: @commit, merge_request: @merge_request, view_details: true, collapsible: collapsible diff --git a/changelogs/unreleased/15356-make-it-possible-to-define-a-cartesian-product-matrix-for-build-job.yml b/changelogs/unreleased/15356-make-it-possible-to-define-a-cartesian-product-matrix-for-build-job.yml new file mode 100644 index 00000000000..edd86677789 --- /dev/null +++ b/changelogs/unreleased/15356-make-it-possible-to-define-a-cartesian-product-matrix-for-build-job.yml @@ -0,0 +1,5 @@ +--- +title: Define matrix builds for more complex pipelines +merge_request: 33705 +author: +type: added diff --git a/changelogs/unreleased/211762-show-full-commit-message-by-default-in-merge-request-diff.yml b/changelogs/unreleased/211762-show-full-commit-message-by-default-in-merge-request-diff.yml new file mode 100644 index 00000000000..8bdb7b45852 --- /dev/null +++ b/changelogs/unreleased/211762-show-full-commit-message-by-default-in-merge-request-diff.yml @@ -0,0 +1,5 @@ +--- +title: Show full commit message by default in merge request diff +merge_request: 27981 +author: Gilang Gumilar +type: changed diff --git a/changelogs/unreleased/227856-configure-emphasis-syntax.yml b/changelogs/unreleased/227856-configure-emphasis-syntax.yml new file mode 100644 index 00000000000..a0a64ff62af --- /dev/null +++ b/changelogs/unreleased/227856-configure-emphasis-syntax.yml @@ -0,0 +1,5 @@ +--- +title: Use _ character for emphasis and * for strong in Static Site Editor markdown syntax +merge_request: 36965 +author: +type: added diff --git a/changelogs/unreleased/cluster_agent.yml b/changelogs/unreleased/cluster_agent.yml new file mode 100644 index 00000000000..341552952c4 --- /dev/null +++ b/changelogs/unreleased/cluster_agent.yml @@ -0,0 +1,5 @@ +--- +title: Adds models and tables for cluster agent and cluster agent tokens +merge_request: 33228 +author: +type: other diff --git a/db/migrate/20200607223047_create_cluster_agents.rb b/db/migrate/20200607223047_create_cluster_agents.rb new file mode 100644 index 00000000000..50dd28562e4 --- /dev/null +++ b/db/migrate/20200607223047_create_cluster_agents.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class CreateClusterAgents < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless table_exists?(:cluster_agents) + with_lock_retries do + create_table :cluster_agents do |t| + t.timestamps_with_timezone null: false + t.belongs_to(:project, null: false, index: true, foreign_key: { on_delete: :cascade }) + t.text :name, null: false + + t.index [:project_id, :name], unique: true + end + end + end + + add_text_limit :cluster_agents, :name, 255 + end + + def down + with_lock_retries do + drop_table :cluster_agents + end + end +end diff --git a/db/migrate/20200607235435_create_cluster_agent_tokens.rb b/db/migrate/20200607235435_create_cluster_agent_tokens.rb new file mode 100644 index 00000000000..30c3ad30fa5 --- /dev/null +++ b/db/migrate/20200607235435_create_cluster_agent_tokens.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class CreateClusterAgentTokens < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + unless table_exists?(:cluster_agent_tokens) + create_table :cluster_agent_tokens do |t| + t.timestamps_with_timezone null: false + t.belongs_to :agent, null: false, index: true, foreign_key: { to_table: :cluster_agents, on_delete: :cascade } + t.text :token_encrypted, null: false + + t.index :token_encrypted, unique: true + end + end + + add_text_limit :cluster_agent_tokens, :token_encrypted, 255 + end + + def down + drop_table :cluster_agent_tokens + end +end diff --git a/db/structure.sql b/db/structure.sql index 5bf930652f5..851ec9c6575 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10374,6 +10374,42 @@ CREATE SEQUENCE public.ci_variables_id_seq ALTER SEQUENCE public.ci_variables_id_seq OWNED BY public.ci_variables.id; +CREATE TABLE public.cluster_agent_tokens ( + id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + agent_id bigint NOT NULL, + token_encrypted text NOT NULL, + CONSTRAINT check_c60daed227 CHECK ((char_length(token_encrypted) <= 255)) +); + +CREATE SEQUENCE public.cluster_agent_tokens_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE public.cluster_agent_tokens_id_seq OWNED BY public.cluster_agent_tokens.id; + +CREATE TABLE public.cluster_agents ( + id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + project_id bigint NOT NULL, + name text NOT NULL, + CONSTRAINT check_3498369510 CHECK ((char_length(name) <= 255)) +); + +CREATE SEQUENCE public.cluster_agents_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE public.cluster_agents_id_seq OWNED BY public.cluster_agents.id; + CREATE TABLE public.cluster_groups ( id integer NOT NULL, cluster_id integer NOT NULL, @@ -16572,6 +16608,10 @@ ALTER TABLE ONLY public.ci_triggers ALTER COLUMN id SET DEFAULT nextval('public. ALTER TABLE ONLY public.ci_variables ALTER COLUMN id SET DEFAULT nextval('public.ci_variables_id_seq'::regclass); +ALTER TABLE ONLY public.cluster_agent_tokens ALTER COLUMN id SET DEFAULT nextval('public.cluster_agent_tokens_id_seq'::regclass); + +ALTER TABLE ONLY public.cluster_agents ALTER COLUMN id SET DEFAULT nextval('public.cluster_agents_id_seq'::regclass); + ALTER TABLE ONLY public.cluster_groups ALTER COLUMN id SET DEFAULT nextval('public.cluster_groups_id_seq'::regclass); ALTER TABLE ONLY public.cluster_platforms_kubernetes ALTER COLUMN id SET DEFAULT nextval('public.cluster_platforms_kubernetes_id_seq'::regclass); @@ -17519,6 +17559,12 @@ ALTER TABLE ONLY public.ci_triggers ALTER TABLE ONLY public.ci_variables ADD CONSTRAINT ci_variables_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.cluster_agent_tokens + ADD CONSTRAINT cluster_agent_tokens_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY public.cluster_agents + ADD CONSTRAINT cluster_agents_pkey PRIMARY KEY (id); + ALTER TABLE ONLY public.cluster_groups ADD CONSTRAINT cluster_groups_pkey PRIMARY KEY (id); @@ -19021,6 +19067,14 @@ CREATE INDEX index_ci_triggers_on_project_id ON public.ci_triggers USING btree ( CREATE UNIQUE INDEX index_ci_variables_on_project_id_and_key_and_environment_scope ON public.ci_variables USING btree (project_id, key, environment_scope); +CREATE INDEX index_cluster_agent_tokens_on_agent_id ON public.cluster_agent_tokens USING btree (agent_id); + +CREATE UNIQUE INDEX index_cluster_agent_tokens_on_token_encrypted ON public.cluster_agent_tokens USING btree (token_encrypted); + +CREATE INDEX index_cluster_agents_on_project_id ON public.cluster_agents USING btree (project_id); + +CREATE UNIQUE INDEX index_cluster_agents_on_project_id_and_name ON public.cluster_agents USING btree (project_id, name); + CREATE UNIQUE INDEX index_cluster_groups_on_cluster_id_and_group_id ON public.cluster_groups USING btree (cluster_id, group_id); CREATE INDEX index_cluster_groups_on_group_id ON public.cluster_groups USING btree (group_id); @@ -21712,6 +21766,9 @@ ALTER TABLE ONLY public.group_custom_attributes ALTER TABLE ONLY public.requirements_management_test_reports ADD CONSTRAINT fk_rails_24cecc1e68 FOREIGN KEY (pipeline_id) REFERENCES public.ci_pipelines(id) ON DELETE SET NULL; +ALTER TABLE ONLY public.cluster_agents + ADD CONSTRAINT fk_rails_25e9fc2d5d FOREIGN KEY (project_id) REFERENCES public.projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY public.group_wiki_repositories ADD CONSTRAINT fk_rails_26f867598c FOREIGN KEY (group_id) REFERENCES public.namespaces(id) ON DELETE CASCADE; @@ -22510,6 +22567,9 @@ ALTER TABLE ONLY public.subscriptions ALTER TABLE ONLY public.operations_strategies ADD CONSTRAINT fk_rails_d183b6e6dd FOREIGN KEY (feature_flag_id) REFERENCES public.operations_feature_flags(id) ON DELETE CASCADE; +ALTER TABLE ONLY public.cluster_agent_tokens + ADD CONSTRAINT fk_rails_d1d26abc25 FOREIGN KEY (agent_id) REFERENCES public.cluster_agents(id) ON DELETE CASCADE; + ALTER TABLE ONLY public.requirements_management_test_reports ADD CONSTRAINT fk_rails_d1e8b498bf FOREIGN KEY (author_id) REFERENCES public.users(id) ON DELETE SET NULL; @@ -23756,6 +23816,8 @@ COPY "schema_migrations" (version) FROM STDIN; 20200605160806 20200605160836 20200605160851 +20200607223047 +20200607235435 20200608072931 20200608075553 20200608195222 diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 43b321382f6..996e58eddd2 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -3403,6 +3403,48 @@ Please be aware that semaphore_test_boosters reports usages statistics to the au You can then navigate to the **Jobs** tab of a new pipeline build and see your RSpec job split into three separate jobs. +#### Parallel `matrix` jobs + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15356) in GitLab 13.2. + +`matrix:` allows you to configure different variables for jobs that are running in parallel. +There can be from 2 to 50 jobs. + +Every job gets the same `CI_NODE_TOTAL` [environment variable](../variables/README.md#predefined-environment-variables) value, and a unique `CI_NODE_INDEX` value. + +```yaml +deploystacks: + stage: deploy + script: + - bin/deploy + parallel: + matrix: + - PROVIDER: aws + STACK: + - monitoring + - app1 + - app2 + - PROVIDER: ovh + STACK: [monitoring, backup, app] + - PROVIDER: [gcp, vultr] + STACK: [data, processing] +``` + +This generates 10 parallel `deploystacks` jobs, each with different values for `PROVIDER` and `STACK`: + +```plaintext +deploystacks 1/10 with PROVIDER=aws and STACK=monitoring +deploystacks 2/10 with PROVIDER=aws and STACK=app1 +deploystacks 3/10 with PROVIDER=aws and STACK=app2 +deploystacks 4/10 with PROVIDER=ovh and STACK=monitoring +deploystacks 5/10 with PROVIDER=ovh and STACK=backup +deploystacks 6/10 with PROVIDER=ovh and STACK=app +deploystacks 7/10 with PROVIDER=gcp and STACK=data +deploystacks 8/10 with PROVIDER=gcp and STACK=processing +deploystacks 9/10 with PROVIDER=vultr and STACK=data +deploystacks 10/10 with PROVIDER=vultr and STACK=processing +``` + ### `trigger` > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8997) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8. diff --git a/lib/api/api.rb b/lib/api/api.rb index a89dc0fa6fa..ef8c2b2cde5 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -237,6 +237,7 @@ module API mount ::API::Internal::Base mount ::API::Internal::Pages + mount ::API::Internal::Kubernetes route :any, '*path' do error!('404 Not Found', 404) diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb new file mode 100644 index 00000000000..567b478ff41 --- /dev/null +++ b/lib/api/internal/kubernetes.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module API + # Kubernetes Internal API + module Internal + class Kubernetes < Grape::API::Instance + helpers do + def repo_type + Gitlab::GlRepository::PROJECT + end + + def gl_repository(project) + repo_type.identifier_for_container(project) + end + + def gl_repository_path(project) + repo_type.repository_for(project).full_path + end + + def check_feature_enabled + not_found! unless Feature.enabled?(:kubernetes_agent_internal_api) + end + end + + namespace 'internal' do + namespace 'kubernetes' do + desc 'Gets agent info' do + detail 'Retrieves agent info for the given token' + end + route_setting :authentication, cluster_agent_token_allowed: true + get '/agent_info' do + check_feature_enabled + + agent_token = cluster_agent_token_from_authorization_token + + if agent_token + agent = agent_token.agent + project = agent.project + @gl_project_string = "project-#{project.id}" + + status 200 + { + project_id: project.id, + agent_id: agent.id, + agent_name: agent.name, + storage_name: project.repository_storage, + relative_path: project.disk_path + '.git', + gl_repository: gl_repository(project), + gl_project_path: gl_repository_path(project) + } + else + status 403 + end + end + + desc 'Gets project info' do + detail 'Retrieves project info (if authorized)' + end + route_setting :authentication, cluster_agent_token_allowed: true + get '/project_info' do + check_feature_enabled + + agent_token = cluster_agent_token_from_authorization_token + + if agent_token + project = find_project(params[:id]) + + # TODO sort out authorization for real + # https://gitlab.com/gitlab-org/gitlab/-/issues/220912 + if !project || !project.public? + not_found! + end + + @gl_project_string = "project-#{project.id}" + + status 200 + { + project_id: project.id, + storage_name: project.repository_storage, + relative_path: project.disk_path + '.git', + gl_repository: gl_repository(project), + gl_project_path: gl_repository_path(project) + } + else + status 403 + end + end + end + end + end + end +end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index bd5aed0d964..ae38dc287c2 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -20,6 +20,7 @@ module Gitlab module AuthFinders include Gitlab::Utils::StrongMemoize include ActionController::HttpAuthentication::Basic + include ActionController::HttpAuthentication::Token PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN' PRIVATE_TOKEN_PARAM = :private_token @@ -131,6 +132,15 @@ module Gitlab deploy_token end + def cluster_agent_token_from_authorization_token + return unless route_authentication_setting[:cluster_agent_token_allowed] + return unless current_request.authorization.present? + + authorization_token, _options = token_and_options(current_request) + + ::Clusters::AgentToken.find_by_token(authorization_token) + end + def find_runner_from_token return unless api_request? diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a615cab1a80..4a18f8ac8b5 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -32,9 +32,6 @@ module Gitlab with_options allow_nil: true do validates :allow_failure, boolean: true - validates :parallel, numericality: { only_integer: true, - greater_than_or_equal_to: 2, - less_than_or_equal_to: 50 } validates :when, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" @@ -124,6 +121,10 @@ module Gitlab description: 'This job will produce a release.', inherit: false + entry :parallel, Entry::Product::Parallel, + description: 'Parallel configuration for this job.', + inherit: false + attributes :script, :tags, :allow_failure, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, :resource_group, :release @@ -174,7 +175,7 @@ module Gitlab environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, - parallel: has_parallel? ? parallel.to_i : nil, + parallel: has_parallel? ? parallel_value : nil, interruptible: interruptible_defined? ? interruptible_value : nil, timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil, artifacts: artifacts_value, diff --git a/lib/gitlab/ci/config/entry/product/matrix.rb b/lib/gitlab/ci/config/entry/product/matrix.rb new file mode 100644 index 00000000000..6af809d46c1 --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/matrix.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents matrix style parallel builds. + # + module Product + class Matrix < ::Gitlab::Config::Entry::Node + include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + validations do + validates :config, array_of_hashes: true + + validate on: :composed do + limit = Entry::Product::Parallel::PARALLEL_LIMIT + + if number_of_generated_jobs > limit + errors.add(:config, "generates too many jobs (maximum is #{limit})") + end + end + end + + def compose!(deps = nil) + super(deps) do + @config.each_with_index do |variables, index| + @entries[index] = ::Gitlab::Config::Entry::Factory.new(Entry::Product::Variables) + .value(variables) + .with(parent: self, description: 'matrix variables definition.') # rubocop:disable CodeReuse/ActiveRecord + .create! + end + + @entries.each_value do |entry| + entry.compose!(deps) + end + end + end + + def value + strong_memoize(:value) do + @entries.values.map(&:value) + end + end + + # rubocop:disable CodeReuse/ActiveRecord + def number_of_generated_jobs + value.sum do |config| + config.values.reduce(1) { |acc, values| acc * values.size } + end + end + # rubocop:enable CodeReuse/ActiveRecord + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb new file mode 100644 index 00000000000..da7079a8328 --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a parallel job config. + # + module Product + class Parallel < ::Gitlab::Config::Entry::Simplifiable + strategy :ParallelBuilds, if: -> (config) { config.is_a?(Numeric) } + strategy :MatrixBuilds, if: -> (config) { ::Gitlab::Ci::Features.parallel_matrix_enabled? && config.is_a?(Hash) } + + PARALLEL_LIMIT = 50 + + class ParallelBuilds < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, numericality: { only_integer: true, + greater_than_or_equal_to: 2, + less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT }, + allow_nil: true + end + + def value + { number: super.to_i } + end + end + + class MatrixBuilds < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable + + PERMITTED_KEYS = %i[matrix].freeze + + validations do + validates :config, allowed_keys: PERMITTED_KEYS + validates :config, required_keys: PERMITTED_KEYS + end + + entry :matrix, Entry::Product::Matrix, + description: 'Variables definition for matrix builds' + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be an integer or a hash"] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb new file mode 100644 index 00000000000..ac4f70fb69e --- /dev/null +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents variables for parallel matrix builds. + # + module Product + class Variables < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, variables: { array_values: true } + validates :config, length: { + minimum: 2, + too_short: 'requires at least %{count} items' + } + end + + def self.default(**) + {} + end + + def value + @config + .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } + .to_h + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb index 1139efee9e8..451ba14bb89 100644 --- a/lib/gitlab/ci/config/normalizer.rb +++ b/lib/gitlab/ci/config/normalizer.rb @@ -32,7 +32,7 @@ module Gitlab return unless job_names job_names.flat_map do |job_name| - parallelized_jobs[job_name.to_sym] || job_name + parallelized_jobs[job_name.to_sym]&.map(&:name) || job_name end end @@ -42,10 +42,8 @@ module Gitlab job_needs.flat_map do |job_need| job_need_name = job_need[:name].to_sym - if all_job_names = parallelized_jobs[job_need_name] - all_job_names.map do |job_name| - job_need.merge(name: job_name) - end + if all_jobs = parallelized_jobs[job_need_name] + all_jobs.map { |job| job_need.merge(name: job.name) } else job_need end @@ -57,7 +55,7 @@ module Gitlab @jobs_config.each_with_object({}) do |(job_name, config), hash| next unless config[:parallel] - hash[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + hash[job_name] = parallelize_job_config(job_name, config[:parallel]) end end end @@ -65,9 +63,9 @@ module Gitlab def expand_parallelize_jobs @jobs_config.each_with_object({}) do |(job_name, config), hash| if parallelized_jobs.key?(job_name) - parallelized_jobs[job_name].each_with_index do |name, index| - hash[name.to_sym] = - yield(name, config.merge(name: name, instance: index + 1)) + parallelized_jobs[job_name].each do |job| + hash[job.name.to_sym] = + yield(job.name, config.deep_merge(job.attributes)) end else hash[job_name] = yield(job_name, config) @@ -75,8 +73,8 @@ module Gitlab end end - def self.parallelize_job_names(name, total) - Array.new(total) { |index| "#{name} #{index + 1}/#{total}" } + def parallelize_job_config(name, config) + Normalizer::Factory.new(name, config).create end end end diff --git a/lib/gitlab/ci/config/normalizer/factory.rb b/lib/gitlab/ci/config/normalizer/factory.rb new file mode 100644 index 00000000000..972da4bbf9a --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/factory.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class Factory + include Gitlab::Utils::StrongMemoize + + def initialize(name, config) + @name = name + @config = config + end + + def create + return [] unless strategy + + strategy.build_from(@name, @config) + end + + private + + def strategy + strong_memoize(:strategy) do + strategies.find do |strategy| + strategy.applies_to?(@config) + end + end + end + + def strategies + if ::Gitlab::Ci::Features.parallel_matrix_enabled? + [NumberStrategy, MatrixStrategy] + else + [NumberStrategy] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb new file mode 100644 index 00000000000..db21274a9ed --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class MatrixStrategy + class << self + def applies_to?(config) + config.is_a?(Hash) && config.key?(:matrix) + end + + def build_from(job_name, initial_config) + config = expand(initial_config[:matrix]) + total = config.size + + config.map.with_index do |vars, index| + new(job_name, index.next, vars, total) + end + end + + # rubocop: disable CodeReuse/ActiveRecord + def expand(config) + config.flat_map do |config| + values = config.values + + values[0] + .product(*values.from(1)) + .map { |vals| config.keys.zip(vals).to_h } + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + + def initialize(job_name, instance, variables, total) + @job_name = job_name + @instance = instance + @variables = variables.to_h + @total = total + end + + def attributes + { + name: name, + instance: instance, + variables: variables, + parallel: { total: total } + } + end + + def name_with_details + vars = variables.map { |key, value| "#{key}=#{value}"}.join('; ') + + "#{job_name} (#{vars})" + end + + def name + "#{job_name} #{instance}/#{total}" + end + + private + + attr_reader :job_name, :instance, :variables, :total + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/number_strategy.rb b/lib/gitlab/ci/config/normalizer/number_strategy.rb new file mode 100644 index 00000000000..4754e7b46d4 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer/number_strategy.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + class NumberStrategy + class << self + def applies_to?(config) + config.is_a?(Integer) || config.is_a?(Hash) && config.key?(:number) + end + + def build_from(job_name, config) + total = config.is_a?(Hash) ? config[:number] : config + + Array.new(total) do |index| + new(job_name, index.next, total) + end + end + end + + def initialize(job_name, instance, total) + @job_name = job_name + @instance = instance + @total = total + end + + def attributes + { + name: name, + instance: instance, + parallel: { total: total } + } + end + + def name + "#{job_name} #{instance}/#{total}" + end + + private + + attr_reader :job_name, :instance, :total + end + end + end + end +end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 6130baeb9d5..3123264ed9d 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -66,6 +66,10 @@ module Gitlab ::Feature.enabled?(:destroy_only_unlocked_expired_artifacts, default_enabled: false) end + def self.parallel_matrix_enabled? + ::Feature.enabled?(:ci_parallel_matrix_enabled, default_enabled: true) + end + def self.bulk_insert_on_create?(project) ::Feature.enabled?(:ci_bulk_insert_on_create, project, default_enabled: true) end diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb index 0a629075302..ea5c887552e 100644 --- a/lib/gitlab/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb @@ -30,10 +30,18 @@ module Gitlab end def validate_variables(variables) + variables.is_a?(Hash) && variables.flatten.all?(&method(:validate_alphanumeric)) + end + + def validate_array_value_variables(variables) variables.is_a?(Hash) && - variables.flatten.all? do |value| - validate_string(value) || validate_integer(value) - end + variables.keys.all?(&method(:validate_alphanumeric)) && + variables.values.all?(&:present?) && + variables.values.flatten(1).all?(&method(:validate_alphanumeric)) + end + + def validate_alphanumeric(value) + validate_string(value) || validate_integer(value) end def validate_integer(value) diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index d1c23c41d35..813ec9126f0 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -272,10 +272,24 @@ module Gitlab include LegacyValidationHelpers def validate_each(record, attribute, value) + if options[:array_values] + validate_key_array_values(record, attribute, value) + else + validate_key_values(record, attribute, value) + end + end + + def validate_key_values(record, attribute, value) unless validate_variables(value) record.errors.add(attribute, 'should be a hash of key value pairs') end end + + def validate_key_array_values(record, attribute, value) + unless validate_array_value_variables(value) + record.errors.add(attribute, 'should be a hash of key value pairs, value can be an array') + end + end end class ExpressionValidator < ActiveModel::EachValidator diff --git a/spec/factories/clusters/agent_tokens.rb b/spec/factories/clusters/agent_tokens.rb new file mode 100644 index 00000000000..6f92f2217b3 --- /dev/null +++ b/spec/factories/clusters/agent_tokens.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :cluster_agent_token, class: 'Clusters::AgentToken' do + association :agent, factory: :cluster_agent + + token_encrypted { Gitlab::CryptoHelper.aes256_gcm_encrypt(SecureRandom.hex(50)) } + end +end diff --git a/spec/factories/clusters/agents.rb b/spec/factories/clusters/agents.rb new file mode 100644 index 00000000000..493b73d1688 --- /dev/null +++ b/spec/factories/clusters/agents.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :cluster_agent, class: 'Clusters::Agent' do + project + + sequence(:name) { |n| "agent_#{n}" } + end +end diff --git a/spec/features/merge_requests/user_views_diffs_commit_spec.rb b/spec/features/merge_requests/user_views_diffs_commit_spec.rb new file mode 100644 index 00000000000..590397df73f --- /dev/null +++ b/spec/features/merge_requests/user_views_diffs_commit_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'User views diff by commit', :js do + let(:merge_request) do + create(:merge_request_with_diffs, source_project: project, target_project: project, source_branch: 'merge-test') + end + let(:project) { create(:project, :public, :repository) } + + before do + stub_feature_flags(diffs_batch_load: false) + visit(diffs_project_merge_request_path(project, merge_request, commit_id: merge_request.diff_head_sha)) + end + + it 'shows full commit description by default' do + expect(page).to have_selector('.commit-row-description', visible: true) + end +end diff --git a/spec/frontend/vue_shared/components/rich_content_editor/services/build_html_to_markdown_renderer_spec.js b/spec/frontend/vue_shared/components/rich_content_editor/services/build_html_to_markdown_renderer_spec.js index c11b0234e91..218ca8f3f5a 100644 --- a/spec/frontend/vue_shared/components/rich_content_editor/services/build_html_to_markdown_renderer_spec.js +++ b/spec/frontend/vue_shared/components/rich_content_editor/services/build_html_to_markdown_renderer_spec.js @@ -67,4 +67,44 @@ describe('HTMLToMarkdownRenderer', () => { }, ); }); + + describe('STRONG, B visitor', () => { + it.each` + input | strongCharacter | result + ${'**strong text**'} | ${'_'} | ${'__strong text__'} + ${'__strong text__'} | ${'*'} | ${'**strong text**'} + `( + 'converts $input to $result when strong character is $strongCharacter', + ({ input, strongCharacter, result }) => { + htmlToMarkdownRenderer = buildHTMLToMarkdownRenderer(baseRenderer, { + strong: strongCharacter, + }); + + baseRenderer.convert.mockReturnValueOnce(input); + + expect(htmlToMarkdownRenderer['STRONG, B'](NODE, input)).toBe(result); + expect(baseRenderer.convert).toHaveBeenCalledWith(NODE, input); + }, + ); + }); + + describe('EM, I visitor', () => { + it.each` + input | emphasisCharacter | result + ${'*strong text*'} | ${'_'} | ${'_strong text_'} + ${'_strong text_'} | ${'*'} | ${'*strong text*'} + `( + 'converts $input to $result when emphasis character is $emphasisCharacter', + ({ input, emphasisCharacter, result }) => { + htmlToMarkdownRenderer = buildHTMLToMarkdownRenderer(baseRenderer, { + emphasis: emphasisCharacter, + }); + + baseRenderer.convert.mockReturnValueOnce(input); + + expect(htmlToMarkdownRenderer['EM, I'](NODE, input)).toBe(result); + expect(baseRenderer.convert).toHaveBeenCalledWith(NODE, input); + }, + ); + }); }); diff --git a/spec/lib/gitlab/auth/auth_finders_spec.rb b/spec/lib/gitlab/auth/auth_finders_spec.rb index d0f5d0a9b35..7ec4bb65613 100644 --- a/spec/lib/gitlab/auth/auth_finders_spec.rb +++ b/spec/lib/gitlab/auth/auth_finders_spec.rb @@ -744,6 +744,56 @@ RSpec.describe Gitlab::Auth::AuthFinders do end end + describe '#cluster_agent_token_from_authorization_token' do + let_it_be(:agent_token) { create(:cluster_agent_token) } + + context 'when route_setting is empty' do + it 'returns nil' do + expect(cluster_agent_token_from_authorization_token).to be_nil + end + end + + context 'when route_setting allows cluster agent token' do + let(:route_authentication_setting) { { cluster_agent_token_allowed: true } } + + context 'Authorization header is empty' do + it 'returns nil' do + expect(cluster_agent_token_from_authorization_token).to be_nil + end + end + + context 'Authorization header is incorrect' do + before do + request.headers['Authorization'] = 'Bearer ABCD' + end + + it 'returns nil' do + expect(cluster_agent_token_from_authorization_token).to be_nil + end + end + + context 'Authorization header is malformed' do + before do + request.headers['Authorization'] = 'Bearer' + end + + it 'returns nil' do + expect(cluster_agent_token_from_authorization_token).to be_nil + end + end + + context 'Authorization header matches agent token' do + before do + request.headers['Authorization'] = "Bearer #{agent_token.token}" + end + + it 'returns the agent token' do + expect(cluster_agent_token_from_authorization_token).to eq(agent_token) + end + end + end + end + describe '#find_runner_from_token' do let(:runner) { create(:ci_runner) } diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 180c52ee1ab..9b1212fee44 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -30,7 +30,7 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do %i[before_script script stage type after_script cache image services only except rules needs variables artifacts environment coverage retry interruptible timeout release tags - inherit] + inherit parallel] end it { is_expected.to include(*result) } @@ -202,56 +202,47 @@ RSpec.describe Gitlab::Ci::Config::Entry::Job do context 'when parallel value is not correct' do context 'when it is not a numeric value' do - let(:config) { { parallel: true } } + let(:config) { { script: 'echo', parallel: true } } it 'returns error about invalid type' do expect(entry).not_to be_valid - expect(entry.errors).to include 'job parallel is not a number' + expect(entry.errors).to include 'parallel should be an integer or a hash' end end context 'when it is lower than two' do - let(:config) { { parallel: 1 } } + let(:config) { { script: 'echo', parallel: 1 } } it 'returns error about value too low' do expect(entry).not_to be_valid expect(entry.errors) - .to include 'job parallel must be greater than or equal to 2' + .to include 'parallel config must be greater than or equal to 2' end end - context 'when it is bigger than 50' do - let(:config) { { parallel: 51 } } + context 'when it is an empty hash' do + let(:config) { { script: 'echo', parallel: {} } } - it 'returns error about value too high' do + it 'returns error about missing matrix' do expect(entry).not_to be_valid expect(entry.errors) - .to include 'job parallel must be less than or equal to 50' + .to include 'parallel config missing required keys: matrix' end end + end - context 'when it is not an integer' do - let(:config) { { parallel: 1.5 } } - - it 'returns error about wrong value' do - expect(entry).not_to be_valid - expect(entry.errors).to include 'job parallel must be an integer' - end + context 'when it uses both "when:" and "rules:"' do + let(:config) do + { + script: 'echo', + when: 'on_failure', + rules: [{ if: '$VARIABLE', when: 'on_success' }] + } end - context 'when it uses both "when:" and "rules:"' do - let(:config) do - { - script: 'echo', - when: 'on_failure', - rules: [{ if: '$VARIABLE', when: 'on_success' }] - } - end - - it 'returns an error about when: being combined with rules' do - expect(entry).not_to be_valid - expect(entry.errors).to include 'job config key may not be used with `rules`: when' - end + it 'returns an error about when: being combined with rules' do + expect(entry).not_to be_valid + expect(entry.errors).to include 'job config key may not be used with `rules`: when' end end diff --git a/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb b/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb new file mode 100644 index 00000000000..39697884e3b --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/product/matrix_spec.rb @@ -0,0 +1,188 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'active_model' + +RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Matrix do + subject(:matrix) { described_class.new(config) } + + describe 'validations' do + before do + matrix.compose! + end + + context 'when entry config value is correct' do + let(:config) do + [ + { 'VAR_1' => [1, 2, 3], 'VAR_2' => [4, 5, 6] }, + { 'VAR_3' => %w[a b], 'VAR_4' => %w[c d] } + ] + end + + describe '#valid?' do + it { is_expected.to be_valid } + end + end + + context 'when entry config generates too many jobs' do + let(:config) do + [ + { + 'VAR_1' => (1..10).to_a, + 'VAR_2' => (11..20).to_a + } + ] + end + + describe '#valid?' do + it { is_expected.not_to be_valid } + end + + describe '#errors' do + it 'returns error about too many jobs' do + expect(matrix.errors) + .to include('matrix config generates too many jobs (maximum is 50)') + end + end + end + + context 'when entry config has only one variable' do + let(:config) do + [ + { + 'VAR_1' => %w[test] + } + ] + end + + describe '#valid?' do + it { is_expected.not_to be_valid } + end + + describe '#errors' do + it 'returns error about too many jobs' do + expect(matrix.errors) + .to include('variables config requires at least 2 items') + end + end + + describe '#value' do + before do + matrix.compose! + end + + it 'returns the value without raising an error' do + expect(matrix.value).to eq([{ 'VAR_1' => ['test'] }]) + end + end + end + + context 'when config value has wrong type' do + let(:config) { {} } + + describe '#valid?' do + it { is_expected.not_to be_valid } + end + + describe '#errors' do + it 'returns error about incorrect type' do + expect(matrix.errors) + .to include('matrix config should be an array of hashes') + end + end + end + end + + describe '.compose!' do + context 'when valid job entries composed' do + let(:config) do + [ + { PROVIDER: 'aws', STACK: %w[monitoring app1 app2] }, + { STACK: %w[monitoring backup app], PROVIDER: 'ovh' }, + { PROVIDER: 'gcp', STACK: %w[data processing], ARGS: 'normal' }, + { PROVIDER: 'vultr', STACK: 'data', ARGS: 'store' } + ] + end + + before do + matrix.compose! + end + + describe '#value' do + it 'returns key value' do + expect(matrix.value).to match( + [ + { 'PROVIDER' => %w[aws], 'STACK' => %w[monitoring app1 app2] }, + { 'PROVIDER' => %w[ovh], 'STACK' => %w[monitoring backup app] }, + { 'ARGS' => %w[normal], 'PROVIDER' => %w[gcp], 'STACK' => %w[data processing] }, + { 'ARGS' => %w[store], 'PROVIDER' => %w[vultr], 'STACK' => %w[data] } + ] + ) + end + end + + describe '#descendants' do + it 'creates valid descendant nodes' do + expect(matrix.descendants.count).to eq(config.size) + expect(matrix.descendants) + .to all(be_an_instance_of(::Gitlab::Ci::Config::Entry::Product::Variables)) + end + end + end + + context 'with empty config' do + let(:config) { [] } + + before do + matrix.compose! + end + + describe '#value' do + it 'returns empty value' do + expect(matrix.value).to eq([]) + end + end + end + end + + describe '#number_of_generated_jobs' do + before do + matrix.compose! + end + + subject { matrix.number_of_generated_jobs } + + context 'with empty config' do + let(:config) { [] } + + it { is_expected.to be_zero } + end + + context 'with only one variable' do + let(:config) do + [{ 'VAR_1' => (1..10).to_a }] + end + + it { is_expected.to eq(10) } + end + + context 'with two variables' do + let(:config) do + [{ 'VAR_1' => (1..10).to_a, 'VAR_2' => (1..5).to_a }] + end + + it { is_expected.to eq(50) } + end + + context 'with two sets of variables' do + let(:config) do + [ + { 'VAR_1' => (1..10).to_a, 'VAR_2' => (1..5).to_a }, + { 'VAR_3' => (1..2).to_a, 'VAR_4' => (1..3).to_a } + ] + end + + it { is_expected.to eq(56) } + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb new file mode 100644 index 00000000000..bc09e20d748 --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/product/parallel_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'active_model' + +RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do + subject(:parallel) { described_class.new(config) } + + context 'with invalid config' do + shared_examples 'invalid config' do |error_message| + describe '#valid?' do + it { is_expected.not_to be_valid } + end + + describe '#errors' do + it 'returns error about invalid type' do + expect(parallel.errors).to match(a_collection_including(error_message)) + end + end + end + + context 'when it is not a numeric value' do + let(:config) { true } + + it_behaves_like 'invalid config', /should be an integer or a hash/ + end + + context 'when it is lower than two' do + let(:config) { 1 } + + it_behaves_like 'invalid config', /must be greater than or equal to 2/ + end + + context 'when it is bigger than 50' do + let(:config) { 51 } + + it_behaves_like 'invalid config', /must be less than or equal to 50/ + end + + context 'when it is not an integer' do + let(:config) { 1.5 } + + it_behaves_like 'invalid config', /must be an integer/ + end + + context 'with empty hash config' do + let(:config) { {} } + + it_behaves_like 'invalid config', /matrix builds config missing required keys: matrix/ + end + end + + context 'with numeric config' do + context 'when job is specified' do + let(:config) { 2 } + + describe '#valid?' do + it { is_expected.to be_valid } + end + + describe '#value' do + it 'returns job needs configuration' do + expect(parallel.value).to match(number: config) + end + end + end + end + + context 'with matrix builds config' do + context 'when matrix is specified' do + let(:config) do + { + matrix: [ + { PROVIDER: 'aws', STACK: %w[monitoring app1 app2] }, + { PROVIDER: 'gcp', STACK: %w[data processing] } + ] + } + end + + describe '#valid?' do + it { is_expected.to be_valid } + end + + describe '#value' do + it 'returns job needs configuration' do + expect(parallel.value).to match(matrix: [ + { PROVIDER: 'aws', STACK: %w[monitoring app1 app2] }, + { PROVIDER: 'gcp', STACK: %w[data processing] } + ]) + end + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/entry/product/variables_spec.rb b/spec/lib/gitlab/ci/config/entry/product/variables_spec.rb new file mode 100644 index 00000000000..230b001d620 --- /dev/null +++ b/spec/lib/gitlab/ci/config/entry/product/variables_spec.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' +require_dependency 'active_model' + +RSpec.describe Gitlab::Ci::Config::Entry::Product::Variables do + let(:entry) { described_class.new(config) } + + describe 'validations' do + context 'when entry config value is correct' do + let(:config) do + { + 'VARIABLE_1' => 1, + 'VARIABLE_2' => 'value 2', + 'VARIABLE_3' => :value_3, + :VARIABLE_4 => 'value 4', + 5 => ['value 5'], + 'VARIABLE_6' => ['value 6'] + } + end + + describe '#value' do + it 'returns hash with key value strings' do + expect(entry.value).to match({ + 'VARIABLE_1' => ['1'], + 'VARIABLE_2' => ['value 2'], + 'VARIABLE_3' => ['value_3'], + 'VARIABLE_4' => ['value 4'], + '5' => ['value 5'], + 'VARIABLE_6' => ['value 6'] + }) + end + end + + describe '#errors' do + it 'does not append errors' do + expect(entry.errors).to be_empty + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'when entry value is not correct' do + shared_examples 'invalid variables' do |message| + describe '#errors' do + it 'saves errors' do + expect(entry.errors).to include(message) + end + end + + describe '#valid?' do + it 'is not valid' do + expect(entry).not_to be_valid + end + end + end + + context 'with array' do + let(:config) { [:VAR, 'test'] } + + it_behaves_like 'invalid variables', /should be a hash of key value pairs/ + end + + context 'with empty array' do + let(:config) { { VAR: 'test', VAR2: [] } } + + it_behaves_like 'invalid variables', /should be a hash of key value pairs/ + end + + context 'with nested array' do + let(:config) { { VAR: 'test', VAR2: [1, [2]] } } + + it_behaves_like 'invalid variables', /should be a hash of key value pairs/ + end + + context 'with only one variable' do + let(:config) { { VAR: 'test' } } + + it_behaves_like 'invalid variables', /variables config requires at least 2 items/ + end + end + end +end diff --git a/spec/lib/gitlab/ci/config/normalizer/factory_spec.rb b/spec/lib/gitlab/ci/config/normalizer/factory_spec.rb new file mode 100644 index 00000000000..e355740222f --- /dev/null +++ b/spec/lib/gitlab/ci/config/normalizer/factory_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Config::Normalizer::Factory do + describe '#create' do + context 'when no strategy applies' do + subject(:subject) { described_class.new(nil, nil).create } # rubocop:disable Rails/SaveBang + + it { is_expected.to be_empty } + end + end +end diff --git a/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb b/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb new file mode 100644 index 00000000000..bab604c4504 --- /dev/null +++ b/spec/lib/gitlab/ci/config/normalizer/matrix_strategy_spec.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Config::Normalizer::MatrixStrategy do + describe '.applies_to?' do + subject { described_class.applies_to?(config) } + + context 'with hash that has :matrix key' do + let(:config) { { matrix: [] } } + + it { is_expected.to be_truthy } + end + + context 'with hash that does not have :matrix key' do + let(:config) { { number: [] } } + + it { is_expected.to be_falsey } + end + + context 'with a number' do + let(:config) { 5 } + + it { is_expected.to be_falsey } + end + end + + describe '.build_from' do + subject { described_class.build_from('test', config) } + + let(:config) do + { + matrix: [ + { 'PROVIDER' => %w[aws], 'STACK' => %w[app1 app2] }, + { 'PROVIDER' => %w[ovh gcp], 'STACK' => %w[app] } + ] + } + end + + it { expect(subject.size).to eq(4) } + + it 'has attributes' do + expect(subject.map(&:attributes)).to match_array( + [ + { + name: 'test 1/4', + instance: 1, + parallel: { total: 4 }, + variables: { + 'PROVIDER' => 'aws', + 'STACK' => 'app1' + } + }, + { + name: 'test 2/4', + instance: 2, + parallel: { total: 4 }, + variables: { + 'PROVIDER' => 'aws', + 'STACK' => 'app2' + } + }, + { + name: 'test 3/4', + instance: 3, + parallel: { total: 4 }, + variables: { + 'PROVIDER' => 'ovh', + 'STACK' => 'app' + } + }, + { + name: 'test 4/4', + instance: 4, + parallel: { total: 4 }, + variables: { + 'PROVIDER' => 'gcp', + 'STACK' => 'app' + } + } + ] + ) + end + + it 'has parallelized name' do + expect(subject.map(&:name)).to match_array( + ['test 1/4', 'test 2/4', 'test 3/4', 'test 4/4'] + ) + end + + it 'has details' do + expect(subject.map(&:name_with_details)).to match_array( + [ + 'test (PROVIDER=aws; STACK=app1)', + 'test (PROVIDER=aws; STACK=app2)', + 'test (PROVIDER=gcp; STACK=app)', + 'test (PROVIDER=ovh; STACK=app)' + ] + ) + end + end +end diff --git a/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb b/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb new file mode 100644 index 00000000000..06f47fe11c6 --- /dev/null +++ b/spec/lib/gitlab/ci/config/normalizer/number_strategy_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'fast_spec_helper' + +RSpec.describe Gitlab::Ci::Config::Normalizer::NumberStrategy do + describe '.applies_to?' do + subject { described_class.applies_to?(config) } + + context 'with numbers' do + let(:config) { 5 } + + it { is_expected.to be_truthy } + end + + context 'with hash that has :number key' do + let(:config) { { number: 5 } } + + it { is_expected.to be_truthy } + end + + context 'with a float number' do + let(:config) { 5.5 } + + it { is_expected.to be_falsey } + end + + context 'with hash that does not have :number key' do + let(:config) { { matrix: 5 } } + + it { is_expected.to be_falsey } + end + end + + describe '.build_from' do + subject { described_class.build_from('test', config) } + + shared_examples 'parallelized job' do + it { expect(subject.size).to eq(3) } + + it 'has attributes' do + expect(subject.map(&:attributes)).to match_array( + [ + { name: 'test 1/3', instance: 1, parallel: { total: 3 } }, + { name: 'test 2/3', instance: 2, parallel: { total: 3 } }, + { name: 'test 3/3', instance: 3, parallel: { total: 3 } } + ] + ) + end + + it 'has parallelized name' do + expect(subject.map(&:name)).to match_array( + ['test 1/3', 'test 2/3', 'test 3/3']) + end + end + + context 'with numbers' do + let(:config) { 3 } + + it_behaves_like 'parallelized job' + end + + context 'with hash that has :number key' do + let(:config) { { number: 3 } } + + it_behaves_like 'parallelized job' + end + end +end diff --git a/spec/lib/gitlab/ci/config/normalizer_spec.rb b/spec/lib/gitlab/ci/config/normalizer_spec.rb index d3d165ba00f..949af8cdc4c 100644 --- a/spec/lib/gitlab/ci/config/normalizer_spec.rb +++ b/spec/lib/gitlab/ci/config/normalizer_spec.rb @@ -4,66 +4,13 @@ require 'fast_spec_helper' RSpec.describe Gitlab::Ci::Config::Normalizer do let(:job_name) { :rspec } - let(:job_config) { { script: 'rspec', parallel: 5, name: 'rspec' } } + let(:job_config) { { script: 'rspec', parallel: parallel_config, name: 'rspec', variables: variables_config } } let(:config) { { job_name => job_config } } - let(:expanded_job_names) do - [ - "rspec 1/5", - "rspec 2/5", - "rspec 3/5", - "rspec 4/5", - "rspec 5/5" - ] - end - describe '.normalize_jobs' do subject { described_class.new(config).normalize_jobs } - it 'does not have original job' do - is_expected.not_to include(job_name) - end - - it 'has parallelized jobs' do - is_expected.to include(*expanded_job_names.map(&:to_sym)) - end - - it 'sets job instance in options' do - expect(subject.values).to all(include(:instance)) - end - - it 'parallelizes jobs with original config' do - original_config = config[job_name].except(:name) - configs = subject.values.map { |config| config.except(:name, :instance) } - - expect(configs).to all(eq(original_config)) - end - - context 'when the job is not parallelized' do - let(:job_config) { { script: 'rspec', name: 'rspec' } } - - it 'returns the same hash' do - is_expected.to eq(config) - end - end - - context 'when there is a job with a slash in it' do - let(:job_name) { :"rspec 35/2" } - - it 'properly parallelizes job names' do - job_names = [ - :"rspec 35/2 1/5", - :"rspec 35/2 2/5", - :"rspec 35/2 3/5", - :"rspec 35/2 4/5", - :"rspec 35/2 5/5" - ] - - is_expected.to include(*job_names) - end - end - - context 'for dependencies' do + shared_examples 'parallel dependencies' do context "when job has dependencies on parallelized jobs" do let(:config) do { @@ -91,9 +38,7 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do end it "parallelizes dependencies" do - job_names = ["rspec 1/5", "rspec 2/5", "rspec 3/5", "rspec 4/5", "rspec 5/5"] - - expect(subject[:final_job][:dependencies]).to include(*job_names) + expect(subject[:final_job][:dependencies]).to include(*expanded_job_names) end it "includes the regular job in dependencies" do @@ -102,14 +47,14 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do end end - context 'for needs' do + shared_examples 'parallel needs' do let(:expanded_job_attributes) do expanded_job_names.map do |job_name| { name: job_name, extra: :key } end end - context "when job has needs on parallelized jobs" do + context 'when job has needs on parallelized jobs' do let(:config) do { job_name => job_config, @@ -124,12 +69,12 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do } end - it "parallelizes needs" do + it 'parallelizes needs' do expect(subject.dig(:other_job, :needs, :job)).to eq(expanded_job_attributes) end end - context "when there are dependencies which are both parallelized and not" do + context 'when there are dependencies which are both parallelized and not' do let(:config) do { job_name => job_config, @@ -141,21 +86,157 @@ RSpec.describe Gitlab::Ci::Config::Normalizer do needs: { job: [ { name: job_name.to_s, extra: :key }, - { name: "other_job", extra: :key } + { name: 'other_job', extra: :key } ] } } } end - it "parallelizes dependencies" do + it 'parallelizes dependencies' do expect(subject.dig(:final_job, :needs, :job)).to include(*expanded_job_attributes) end - it "includes the regular job in dependencies" do + it 'includes the regular job in dependencies' do expect(subject.dig(:final_job, :needs, :job)).to include(name: 'other_job', extra: :key) end end end + + context 'with parallel config as integer' do + let(:variables_config) { {} } + let(:parallel_config) { 5 } + + let(:expanded_job_names) do + [ + 'rspec 1/5', + 'rspec 2/5', + 'rspec 3/5', + 'rspec 4/5', + 'rspec 5/5' + ] + end + + it 'does not have original job' do + is_expected.not_to include(job_name) + end + + it 'has parallelized jobs' do + is_expected.to include(*expanded_job_names.map(&:to_sym)) + end + + it 'sets job instance in options' do + expect(subject.values).to all(include(:instance)) + end + + it 'parallelizes jobs with original config' do + original_config = config[job_name] + .except(:name) + .deep_merge(parallel: { total: parallel_config }) + + configs = subject.values.map { |config| config.except(:name, :instance) } + + expect(configs).to all(eq(original_config)) + end + + context 'when the job is not parallelized' do + let(:job_config) { { script: 'rspec', name: 'rspec' } } + + it 'returns the same hash' do + is_expected.to eq(config) + end + end + + context 'when there is a job with a slash in it' do + let(:job_name) { :"rspec 35/2" } + + it 'properly parallelizes job names' do + job_names = [ + :"rspec 35/2 1/5", + :"rspec 35/2 2/5", + :"rspec 35/2 3/5", + :"rspec 35/2 4/5", + :"rspec 35/2 5/5" + ] + + is_expected.to include(*job_names) + end + end + + it_behaves_like 'parallel dependencies' + it_behaves_like 'parallel needs' + end + + context 'with parallel matrix config' do + let(:variables_config) do + { + USER_VARIABLE: 'user value' + } + end + + let(:parallel_config) do + { + matrix: [ + { + VAR_1: [1], + VAR_2: [2, 3] + } + ] + } + end + + let(:expanded_job_names) do + [ + 'rspec 1/2', + 'rspec 2/2' + ] + end + + it 'does not have original job' do + is_expected.not_to include(job_name) + end + + it 'has parallelized jobs' do + is_expected.to include(*expanded_job_names.map(&:to_sym)) + end + + it 'sets job instance in options' do + expect(subject.values).to all(include(:instance)) + end + + it 'sets job variables', :aggregate_failures do + expect(subject.values[0]).to match( + a_hash_including(variables: { VAR_1: 1, VAR_2: 2, USER_VARIABLE: 'user value' }) + ) + + expect(subject.values[1]).to match( + a_hash_including(variables: { VAR_1: 1, VAR_2: 3, USER_VARIABLE: 'user value' }) + ) + end + + it 'parallelizes jobs with original config' do + configs = subject.values.map do |config| + config.except(:name, :instance, :variables) + end + + original_config = config[job_name] + .except(:name, :variables) + .deep_merge(parallel: { total: 2 }) + + expect(configs).to all(match(a_hash_including(original_config))) + end + + it_behaves_like 'parallel dependencies' + it_behaves_like 'parallel needs' + end + + context 'when parallel config does not matches a factory' do + let(:variables_config) { {} } + let(:parallel_config) { } + + it 'does not alter the job config' do + is_expected.to match(config) + end + end end end diff --git a/spec/lib/gitlab/ci/yaml_processor_spec.rb b/spec/lib/gitlab/ci/yaml_processor_spec.rb index 5c6d748d66c..5a2a374a39e 100644 --- a/spec/lib/gitlab/ci/yaml_processor_spec.rb +++ b/spec/lib/gitlab/ci/yaml_processor_spec.rb @@ -1269,27 +1269,104 @@ module Gitlab end describe 'Parallel' do + let(:config) do + YAML.dump(rspec: { script: 'rspec', + parallel: parallel, + variables: { 'VAR1' => 1 } }) + end + + let(:config_processor) { Gitlab::Ci::YamlProcessor.new(config) } + let(:builds) { config_processor.stage_builds_attributes('test') } + context 'when job is parallelized' do let(:parallel) { 5 } - let(:config) do - YAML.dump(rspec: { script: 'rspec', - parallel: parallel }) - end - it 'returns parallelized jobs' do - config_processor = Gitlab::Ci::YamlProcessor.new(config) - builds = config_processor.stage_builds_attributes('test') build_options = builds.map { |build| build[:options] } expect(builds.size).to eq(5) - expect(build_options).to all(include(:instance, parallel: parallel)) + expect(build_options).to all(include(:instance, parallel: { number: parallel, total: parallel })) end it 'does not have the original job' do - config_processor = Gitlab::Ci::YamlProcessor.new(config) - builds = config_processor.stage_builds_attributes('test') + expect(builds).not_to include(:rspec) + end + end + + context 'with build matrix' do + let(:parallel) do + { + matrix: [ + { 'PROVIDER' => 'aws', 'STACK' => %w[monitoring app1 app2] }, + { 'PROVIDER' => 'ovh', 'STACK' => %w[monitoring backup app] }, + { 'PROVIDER' => 'gcp', 'STACK' => %w[data processing] } + ] + } + end + + it 'returns the number of parallelized jobs' do + expect(builds.size).to eq(8) + end + + it 'returns the parallel config' do + build_options = builds.map { |build| build[:options] } + parallel_config = { + matrix: parallel[:matrix].map { |var| var.transform_values { |v| Array(v).flatten }}, + total: build_options.size + } + + expect(build_options).to all(include(:instance, parallel: parallel_config)) + end + it 'sets matrix variables' do + build_variables = builds.map { |build| build[:yaml_variables] } + expected_variables = [ + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'aws' }, + { key: 'STACK', value: 'monitoring' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'aws' }, + { key: 'STACK', value: 'app1' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'aws' }, + { key: 'STACK', value: 'app2' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'ovh' }, + { key: 'STACK', value: 'monitoring' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'ovh' }, + { key: 'STACK', value: 'backup' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'ovh' }, + { key: 'STACK', value: 'app' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'gcp' }, + { key: 'STACK', value: 'data' } + ], + [ + { key: 'VAR1', value: '1' }, + { key: 'PROVIDER', value: 'gcp' }, + { key: 'STACK', value: 'processing' } + ] + ].map { |vars| vars.map { |var| a_hash_including(var) } } + + expect(build_variables).to match(expected_variables) + end + + it 'does not have the original job' do expect(builds).not_to include(:rspec) end end @@ -2619,6 +2696,14 @@ module Gitlab .to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, 'rspec: unknown keys in `extends` (something)') end + + it 'returns errors if parallel is invalid' do + config = YAML.dump({ rspec: { parallel: 'test', script: 'test' } }) + + expect { Gitlab::Ci::YamlProcessor.new(config) } + .to raise_error(Gitlab::Ci::YamlProcessor::ValidationError, + 'jobs:rspec:parallel should be an integer or a hash') + end end describe "#validation_message" do diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 02500778426..4c3ef47da60 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -312,6 +312,7 @@ project: - chat_services - cluster - clusters +- cluster_agents - cluster_project - creator - cycle_analytics_stages diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 857b238981b..75347227c19 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -3007,25 +3007,46 @@ RSpec.describe Ci::Build do end context 'when build is parallelized' do - let(:total) { 5 } - let(:index) { 3 } + shared_examples 'parallelized jobs config' do + let(:index) { 3 } + let(:total) { 5 } - before do - build.options[:parallel] = total - build.options[:instance] = index - build.name = "#{build.name} #{index}/#{total}" + before do + build.options[:parallel] = config + build.options[:instance] = index + end + + it 'includes CI_NODE_INDEX' do + is_expected.to include( + { key: 'CI_NODE_INDEX', value: index.to_s, public: true, masked: false } + ) + end + + it 'includes correct CI_NODE_TOTAL' do + is_expected.to include( + { key: 'CI_NODE_TOTAL', value: total.to_s, public: true, masked: false } + ) + end end - it 'includes CI_NODE_INDEX' do - is_expected.to include( - { key: 'CI_NODE_INDEX', value: index.to_s, public: true, masked: false } - ) + context 'when parallel is a number' do + let(:config) { 5 } + + it_behaves_like 'parallelized jobs config' end - it 'includes correct CI_NODE_TOTAL' do - is_expected.to include( - { key: 'CI_NODE_TOTAL', value: total.to_s, public: true, masked: false } - ) + context 'when parallel is hash with the total key' do + let(:config) { { total: 5 } } + + it_behaves_like 'parallelized jobs config' + end + + context 'when parallel is nil' do + let(:config) {} + + it_behaves_like 'parallelized jobs config' do + let(:total) { 1 } + end end end diff --git a/spec/models/clusters/agent_spec.rb b/spec/models/clusters/agent_spec.rb new file mode 100644 index 00000000000..88283742f38 --- /dev/null +++ b/spec/models/clusters/agent_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Clusters::Agent do + subject { create(:cluster_agent) } + + it { is_expected.to belong_to(:project).class_name('::Project') } + it { is_expected.to have_many(:agent_tokens).class_name('Clusters::AgentToken') } + + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to validate_length_of(:name).is_at_most(255) } + it { is_expected.to validate_uniqueness_of(:name).scoped_to(:project_id) } +end diff --git a/spec/models/clusters/agent_token_spec.rb b/spec/models/clusters/agent_token_spec.rb new file mode 100644 index 00000000000..ad9dd11b24e --- /dev/null +++ b/spec/models/clusters/agent_token_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Clusters::AgentToken do + it { is_expected.to belong_to(:agent).class_name('Clusters::Agent') } + + describe '#token' do + it 'is generated on save' do + agent_token = build(:cluster_agent_token, token_encrypted: nil) + expect(agent_token.token).to be_nil + + agent_token.save! + + expect(agent_token.token).to be_present + end + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 722bf77faed..8b0f862932d 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -104,6 +104,7 @@ RSpec.describe Project do it { is_expected.to have_many(:clusters) } it { is_expected.to have_many(:management_clusters).class_name('Clusters::Cluster') } it { is_expected.to have_many(:kubernetes_namespaces) } + it { is_expected.to have_many(:cluster_agents).class_name('Clusters::Agent') } it { is_expected.to have_many(:custom_attributes).class_name('ProjectCustomAttribute') } it { is_expected.to have_many(:project_badges).class_name('ProjectBadge') } it { is_expected.to have_many(:lfs_file_locks) } diff --git a/spec/requests/api/internal/kubernetes_spec.rb b/spec/requests/api/internal/kubernetes_spec.rb new file mode 100644 index 00000000000..4c7e83abf19 --- /dev/null +++ b/spec/requests/api/internal/kubernetes_spec.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe API::Internal::Kubernetes do + describe "GET /internal/kubernetes/agent_info" do + context 'kubernetes_agent_internal_api feature flag disabled' do + before do + stub_feature_flags(kubernetes_agent_internal_api: false) + end + + it 'returns 404' do + get api('/internal/kubernetes/agent_info') + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + it 'returns 403 if Authorization header not sent' do + get api('/internal/kubernetes/agent_info') + + expect(response).to have_gitlab_http_status(:forbidden) + end + + context 'an agent is found' do + let!(:agent_token) { create(:cluster_agent_token) } + + let(:agent) { agent_token.agent } + let(:project) { agent.project } + + it 'returns expected data', :aggregate_failures do + get api('/internal/kubernetes/agent_info'), headers: { 'Authorization' => "Bearer #{agent_token.token}" } + + expect(response).to have_gitlab_http_status(:success) + + expect(json_response['project_id']).to eq(project.id) + expect(json_response['agent_id']).to eq(agent.id) + expect(json_response['agent_name']).to eq(agent.name) + expect(json_response['storage_name']).to eq(project.repository_storage) + expect(json_response['relative_path']).to eq(project.disk_path + '.git') + expect(json_response['gl_repository']).to eq("project-#{project.id}") + expect(json_response['gl_project_path']).to eq(project.full_path) + end + end + + context 'no such agent exists' do + it 'returns 404' do + get api('/internal/kubernetes/agent_info'), headers: { 'Authorization' => 'Bearer ABCD' } + + expect(response).to have_gitlab_http_status(:forbidden) + end + end + end + + describe 'GET /internal/kubernetes/project_info' do + context 'kubernetes_agent_internal_api feature flag disabled' do + before do + stub_feature_flags(kubernetes_agent_internal_api: false) + end + + it 'returns 404' do + get api('/internal/kubernetes/project_info') + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + it 'returns 403 if Authorization header not sent' do + get api('/internal/kubernetes/project_info') + + expect(response).to have_gitlab_http_status(:forbidden) + end + + context 'no such agent exists' do + it 'returns 404' do + get api('/internal/kubernetes/project_info'), headers: { 'Authorization' => 'Bearer ABCD' } + + expect(response).to have_gitlab_http_status(:forbidden) + end + end + + context 'an agent is found' do + let!(:agent_token) { create(:cluster_agent_token) } + + let(:agent) { agent_token.agent } + + context 'project is public' do + let(:project) { create(:project, :public) } + + it 'returns expected data', :aggregate_failures do + get api('/internal/kubernetes/project_info'), params: { id: project.id }, headers: { 'Authorization' => "Bearer #{agent_token.token}" } + + expect(response).to have_gitlab_http_status(:success) + + expect(json_response['project_id']).to eq(project.id) + expect(json_response['storage_name']).to eq(project.repository_storage) + expect(json_response['relative_path']).to eq(project.disk_path + '.git') + expect(json_response['gl_repository']).to eq("project-#{project.id}") + expect(json_response['gl_project_path']).to eq(project.full_path) + end + end + + context 'project is private' do + let(:project) { create(:project, :private) } + + it 'returns 404' do + get api('/internal/kubernetes/project_info'), params: { id: project.id }, headers: { 'Authorization' => "Bearer #{agent_token.token}" } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + context 'project is internal' do + let(:project) { create(:project, :internal) } + + it 'returns 404' do + get api('/internal/kubernetes/project_info'), params: { id: project.id }, headers: { 'Authorization' => "Bearer #{agent_token.token}" } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + + context 'project does not exist' do + it 'returns 404' do + get api('/internal/kubernetes/project_info'), params: { id: 0 }, headers: { 'Authorization' => "Bearer #{agent_token.token}" } + + expect(response).to have_gitlab_http_status(:not_found) + end + end + end + end +end diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index f787aedf7aa..4663993c8c0 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -6,8 +6,6 @@ module TestEnv ComponentFailedToInstallError = Class.new(StandardError) - SHA_REGEX = /\A[0-9a-f]{5,40}\z/i.freeze - # When developing the seed repository, comment out the branch you will modify. BRANCH_SHA = { 'signed-commits' => '6101e87', @@ -524,7 +522,7 @@ module TestEnv def component_matches_git_sha?(component_folder, expected_version) # Not a git SHA, so return early - return false unless expected_version =~ SHA_REGEX + return false unless expected_version =~ ::Gitlab::Git::COMMIT_ID sha, exit_status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} rev-parse HEAD), component_folder) return false if exit_status != 0 diff --git a/spec/support/shared_examples/models/cluster_application_status_shared_examples.rb b/spec/support/shared_examples/models/cluster_application_status_shared_examples.rb index f80ca235220..2bc5566e137 100644 --- a/spec/support/shared_examples/models/cluster_application_status_shared_examples.rb +++ b/spec/support/shared_examples/models/cluster_application_status_shared_examples.rb @@ -262,6 +262,14 @@ RSpec.shared_examples 'cluster application status specs' do |application_name| expect(subject).to be_installed end + + it 'clears #status_reason' do + expect(subject.status_reason).not_to be_nil + + subject.make_externally_installed! + + expect(subject.status_reason).to be_nil + end end end @@ -292,6 +300,14 @@ RSpec.shared_examples 'cluster application status specs' do |application_name| expect(subject).to be_uninstalled end + + it 'clears #status_reason' do + expect(subject.status_reason).not_to be_nil + + subject.make_externally_uninstalled! + + expect(subject.status_reason).to be_nil + end end end |