Merge branch 'backstage/gb/refactor-ci-cd-variables-collections' into 'master'42762-project-settings-sections-cannot-be-expanded-after-submission-error

Introduce CI/CD variables collection

Closes #33042

See merge request gitlab-org/gitlab-ce!14439

8 files changed, 130 insertions, 134 deletions

diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb
index b230b7f47ef..f8a3600e863 100644
--- a/app/models/ci/build.rb
+++ b/app/models/ci/build.rb
@@ -252,23 +252,23 @@ module Ci
     # All variables, including those dependent on environment, which could
     # contain unexpanded variables.
     def variables(environment: persisted_environment)
-      variables = predefined_variables
-      variables += project.predefined_variables
-      variables += pipeline.predefined_variables
-      variables += runner.predefined_variables if runner
-      variables += project.container_registry_variables
-      variables += project.deployment_variables if has_environment?
-      variables += project.auto_devops_variables
-      variables += yaml_variables
-      variables += user_variables
-      variables += project.group.secret_variables_for(ref, project).map(&:to_runner_variable) if project.group
-      variables += secret_variables(environment: environment)
-      variables += trigger_request.user_variables if trigger_request
-      variables += pipeline.variables.map(&:to_runner_variable)
-      variables += pipeline.pipeline_schedule.job_variables if pipeline.pipeline_schedule
-      variables += persisted_environment_variables if environment
-
-      variables
+      collection = Gitlab::Ci::Variables::Collection.new.tap do |variables|
+        variables.concat(predefined_variables)
+        variables.concat(project.predefined_variables)
+        variables.concat(pipeline.predefined_variables)
+        variables.concat(runner.predefined_variables) if runner
+        variables.concat(project.deployment_variables(environment: environment)) if has_environment?
+        variables.concat(yaml_variables)
+        variables.concat(user_variables)
+        variables.concat(project.group.secret_variables_for(ref, project)) if project.group
+        variables.concat(secret_variables(environment: environment))
+        variables.concat(trigger_request.user_variables) if trigger_request
+        variables.concat(pipeline.variables)
+        variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule
+        variables.concat(persisted_environment_variables) if environment
+      end
+
+      collection.to_runner_variables
     end
 
     def features
@@ -430,14 +430,14 @@ module Ci
     end
 
     def user_variables
-      return [] if user.blank?
+      Gitlab::Ci::Variables::Collection.new.tap do |variables|
+        return variables if user.blank?
 
-      [
-        { key: 'GITLAB_USER_ID', value: user.id.to_s, public: true },
-        { key: 'GITLAB_USER_EMAIL', value: user.email, public: true },
-        { key: 'GITLAB_USER_LOGIN', value: user.username, public: true },
-        { key: 'GITLAB_USER_NAME', value: user.name, public: true }
-      ]
+        variables.append(key: 'GITLAB_USER_ID', value: user.id.to_s)
+        variables.append(key: 'GITLAB_USER_EMAIL', value: user.email)
+        variables.append(key: 'GITLAB_USER_LOGIN', value: user.username)
+        variables.append(key: 'GITLAB_USER_NAME', value: user.name)
+      end
     end
 
     def secret_variables(environment: persisted_environment)
@@ -540,60 +540,57 @@ module Ci
     CI_REGISTRY_USER = 'gitlab-ci-token'.freeze
 
     def predefined_variables
-      variables = [
-        { key: 'CI', value: 'true', public: true },
-        { key: 'GITLAB_CI', value: 'true', public: true },
-        { key: 'GITLAB_FEATURES', value: project.namespace.features.join(','), public: true },
-        { key: 'CI_SERVER_NAME', value: 'GitLab', public: true },
-        { key: 'CI_SERVER_VERSION', value: Gitlab::VERSION, public: true },
-        { key: 'CI_SERVER_REVISION', value: Gitlab::REVISION, public: true },
-        { key: 'CI_JOB_ID', value: id.to_s, public: true },
-        { key: 'CI_JOB_NAME', value: name, public: true },
-        { key: 'CI_JOB_STAGE', value: stage, public: true },
-        { key: 'CI_JOB_TOKEN', value: token, public: false },
-        { key: 'CI_COMMIT_SHA', value: sha, public: true },
-        { key: 'CI_COMMIT_REF_NAME', value: ref, public: true },
-        { key: 'CI_COMMIT_REF_SLUG', value: ref_slug, public: true },
-        { key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER, public: true },
-        { key: 'CI_REGISTRY_PASSWORD', value: token, public: false },
-        { key: 'CI_REPOSITORY_URL', value: repo_url, public: false }
-      ]
-
-      variables << { key: "CI_COMMIT_TAG", value: ref, public: true } if tag?
-      variables << { key: "CI_PIPELINE_TRIGGERED", value: 'true', public: true } if trigger_request
-      variables << { key: "CI_JOB_MANUAL", value: 'true', public: true } if action?
-      variables.concat(legacy_variables)
+      Gitlab::Ci::Variables::Collection.new.tap do |variables|
+        variables.append(key: 'CI', value: 'true')
+        variables.append(key: 'GITLAB_CI', value: 'true')
+        variables.append(key: 'GITLAB_FEATURES', value: project.namespace.features.join(','))
+        variables.append(key: 'CI_SERVER_NAME', value: 'GitLab')
+        variables.append(key: 'CI_SERVER_VERSION', value: Gitlab::VERSION)
+        variables.append(key: 'CI_SERVER_REVISION', value: Gitlab::REVISION)
+        variables.append(key: 'CI_JOB_ID', value: id.to_s)
+        variables.append(key: 'CI_JOB_NAME', value: name)
+        variables.append(key: 'CI_JOB_STAGE', value: stage)
+        variables.append(key: 'CI_JOB_TOKEN', value: token, public: false)
+        variables.append(key: 'CI_COMMIT_SHA', value: sha)
+        variables.append(key: 'CI_COMMIT_REF_NAME', value: ref)
+        variables.append(key: 'CI_COMMIT_REF_SLUG', value: ref_slug)
+        variables.append(key: 'CI_REGISTRY_USER', value: CI_REGISTRY_USER)
+        variables.append(key: 'CI_REGISTRY_PASSWORD', value: token, public: false)
+        variables.append(key: 'CI_REPOSITORY_URL', value: repo_url, public: false)
+        variables.append(key: "CI_COMMIT_TAG", value: ref) if tag?
+        variables.append(key: "CI_PIPELINE_TRIGGERED", value: 'true') if trigger_request
+        variables.append(key: "CI_JOB_MANUAL", value: 'true') if action?
+        variables.concat(legacy_variables)
+      end
     end
 
     def persisted_environment_variables
-      return [] unless persisted_environment
+      Gitlab::Ci::Variables::Collection.new.tap do |variables|
+        return variables unless persisted_environment
 
-      variables = persisted_environment.predefined_variables
+        variables.concat(persisted_environment.predefined_variables)
 
-      # Here we're passing unexpanded environment_url for runner to expand,
-      # and we need to make sure that CI_ENVIRONMENT_NAME and
-      # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded.
-      variables << { key: 'CI_ENVIRONMENT_URL', value: environment_url, public: true } if environment_url
-
-      variables
+        # Here we're passing unexpanded environment_url for runner to expand,
+        # and we need to make sure that CI_ENVIRONMENT_NAME and
+        # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded.
+        variables.append(key: 'CI_ENVIRONMENT_URL', value: environment_url) if environment_url
+      end
     end
 
     def legacy_variables
-      variables = [
-        { key: 'CI_BUILD_ID', value: id.to_s, public: true },
-        { key: 'CI_BUILD_TOKEN', value: token, public: false },
-        { key: 'CI_BUILD_REF', value: sha, public: true },
-        { key: 'CI_BUILD_BEFORE_SHA', value: before_sha, public: true },
-        { key: 'CI_BUILD_REF_NAME', value: ref, public: true },
-        { key: 'CI_BUILD_REF_SLUG', value: ref_slug, public: true },
-        { key: 'CI_BUILD_NAME', value: name, public: true },
-        { key: 'CI_BUILD_STAGE', value: stage, public: true }
-      ]
-
-      variables << { key: "CI_BUILD_TAG", value: ref, public: true } if tag?
-      variables << { key: "CI_BUILD_TRIGGERED", value: 'true', public: true } if trigger_request
-      variables << { key: "CI_BUILD_MANUAL", value: 'true', public: true } if action?
-      variables
+      Gitlab::Ci::Variables::Collection.new.tap do |variables|
+        variables.append(key: 'CI_BUILD_ID', value: id.to_s)
+        variables.append(key: 'CI_BUILD_TOKEN', value: token, public: false)
+        variables.append(key: 'CI_BUILD_REF', value: sha)
+        variables.append(key: 'CI_BUILD_BEFORE_SHA', value: before_sha)
+        variables.append(key: 'CI_BUILD_REF_NAME', value: ref)
+        variables.append(key: 'CI_BUILD_REF_SLUG', value: ref_slug)
+        variables.append(key: 'CI_BUILD_NAME', value: name)
+        variables.append(key: 'CI_BUILD_STAGE', value: stage)
+        variables.append(key: "CI_BUILD_TAG", value: ref) if tag?
+        variables.append(key: "CI_BUILD_TRIGGERED", value: 'true') if trigger_request
+        variables.append(key: "CI_BUILD_MANUAL", value: 'true') if action?
+      end
     end
 
     def environment_url
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb
index a72a815bfe8..4966ea62df9 100644
--- a/app/models/ci/pipeline.rb
+++ b/app/models/ci/pipeline.rb
@@ -473,11 +473,10 @@ module Ci
     end
 
     def predefined_variables
-      [
-        { key: 'CI_PIPELINE_ID', value: id.to_s, public: true },
-        { key: 'CI_CONFIG_PATH', value: ci_yaml_file_path, public: true },
-        { key: 'CI_PIPELINE_SOURCE', value: source.to_s, public: true }
-      ]
+      Gitlab::Ci::Variables::Collection.new
+        .append(key: 'CI_PIPELINE_ID', value: id.to_s)
+        .append(key: 'CI_CONFIG_PATH', value: ci_yaml_file_path)
+        .append(key: 'CI_PIPELINE_SOURCE', value: source.to_s)
     end
 
     def queued_duration
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index 609620a62bb..7173f88f1c7 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -132,11 +132,10 @@ module Ci
     end
 
     def predefined_variables
-      [
-        { key: 'CI_RUNNER_ID', value: id.to_s, public: true },
-        { key: 'CI_RUNNER_DESCRIPTION', value: description, public: true },
-        { key: 'CI_RUNNER_TAGS', value: tag_list.to_s, public: true }
-      ]
+      Gitlab::Ci::Variables::Collection.new
+        .append(key: 'CI_RUNNER_ID', value: id.to_s)
+        .append(key: 'CI_RUNNER_DESCRIPTION', value: description)
+        .append(key: 'CI_RUNNER_TAGS', value: tag_list.to_s)
     end
 
     def tick_runner_queue
diff --git a/app/models/clusters/platforms/kubernetes.rb b/app/models/clusters/platforms/kubernetes.rb
index 7ce8befeeeb..015c8e9bcf8 100644
--- a/app/models/clusters/platforms/kubernetes.rb
+++ b/app/models/clusters/platforms/kubernetes.rb
@@ -56,19 +56,19 @@ module Clusters
       def predefined_variables
         config = YAML.dump(kubeconfig)
 
-        variables = [
-          { key: 'KUBE_URL', value: api_url, public: true },
-          { key: 'KUBE_TOKEN', value: token, public: false },
-          { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true },
-          { key: 'KUBECONFIG', value: config, public: false, file: true }
-        ]
-
-        if ca_pem.present?
-          variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true }
-          variables << { key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true }
+        Gitlab::Ci::Variables::Collection.new.tap do |variables|
+          variables
+            .append(key: 'KUBE_URL', value: api_url)
+            .append(key: 'KUBE_TOKEN', value: token, public: false)
+            .append(key: 'KUBE_NAMESPACE', value: actual_namespace)
+            .append(key: 'KUBECONFIG', value: config, public: false, file: true)
+
+          if ca_pem.present?
+            variables
+              .append(key: 'KUBE_CA_PEM', value: ca_pem)
+              .append(key: 'KUBE_CA_PEM_FILE', value: ca_pem, file: true)
+          end
         end
-
-        variables
       end
 
       # Constructs a list of terminals from the reactive cache
diff --git a/app/models/environment.rb b/app/models/environment.rb
index 2b0a88ac5b4..9517723d9d9 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -65,10 +65,9 @@ class Environment < ActiveRecord::Base
   end
 
   def predefined_variables
-    [
-      { key: 'CI_ENVIRONMENT_NAME', value: name, public: true },
-      { key: 'CI_ENVIRONMENT_SLUG', value: slug, public: true }
-    ]
+    Gitlab::Ci::Variables::Collection.new
+      .append(key: 'CI_ENVIRONMENT_NAME', value: name)
+      .append(key: 'CI_ENVIRONMENT_SLUG', value: slug)
   end
 
   def recently_updated_on_branch?(ref)
diff --git a/app/models/project.rb b/app/models/project.rb
index 0183e3d0a38..a291ad7eed5 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1572,29 +1572,30 @@ class Project < ActiveRecord::Base
   end
 
   def predefined_variables
-    [
-      { key: 'CI_PROJECT_ID', value: id.to_s, public: true },
-      { key: 'CI_PROJECT_NAME', value: path, public: true },
-      { key: 'CI_PROJECT_PATH', value: full_path, public: true },
-      { key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug, public: true },
-      { key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path, public: true },
-      { key: 'CI_PROJECT_URL', value: web_url, public: true },
-      { key: 'CI_PROJECT_VISIBILITY', value: Gitlab::VisibilityLevel.string_level(visibility_level), public: true }
-    ]
+    visibility = Gitlab::VisibilityLevel.string_level(visibility_level)
+
+    Gitlab::Ci::Variables::Collection.new
+      .append(key: 'CI_PROJECT_ID', value: id.to_s)
+      .append(key: 'CI_PROJECT_NAME', value: path)
+      .append(key: 'CI_PROJECT_PATH', value: full_path)
+      .append(key: 'CI_PROJECT_PATH_SLUG', value: full_path_slug)
+      .append(key: 'CI_PROJECT_NAMESPACE', value: namespace.full_path)
+      .append(key: 'CI_PROJECT_URL', value: web_url)
+      .append(key: 'CI_PROJECT_VISIBILITY', value: visibility)
+      .concat(container_registry_variables)
+      .concat(auto_devops_variables)
   end
 
   def container_registry_variables
-    return [] unless Gitlab.config.registry.enabled
+    Gitlab::Ci::Variables::Collection.new.tap do |variables|
+      return variables unless Gitlab.config.registry.enabled
 
-    variables = [
-      { key: 'CI_REGISTRY', value: Gitlab.config.registry.host_port, public: true }
-    ]
+      variables.append(key: 'CI_REGISTRY', value: Gitlab.config.registry.host_port)
 
-    if container_registry_enabled?
-      variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_url, public: true }
+      if container_registry_enabled?
+        variables.append(key: 'CI_REGISTRY_IMAGE', value: container_registry_url)
+      end
     end
-
-    variables
   end
 
   def secret_variables_for(ref:, environment: nil)
@@ -1614,16 +1615,14 @@ class Project < ActiveRecord::Base
     end
   end
 
-  def deployment_variables
-    return [] unless deployment_platform
-
-    deployment_platform.predefined_variables
+  def deployment_variables(environment: nil)
+    deployment_platform(environment: environment)&.predefined_variables || []
   end
 
   def auto_devops_variables
     return [] unless auto_devops_enabled?
 
-    (auto_devops || build_auto_devops)&.variables
+    (auto_devops || build_auto_devops)&.predefined_variables
   end
 
   def append_or_update_attribute(name, value)
diff --git a/app/models/project_auto_devops.rb b/app/models/project_auto_devops.rb
index 112ed7ed434..ed6c1eddbc1 100644
--- a/app/models/project_auto_devops.rb
+++ b/app/models/project_auto_devops.rb
@@ -14,9 +14,12 @@ class ProjectAutoDevops < ActiveRecord::Base
     domain.present? || instance_domain.present?
   end
 
-  def variables
-    variables = []
-    variables << { key: 'AUTO_DEVOPS_DOMAIN', value: domain.presence || instance_domain, public: true } if has_domain?
-    variables
+  def predefined_variables
+    Gitlab::Ci::Variables::Collection.new.tap do |variables|
+      if has_domain?
+        variables.append(key: 'AUTO_DEVOPS_DOMAIN',
+                         value: domain.presence || instance_domain)
+      end
+    end
   end
 end
diff --git a/app/models/project_services/kubernetes_service.rb b/app/models/project_services/kubernetes_service.rb
index ad4ad7903ad..b4cd1c1a155 100644
--- a/app/models/project_services/kubernetes_service.rb
+++ b/app/models/project_services/kubernetes_service.rb
@@ -105,19 +105,19 @@ class KubernetesService < DeploymentService
   def predefined_variables
     config = YAML.dump(kubeconfig)
 
-    variables = [
-      { key: 'KUBE_URL', value: api_url, public: true },
-      { key: 'KUBE_TOKEN', value: token, public: false },
-      { key: 'KUBE_NAMESPACE', value: actual_namespace, public: true },
-      { key: 'KUBECONFIG', value: config, public: false, file: true }
-    ]
-
-    if ca_pem.present?
-      variables << { key: 'KUBE_CA_PEM', value: ca_pem, public: true }
-      variables << { key: 'KUBE_CA_PEM_FILE', value: ca_pem, public: true, file: true }
+    Gitlab::Ci::Variables::Collection.new.tap do |variables|
+      variables
+        .append(key: 'KUBE_URL', value: api_url)
+        .append(key: 'KUBE_TOKEN', value: token, public: false)
+        .append(key: 'KUBE_NAMESPACE', value: actual_namespace)
+        .append(key: 'KUBECONFIG', value: config, public: false, file: true)
+
+      if ca_pem.present?
+        variables
+          .append(key: 'KUBE_CA_PEM', value: ca_pem)
+          .append(key: 'KUBE_CA_PEM_FILE', value: ca_pem, file: true)
+      end
     end
-
-    variables
   end
 
   # Constructs a list of terminals from the reactive cache