summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-04-06 12:10:44 +0000
committerGitLab Bot <gitlab-bot@gitlab.com>2020-04-06 12:10:44 +0000
commitba174c982f40d71a87fd511b091753807174f7e7 (patch)
treeb89d55c715288f2c2f76724c1b7ff4a6ebf90154 /lib
parenteaea945e0355826c58c3dcf887496ea91064f85c (diff)
downloadgitlab-ce-ba174c982f40d71a87fd511b091753807174f7e7.tar.gz
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r--lib/gitlab/ci/config/entry/include.rb5
-rw-r--r--lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml20
-rw-r--r--lib/gitlab/ci/yaml_processor.rb50
-rw-r--r--lib/gitlab/import_export/json/ndjson_writer.rb59
-rw-r--r--lib/gitlab/import_export/project/tree_saver.rb18
-rw-r--r--lib/gitlab/sidekiq_middleware.rb6
6 files changed, 93 insertions, 65 deletions
diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb
index b2586714636..cd09d83b728 100644
--- a/lib/gitlab/ci/config/entry/include.rb
+++ b/lib/gitlab/ci/config/entry/include.rb
@@ -15,11 +15,6 @@ module Gitlab
validations do
validates :config, hash_or_string: true
validates :config, allowed_keys: ALLOWED_KEYS
- validate do
- if config[:artifact] && config[:job].blank?
- errors.add(:config, "must specify the job where to fetch the artifact from")
- end
- end
end
end
end
diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
index 020d1f323ee..10ef33e71d5 100644
--- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml
@@ -1,7 +1,7 @@
# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/
# Configure the scanning tool through the environment variables.
-# List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings
+# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables
# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
stages:
@@ -19,26 +19,10 @@ dast:
name: "registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION"
variables:
GIT_STRATEGY: none
- # URL to scan:
- # DAST_WEBSITE: https://example.com/
- #
- # Time limit for target availability (scan is attempted even when timeout):
- # DAST_TARGET_AVAILABILITY_TIMEOUT: 60
- #
- # Set these variables to scan with an authenticated user:
- # DAST_AUTH_URL: https://example.com/sign-in
- # DAST_USERNAME: john.doe@example.com
- # DAST_PASSWORD: john-doe-password
- # DAST_USERNAME_FIELD: session[user] # the name of username field at the sign-in HTML form
- # DAST_PASSWORD_FIELD: session[password] # the name of password field at the sign-in HTML form
- # DAST_AUTH_EXCLUDE_URLS: http://example.com/sign-out,http://example.com/sign-out-2 # optional: URLs to skip during the authenticated scan; comma-separated, no spaces in between
- #
- # Perform ZAP Full Scan, which includes both passive and active scanning:
- # DAST_FULL_SCAN_ENABLED: "true"
allow_failure: true
script:
- export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
- - /analyze -t $DAST_WEBSITE
+ - /analyze
artifacts:
reports:
dast: gl-dast-report.json
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 4b0062549f0..764047dae6d 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -142,7 +142,6 @@ module Gitlab
validate_job_stage!(name, job)
validate_job_dependencies!(name, job)
validate_job_needs!(name, job)
- validate_dynamic_child_pipeline_dependencies!(name, job)
validate_job_environment!(name, job)
end
end
@@ -164,50 +163,35 @@ module Gitlab
def validate_job_dependencies!(name, job)
return unless job[:dependencies]
- job[:dependencies].each do |dependency|
- validate_job_dependency!(name, dependency)
- end
- end
+ stage_index = @stages.index(job[:stage])
- def validate_dynamic_child_pipeline_dependencies!(name, job)
- return unless includes = job.dig(:trigger, :include)
+ job[:dependencies].each do |dependency|
+ raise ValidationError, "#{name} job: undefined dependency: #{dependency}" unless @jobs[dependency.to_sym]
- includes.each do |included|
- next unless dependency = included[:job]
+ dependency_stage_index = @stages.index(@jobs[dependency.to_sym][:stage])
- validate_job_dependency!(name, dependency)
+ unless dependency_stage_index.present? && dependency_stage_index < stage_index
+ raise ValidationError, "#{name} job: dependency #{dependency} is not defined in prior stages"
+ end
end
end
def validate_job_needs!(name, job)
- return unless needs = job.dig(:needs, :job)
+ return unless job.dig(:needs, :job)
- needs.each do |need|
- dependency = need[:name]
- validate_job_dependency!(name, dependency, 'need')
- end
- end
+ stage_index = @stages.index(job[:stage])
- def validate_job_dependency!(name, dependency, dependency_type = 'dependency')
- unless @jobs[dependency.to_sym]
- raise ValidationError, "#{name} job: undefined #{dependency_type}: #{dependency}"
- end
+ job.dig(:needs, :job).each do |need|
+ need_job_name = need[:name]
- job_stage_index = stage_index(name)
- dependency_stage_index = stage_index(dependency)
+ raise ValidationError, "#{name} job: undefined need: #{need_job_name}" unless @jobs[need_job_name.to_sym]
- # A dependency might be defined later in the configuration
- # with a stage that does not exist
- unless dependency_stage_index.present? && dependency_stage_index < job_stage_index
- raise ValidationError, "#{name} job: #{dependency_type} #{dependency} is not defined in prior stages"
- end
- end
+ needs_stage_index = @stages.index(@jobs[need_job_name.to_sym][:stage])
- def stage_index(name)
- job = @jobs[name.to_sym]
- return unless job
-
- @stages.index(job[:stage])
+ unless needs_stage_index.present? && needs_stage_index < stage_index
+ raise ValidationError, "#{name} job: need #{need_job_name} is not defined in prior stages"
+ end
+ end
end
def validate_job_environment!(name, job)
diff --git a/lib/gitlab/import_export/json/ndjson_writer.rb b/lib/gitlab/import_export/json/ndjson_writer.rb
new file mode 100644
index 00000000000..e74fdd74049
--- /dev/null
+++ b/lib/gitlab/import_export/json/ndjson_writer.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module JSON
+ class NdjsonWriter
+ include Gitlab::ImportExport::CommandLineUtil
+
+ def initialize(dir_path)
+ @dir_path = dir_path
+ end
+
+ def close
+ end
+
+ def write_attributes(exportable_path, hash)
+ # It will create:
+ # tree/project.json
+ with_file("#{exportable_path}.json") do |file|
+ file.write(hash.to_json)
+ end
+ end
+
+ def write_relation(exportable_path, relation, value)
+ # It will create:
+ # tree/project/ci_cd_setting.ndjson
+ with_file(exportable_path, "#{relation}.ndjson") do |file|
+ file.write(value.to_json)
+ end
+ end
+
+ def write_relation_array(exportable_path, relation, items)
+ # It will create:
+ # tree/project/merge_requests.ndjson
+ with_file(exportable_path, "#{relation}.ndjson") do |file|
+ items.each do |item|
+ file.write(item.to_json)
+ file.write("\n")
+ end
+ end
+ end
+
+ private
+
+ def with_file(*path)
+ file_path = File.join(@dir_path, *path)
+ raise ArgumentError, "The #{file_path} already exist" if File.exist?(file_path)
+
+ # ensure that path is created
+ mkdir_p(File.dirname(file_path))
+
+ File.open(file_path, "wb") do |file|
+ yield(file)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 988776fe600..0017aa523c1 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -11,15 +11,9 @@ module Gitlab
@project = project
@current_user = current_user
@shared = shared
- @full_path = File.join(@shared.export_path, ImportExport.project_filename)
end
def save
- json_writer = ImportExport::JSON::LegacyWriter.new(
- @full_path,
- allowed_path: "project"
- )
-
ImportExport::JSON::StreamingSerializer.new(
exportable,
reader.project_tree,
@@ -57,6 +51,18 @@ module Gitlab
def presenter_class
Projects::ImportExport::ProjectExportPresenter
end
+
+ def json_writer
+ @json_writer ||= begin
+ if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace)
+ full_path = File.join(@shared.export_path, 'tree')
+ Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path)
+ else
+ full_path = File.join(@shared.export_path, ImportExport.project_filename)
+ Gitlab::ImportExport::JSON::LegacyWriter.new(full_path, allowed_path: 'project')
+ end
+ end
+ end
end
end
end
diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb
index 37165d787c7..1b155570f18 100644
--- a/lib/gitlab/sidekiq_middleware.rb
+++ b/lib/gitlab/sidekiq_middleware.rb
@@ -29,12 +29,12 @@ module Gitlab
# eg: `config.client_middleware(&Gitlab::SidekiqMiddleware.client_configurator)`
def self.client_configurator
lambda do |chain|
- chain.add ::Gitlab::SidekiqStatus::ClientMiddleware
- chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics
chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Client # needs to be before the Labkit middleware
chain.add ::Labkit::Middleware::Sidekiq::Client
- chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client
chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client
+ chain.add ::Gitlab::SidekiqStatus::ClientMiddleware
+ chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client
+ chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics
end
end
end