diff options
Diffstat (limited to 'lib/gitlab')
75 files changed, 699 insertions, 107 deletions
diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb new file mode 100644 index 00000000000..35bfc381180 --- /dev/null +++ b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true +# +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateClusterKubernetesNamespaceTable + include Gitlab::Database::MigrationHelpers + + BATCH_SIZE = 1_000 + + module Migratable + class KubernetesNamespace < ActiveRecord::Base + self.table_name = 'clusters_kubernetes_namespaces' + end + + class ClusterProject < ActiveRecord::Base + include EachBatch + + self.table_name = 'cluster_projects' + + belongs_to :project + + def self.with_no_kubernetes_namespace + where.not(id: Migratable::KubernetesNamespace.select(:cluster_project_id)) + end + + def namespace + slug = "#{project.path}-#{project.id}".downcase + slug.gsub(/[^-a-z0-9]/, '-').gsub(/^-+/, '') + end + + def service_account + "#{namespace}-service-account" + end + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + end + end + + def perform + cluster_projects_with_no_kubernetes_namespace.each_batch(of: BATCH_SIZE) do |cluster_projects_batch, index| + sql_values = sql_values_for(cluster_projects_batch) + + insert_into_cluster_kubernetes_namespace(sql_values) + end + end + + private + + def cluster_projects_with_no_kubernetes_namespace + Migratable::ClusterProject.with_no_kubernetes_namespace + end + + def sql_values_for(cluster_projects) + cluster_projects.map do |cluster_project| + values_for_cluster_project(cluster_project) + end + end + + def values_for_cluster_project(cluster_project) + { + cluster_project_id: cluster_project.id, + cluster_id: cluster_project.cluster_id, + project_id: cluster_project.project_id, + namespace: cluster_project.namespace, + service_account_name: cluster_project.service_account, + created_at: 'NOW()', + updated_at: 'NOW()' + } + end + + def insert_into_cluster_kubernetes_namespace(rows) + Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, + rows, + disable_quote: [:created_at, :updated_at]) + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_restricted_todos.rb b/lib/gitlab/background_migration/remove_restricted_todos.rb index 9941c2fe6d9..47579d46c1b 100644 --- a/lib/gitlab/background_migration/remove_restricted_todos.rb +++ b/lib/gitlab/background_migration/remove_restricted_todos.rb @@ -67,7 +67,7 @@ module Gitlab .where('access_level >= ?', 20) confidential_issues = Issue.select(:id, :author_id).where(confidential: true, project_id: project_id) - confidential_issues.each_batch(of: 100) do |batch| + confidential_issues.each_batch(of: 100, order_hint: :confidential) do |batch| batch.each do |issue| assigned_users = IssueAssignee.select(:user_id).where(issue_id: issue.id) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index e4610faa327..c8cb3248fa7 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when start_in artifacts cache dependencies before_script after_script variables - environment coverage retry extends].freeze + environment coverage retry parallel extends].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -26,14 +26,13 @@ module Gitlab with_options allow_nil: true do validates :tags, array_of_strings: true validates :allow_failure, boolean: true - validates :retry, numericality: { only_integer: true, - greater_than_or_equal_to: 0, - less_than_or_equal_to: 2 } + validates :parallel, numericality: { only_integer: true, + greater_than_or_equal_to: 2, + less_than_or_equal_to: 50 } validates :when, inclusion: { in: %w[on_success on_failure always manual delayed], message: 'should be on_success, on_failure, ' \ 'always, manual or delayed' } - validates :dependencies, array_of_strings: true validates :extends, type: String end @@ -79,17 +78,21 @@ module Gitlab description: 'Artifacts configuration for this job.' entry :environment, Entry::Environment, - description: 'Environment configuration for this job.' + description: 'Environment configuration for this job.' entry :coverage, Entry::Coverage, - description: 'Coverage configuration for this job.' + description: 'Coverage configuration for this job.' + + entry :retry, Entry::Retry, + description: 'Retry configuration for this job.' helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment, :coverage, :retry + :artifacts, :commands, :environment, :coverage, :retry, + :parallel attributes :script, :tags, :allow_failure, :when, :dependencies, - :retry, :extends, :start_in + :retry, :parallel, :extends, :start_in def compose!(deps = nil) super do @@ -157,7 +160,8 @@ module Gitlab environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, - retry: retry_defined? ? retry_value.to_i : nil, + retry: retry_defined? ? retry_value : nil, + parallel: parallel_defined? ? parallel_value.to_i : nil, artifacts: artifacts_value, after_script: after_script_value, ignore: ignored? } diff --git a/lib/gitlab/ci/config/entry/retry.rb b/lib/gitlab/ci/config/entry/retry.rb new file mode 100644 index 00000000000..e39cc5de229 --- /dev/null +++ b/lib/gitlab/ci/config/entry/retry.rb @@ -0,0 +1,90 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a retry config for a job. + # + class Retry < Simplifiable + strategy :SimpleRetry, if: -> (config) { config.is_a?(Integer) } + strategy :FullRetry, if: -> (config) { config.is_a?(Hash) } + + class SimpleRetry < Entry::Node + include Entry::Validatable + + validations do + validates :config, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } + end + + def value + { + max: config + } + end + + def location + 'retry' + end + end + + class FullRetry < Entry::Node + include Entry::Validatable + include Entry::Attributable + + ALLOWED_KEYS = %i[max when].freeze + attributes :max, :when + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :max, numericality: { only_integer: true, + greater_than_or_equal_to: 0, + less_than_or_equal_to: 2 } + + validates :when, array_of_strings_or_string: true + validates :when, + allowed_array_values: { in: FullRetry.possible_retry_when_values }, + if: -> (config) { config.when.is_a?(Array) } + validates :when, + inclusion: { in: FullRetry.possible_retry_when_values }, + if: -> (config) { config.when.is_a?(String) } + end + end + + def self.possible_retry_when_values + @possible_retry_when_values ||= ::Ci::Build.failure_reasons.keys.map(&:to_s) + ['always'] + end + + def value + super.tap do |config| + # make sure that `when` is an array, because we allow it to + # be passed as a String in config for simplicity + config[:when] = Array.wrap(config[:when]) if config[:when] + end + end + + def location + 'retry' + end + end + + class UnknownStrategy < Entry::Node + def errors + ["#{location} has to be either an integer or a hash"] + end + + def location + 'retry config' + end + end + + def self.default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index 805d26ca8d8..a1d552fb2e5 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -7,11 +7,11 @@ module Gitlab module Validators class AllowedKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - unknown_keys = record.config.try(:keys).to_a - options[:in] + unknown_keys = value.try(:keys).to_a - options[:in] if unknown_keys.any? - record.errors.add(:config, 'contains unknown keys: ' + - unknown_keys.join(', ')) + record.errors.add(attribute, "contains unknown keys: " + + unknown_keys.join(', ')) end end end @@ -24,6 +24,16 @@ module Gitlab end end + class AllowedArrayValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unkown_values = value - options[:in] + unless unkown_values.empty? + record.errors.add(attribute, "contains unknown values: " + + unkown_values.join(', ')) + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers @@ -68,6 +78,14 @@ module Gitlab end end + class HashOrIntegerValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless value.is_a?(Hash) || value.is_a?(Integer) + record.errors.add(attribute, 'should be a hash or an integer') + end + end + end + class KeyValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/lib/gitlab/ci/config/normalizer.rb b/lib/gitlab/ci/config/normalizer.rb new file mode 100644 index 00000000000..b7743bd2090 --- /dev/null +++ b/lib/gitlab/ci/config/normalizer.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + class Normalizer + def initialize(jobs_config) + @jobs_config = jobs_config + end + + def normalize_jobs + extract_parallelized_jobs! + return @jobs_config if @parallelized_jobs.empty? + + parallelized_config = parallelize_jobs + parallelize_dependencies(parallelized_config) + end + + private + + def extract_parallelized_jobs! + @parallelized_jobs = {} + + @jobs_config.each do |job_name, config| + if config[:parallel] + @parallelized_jobs[job_name] = self.class.parallelize_job_names(job_name, config[:parallel]) + end + end + + @parallelized_jobs + end + + def parallelize_jobs + @jobs_config.each_with_object({}) do |(job_name, config), hash| + if @parallelized_jobs.key?(job_name) + @parallelized_jobs[job_name].each { |name, index| hash[name.to_sym] = config.merge(name: name, instance: index) } + else + hash[job_name] = config + end + + hash + end + end + + def parallelize_dependencies(parallelized_config) + parallelized_job_names = @parallelized_jobs.keys.map(&:to_s) + parallelized_config.each_with_object({}) do |(job_name, config), hash| + if config[:dependencies] && (intersection = config[:dependencies] & parallelized_job_names).any? + deps = intersection.map { |dep| @parallelized_jobs[dep.to_sym].map(&:first) }.flatten + hash[job_name] = config.merge(dependencies: deps) + else + hash[job_name] = config + end + + hash + end + end + + def self.parallelize_job_names(name, total) + Array.new(total) { |index| ["#{name} #{index + 1}/#{total}", index + 1] } + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index c882241ef6a..aa627bdb009 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -7,26 +7,11 @@ module Gitlab class Create < Chain::Base include Chain::Helpers - # rubocop: disable CodeReuse/ActiveRecord def perform! - ::Ci::Pipeline.transaction do - pipeline.save! - - ## - # Create environments before the pipeline starts. - # - pipeline.builds.each do |build| - if build.has_environment? - project.environments.find_or_create_by( - name: build.expanded_environment_name - ) - end - end - end + pipeline.save! rescue ActiveRecord::RecordInvalid => e error("Failed to persist the pipeline: #{e}") end - # rubocop: enable CodeReuse/ActiveRecord def break? !pipeline.persisted? diff --git a/lib/gitlab/ci/status/build/action.rb b/lib/gitlab/ci/status/build/action.rb index 6c9125647ad..45d9ba41e92 100644 --- a/lib/gitlab/ci/status/build/action.rb +++ b/lib/gitlab/ci/status/build/action.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 024047d4983..43fb5cdbbe6 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/canceled.rb b/lib/gitlab/ci/status/build/canceled.rb index c83e2734a73..0518b9e673d 100644 --- a/lib/gitlab/ci/status/build/canceled.rb +++ b/lib/gitlab/ci/status/build/canceled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb index c1fc70ac266..6a75ec5c37f 100644 --- a/lib/gitlab/ci/status/build/common.rb +++ b/lib/gitlab/ci/status/build/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/created.rb b/lib/gitlab/ci/status/build/created.rb index 5be8e9de425..780fea23123 100644 --- a/lib/gitlab/ci/status/build/created.rb +++ b/lib/gitlab/ci/status/build/created.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb index 495227c2ffb..d74cfc1ee77 100644 --- a/lib/gitlab/ci/status/build/erased.rb +++ b/lib/gitlab/ci/status/build/erased.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index 4a74d6d6ed1..6e4bfe23f2b 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 4babc23a495..d40454df737 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status @@ -12,7 +14,8 @@ module Gitlab missing_dependency_failure: 'missing dependency failure', runner_unsupported: 'unsupported runner', stale_schedule: 'stale schedule', - job_execution_timeout: 'job execution timeout' + job_execution_timeout: 'job execution timeout', + archived_failure: 'archived failure' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb index ca0046fb1f7..d7570fdd3e2 100644 --- a/lib/gitlab/ci/status/build/failed_allowed.rb +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/manual.rb b/lib/gitlab/ci/status/build/manual.rb index 042da6392d3..d01b09f1398 100644 --- a/lib/gitlab/ci/status/build/manual.rb +++ b/lib/gitlab/ci/status/build/manual.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/pending.rb b/lib/gitlab/ci/status/build/pending.rb index 9dd9a27ad57..95f668295dd 100644 --- a/lib/gitlab/ci/status/build/pending.rb +++ b/lib/gitlab/ci/status/build/pending.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index a8b9ebf0803..c66b8ca5654 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/retried.rb b/lib/gitlab/ci/status/build/retried.rb index 6e190e4ee3c..b489dc68733 100644 --- a/lib/gitlab/ci/status/build/retried.rb +++ b/lib/gitlab/ci/status/build/retried.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 5aeb8e51480..eb6b3f21604 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/scheduled.rb b/lib/gitlab/ci/status/build/scheduled.rb index 62ad9083616..b3452eae189 100644 --- a/lib/gitlab/ci/status/build/scheduled.rb +++ b/lib/gitlab/ci/status/build/scheduled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status @@ -7,7 +9,7 @@ module Gitlab { image: 'illustrations/illustrations_scheduled-job_countdown.svg', size: 'svg-394', - title: _("This is a delayed to run in ") + " #{execute_in}", + title: _("This is a delayed job to run in %{remainingTime}"), content: _("This job will automatically run after it's timer finishes. " \ "Often they are used for incremental roll-out deploys " \ "to production environments. When unscheduled it converts " \ @@ -16,21 +18,12 @@ module Gitlab end def status_tooltip - "delayed manual action (#{execute_in})" + "delayed manual action (%{remainingTime})" end def self.matches?(build, user) build.scheduled? && build.scheduled_at end - - private - - include TimeHelper - - def execute_in - remaining_seconds = [0, subject.scheduled_at - Time.now].max - duration_in_numbers(remaining_seconds) - end end end end diff --git a/lib/gitlab/ci/status/build/skipped.rb b/lib/gitlab/ci/status/build/skipped.rb index 3e678d0baee..4fe2f7b3114 100644 --- a/lib/gitlab/ci/status/build/skipped.rb +++ b/lib/gitlab/ci/status/build/skipped.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index dea838bfa39..a620e7ad126 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/build/unschedule.rb b/lib/gitlab/ci/status/build/unschedule.rb index e1b7b83428c..9110839cb55 100644 --- a/lib/gitlab/ci/status/build/unschedule.rb +++ b/lib/gitlab/ci/status/build/unschedule.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index e6195a60d4f..07f37732023 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index 9d6a2f51c11..ea773ee9944 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index 846f00b83dd..fface4bb97b 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb index 1e8101f8949..b72a28ed0b6 100644 --- a/lib/gitlab/ci/status/extended.rb +++ b/lib/gitlab/ci/status/extended.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb index 9307545b5b1..4169f5b3210 100644 --- a/lib/gitlab/ci/status/external/common.rb +++ b/lib/gitlab/ci/status/external/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/external/factory.rb b/lib/gitlab/ci/status/external/factory.rb index 07b15bd8d97..91fafb940a8 100644 --- a/lib/gitlab/ci/status/external/factory.rb +++ b/lib/gitlab/ci/status/external/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index 15836c699c7..3446644eff8 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index 27ce85bd3ed..770ed7d4d5a 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/group/common.rb b/lib/gitlab/ci/status/group/common.rb index cfd4329a923..0b5ea0712ca 100644 --- a/lib/gitlab/ci/status/group/common.rb +++ b/lib/gitlab/ci/status/group/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb index d118116cfc3..ee785856fdd 100644 --- a/lib/gitlab/ci/status/group/factory.rb +++ b/lib/gitlab/ci/status/group/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index fc387e2fd25..50c92add400 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index 6780780db32..cea7e6ed938 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index bf7e484ee9b..ed13a439be0 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb index 61bb07beb0f..7b34a2ea858 100644 --- a/lib/gitlab/ci/status/pipeline/common.rb +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/delayed.rb b/lib/gitlab/ci/status/pipeline/delayed.rb index 12736861c89..e61acdcd167 100644 --- a/lib/gitlab/ci/status/pipeline/delayed.rb +++ b/lib/gitlab/ci/status/pipeline/delayed.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 0adf83fa197..5d1a8bbd924 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index ee13905e46d..ac7dd74cdce 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb index 3adcfa36af2..16ad1da89e3 100644 --- a/lib/gitlab/ci/status/scheduled.rb +++ b/lib/gitlab/ci/status/scheduled.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index 0dbdc4de426..aaec1e1d201 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb index f60a7662075..f12daaa9676 100644 --- a/lib/gitlab/ci/status/stage/common.rb +++ b/lib/gitlab/ci/status/stage/common.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb index 4c37f084d07..58f4642510b 100644 --- a/lib/gitlab/ci/status/stage/factory.rb +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index 731013ec017..020f2c5b89f 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index 32b4cf43e48..6632cd9b143 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Status diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 734af5eba59..c759bb7098e 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -49,7 +49,7 @@ variables: POSTGRES_ENABLED: "true" POSTGRES_DB: $CI_ENVIRONMENT_SLUG - KUBERNETES_VERSION: 1.8.6 + KUBERNETES_VERSION: 1.10.9 HELM_VERSION: 2.11.0 DOCKER_DRIVER: overlay2 @@ -116,12 +116,9 @@ code_quality: license_management: stage: test - image: docker:stable + image: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable" allow_failure: true - services: - - docker:stable-dind script: - - setup_docker - license_management artifacts: paths: [gl-license-management-report.json] @@ -525,11 +522,7 @@ rollout 100%: } function license_management() { - # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" - LICENSE_MANAGEMENT_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - docker run --volume "$PWD:/code" \ - "registry.gitlab.com/gitlab-org/security-products/license-management:$LICENSE_MANAGEMENT_VERSION" analyze /code + /run.sh analyze . } function sast() { diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 2147f62a84a..e9b3199d56e 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + ## # This class is compatible with IO class (https://ruby-doc.org/core-2.3.1/IO.html) # source: https://gitlab.com/snippets/1685610 @@ -66,8 +68,8 @@ module Gitlab end end - def read(length = nil, outbuf = "") - out = "" + def read(length = nil, outbuf = nil) + out = [] length ||= size - tell @@ -83,17 +85,18 @@ module Gitlab length -= chunk_data.bytesize end + out = out.join + # If outbuf is passed, we put the output into the buffer. This supports IO.copy_stream functionality if outbuf - outbuf.slice!(0, outbuf.bytesize) - outbuf << out + outbuf.replace(out) end out end def readline - out = "" + out = [] until eof? data = chunk_slice_from_offset @@ -109,7 +112,7 @@ module Gitlab end end - out + out.join end def write(data) diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb index c09089d6475..f33f8cc56c1 100644 --- a/lib/gitlab/ci/trace/section_parser.rb +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Trace diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index a71040e5e56..bd40fdf59b1 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci class Trace @@ -129,8 +131,7 @@ module Gitlab debris = '' until (buf = read_backward(BUFFER_SIZE)).empty? - buf += debris - debris, *lines = buf.each_line.to_a + debris, *lines = (buf + debris).each_line.to_a lines.reverse_each do |line| yield(line.force_encoding(Encoding.default_external)) end diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index ad30b3f427c..a7b4e0348c2 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Variables diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 7da6d09d440..fdf852e8788 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module Gitlab module Ci module Variables diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 39a1b52e531..e6ec400e476 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -52,6 +52,8 @@ module Gitlab after_script: job[:after_script], environment: job[:environment], retry: job[:retry], + parallel: job[:parallel], + instance: job[:instance], start_in: job[:start_in] }.compact } end @@ -104,7 +106,7 @@ module Gitlab ## # Jobs # - @jobs = @ci_config.jobs + @jobs = Ci::Config::Normalizer.new(@ci_config.jobs).normalize_jobs @jobs.each do |name, job| # logical validation for job diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index e68ae60ff98..5772727e855 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -44,10 +44,26 @@ module Gitlab @project ||= Project.find_by_full_path(project_path) end + def metrics_params + super.merge(includes_patches: patch_attachments.any?) + end + private + def build_merge_request + MergeRequests::BuildService.new(project, author, merge_request_params).execute + end + def create_merge_request - merge_request = MergeRequests::BuildService.new(project, author, merge_request_params).execute + merge_request = build_merge_request + + if patch_attachments.any? + apply_patches_to_source_branch(start_branch: merge_request.target_branch) + remove_patch_attachments + # Rebuild the merge request as the source branch might just have + # been created, so we should re-validate. + merge_request = build_merge_request + end if merge_request.errors.any? merge_request @@ -59,12 +75,42 @@ module Gitlab def merge_request_params params = { source_project_id: project.id, - source_branch: mail.subject, + source_branch: source_branch, target_project_id: project.id } params[:description] = message if message.present? params end + + def apply_patches_to_source_branch(start_branch:) + patches = patch_attachments.map { |patch| patch.body.decoded } + + result = Commits::CommitPatchService + .new(project, author, branch_name: source_branch, patches: patches, start_branch: start_branch) + .execute + + if result[:status] != :success + message = "Could not apply patches to #{source_branch}:\n#{result[:message]}" + raise InvalidAttachment, message + end + end + + def remove_patch_attachments + patch_attachments.each { |patch| mail.parts.delete(patch) } + # reset the message, so it needs to be reporocessed when the attachments + # have been modified + @message = nil + end + + def patch_attachments + @patches ||= mail.attachments + .select { |attachment| attachment.filename.ends_with?('.patch') } + .sort_by(&:filename) + end + + def source_branch + @source_branch ||= mail.subject + end end end end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index d8c594ad0e7..3a689967a64 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -18,6 +18,7 @@ module Gitlab InvalidIssueError = Class.new(InvalidRecordError) InvalidMergeRequestError = Class.new(InvalidRecordError) UnknownIncomingEmail = Class.new(ProcessingError) + InvalidAttachment = Class.new(ProcessingError) class Receiver def initialize(raw) diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 4d89ee5a669..d6338b09e3d 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -8,7 +8,7 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: %r{\Areadme[^/]*\z}i, + readme: %r{\A(readme|index)[^/]*\z}i, changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, license: %r{\A((un)?licen[sc]e|copying)(\.[^/]+)?\z}i, contributing: %r{\Acontributing[^/]*\z}i, diff --git a/lib/gitlab/git/patches/collection.rb b/lib/gitlab/git/patches/collection.rb new file mode 100644 index 00000000000..ad6b5d32abc --- /dev/null +++ b/lib/gitlab/git/patches/collection.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class Collection + MAX_PATCH_SIZE = 2.megabytes + + def initialize(one_or_more_patches) + @patches = Array(one_or_more_patches).map do |patch_content| + Gitlab::Git::Patches::Patch.new(patch_content) + end + end + + def content + @patches.map(&:content).join("\n") + end + + def valid_size? + size < MAX_PATCH_SIZE + end + + # rubocop: disable CodeReuse/ActiveRecord + # `@patches` is not an `ActiveRecord` relation, but an `Enumerable` + # We're using sum from `ActiveSupport` + def size + @size ||= @patches.sum(&:size) + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/git/patches/commit_patches.rb b/lib/gitlab/git/patches/commit_patches.rb new file mode 100644 index 00000000000..c62994432d3 --- /dev/null +++ b/lib/gitlab/git/patches/commit_patches.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class CommitPatches + include Gitlab::Git::WrapsGitalyErrors + + def initialize(user, repository, branch, patch_collection) + @user, @repository, @branch, @patches = user, repository, branch, patch_collection + end + + def commit + repository.with_cache_hooks do + wrapped_gitaly_errors do + operation_service.user_commit_patches(user, branch, patches.content) + end + end + end + + private + + attr_reader :user, :repository, :branch, :patches + + def operation_service + repository.raw.gitaly_operation_client + end + end + end + end +end diff --git a/lib/gitlab/git/patches/patch.rb b/lib/gitlab/git/patches/patch.rb new file mode 100644 index 00000000000..fe6ae1b5b00 --- /dev/null +++ b/lib/gitlab/git/patches/patch.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Patches + class Patch + attr_reader :content + + def initialize(content) + @content = content + end + + def size + content.bytesize + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index fcc92341c40..1642c4c5687 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -571,6 +571,20 @@ module Gitlab end end + def update_submodule(user:, submodule:, commit_sha:, message:, branch:) + args = { + user: user, + submodule: submodule, + commit_sha: commit_sha, + branch: branch, + message: message + } + + wrapped_gitaly_errors do + gitaly_operation_client.user_update_submodule(args) + end + end + # Delete the specified branch from the repository def delete_branch(branch_name) wrapped_gitaly_errors do @@ -720,6 +734,26 @@ module Gitlab end end + # Fetch remote for repository + # + # remote - remote name + # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication + # forced - should we use --force flag? + # no_tags - should we use --no-tags flag? + # prune - should we use --prune flag? + def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + wrapped_gitaly_errors do + gitaly_repository_client.fetch_remote( + remote, + ssh_auth: ssh_auth, + forced: forced, + no_tags: no_tags, + prune: prune, + timeout: GITLAB_PROJECTS_TIMEOUT + ) + end + end + def blob_at(sha, path) Gitlab::Git::Blob.find(self, sha, path) unless Gitlab::Git.blank_ref?(sha) end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 0f148614b20..4c78b790ce5 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -230,6 +230,32 @@ module Gitlab response.squash_sha end + def user_update_submodule(user:, submodule:, commit_sha:, branch:, message:) + request = Gitaly::UserUpdateSubmoduleRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + commit_sha: commit_sha, + branch: encode_binary(branch), + submodule: encode_binary(submodule), + commit_message: encode_binary(message) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_update_submodule, + request + ) + + if response.pre_receive_error.present? + raise Gitlab::Git::PreReceiveError, response.pre_receive_error + elsif response.commit_error.present? + raise Gitlab::Git::CommitError, response.commit_error + else + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + end + def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, start_branch_name, start_repository) @@ -273,6 +299,29 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) end + def user_commit_patches(user, branch_name, patches) + header = Gitaly::UserApplyPatchRequest::Header.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + target_branch: encode_binary(branch_name) + ) + reader = binary_stringio(patches) + + chunks = Enumerator.new do |chunk| + chunk.yield Gitaly::UserApplyPatchRequest.new(header: header) + + until reader.eof? + patch_chunk = reader.read(MAX_MSG_SIZE) + + chunk.yield(Gitaly::UserApplyPatchRequest.new(patches: patch_chunk)) + end + end + + response = GitalyClient.call(@repository.storage, :operation_service, :user_apply_patch, chunks) + + Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + end + private def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2bed470514b..9790818ecaf 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -92,6 +92,7 @@ excluded_attributes: - :path - :namespace_id - :creator_id + - :pool_repository_id - :import_url - :import_status - :avatar diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 4a1bdf34c3e..1cd4f9e17b7 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -2,6 +2,7 @@ module Gitlab module Kubernetes module Helm HELM_VERSION = '2.7.2'.freeze + KUBECTL_VERSION = '1.11.0'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze SERVICE_ACCOUNT = 'tiller'.freeze CLUSTER_ROLE_BINDING = 'tiller-admin'.freeze diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index 6752f2cff43..008cba9d33c 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -11,12 +11,6 @@ module Gitlab def generate_script <<~HEREDOC set -eo pipefail - ALPINE_VERSION=$(cat /etc/alpine-release | cut -d '.' -f 1,2) - echo http://mirror.clarkson.edu/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories - echo http://mirror1.hs-esslingen.de/pub/Mirrors/alpine/v$ALPINE_VERSION/main >> /etc/apk/repositories - apk add -U wget ca-certificates openssl >/dev/null - wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null - mv /tmp/linux-amd64/helm /usr/bin/ HEREDOC end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 1be7924d6ac..55add06bdb4 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -4,22 +4,27 @@ module Gitlab class InstallCommand include BaseCommand - attr_reader :name, :files, :chart, :version, :repository + attr_reader :name, :files, :chart, :version, :repository, :preinstall, :postinstall - def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil) + def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil, preinstall: nil, postinstall: nil) @name = name @chart = chart @version = version @rbac = rbac @files = files @repository = repository + @preinstall = preinstall + @postinstall = postinstall end def generate_script super + [ init_command, repository_command, - script_command + repository_update_command, + preinstall_command, + install_command, + postinstall_command ].compact.join("\n") end @@ -37,12 +42,24 @@ module Gitlab ['helm', 'repo', 'add', name, repository].shelljoin if repository end - def script_command + def repository_update_command + 'helm repo update >/dev/null' if repository + end + + def install_command command = ['helm', 'install', chart] + install_command_flags command.shelljoin + " >/dev/null\n" end + def preinstall_command + preinstall.join("\n") if preinstall + end + + def postinstall_command + postinstall.join("\n") if postinstall + end + def install_command_flags name_flag = ['--name', name] namespace_flag = ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 95192b11c0d..e9c621d96f0 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -25,7 +25,7 @@ module Gitlab def container_specification { name: 'helm', - image: 'alpine:3.6', + image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{Gitlab::Kubernetes::Helm::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}", env: generate_pod_env(command), command: %w(/bin/sh), args: %w(-c $(COMMAND_SCRIPT)) diff --git a/lib/gitlab/kubernetes/role_binding.rb b/lib/gitlab/kubernetes/role_binding.rb index 4f3ee040bf2..cb0cb42d007 100644 --- a/lib/gitlab/kubernetes/role_binding.rb +++ b/lib/gitlab/kubernetes/role_binding.rb @@ -3,9 +3,8 @@ module Gitlab module Kubernetes class RoleBinding - attr_reader :role_name, :namespace, :service_account_name - - def initialize(role_name:, namespace:, service_account_name:) + def initialize(name:, role_name:, namespace:, service_account_name:) + @name = name @role_name = role_name @namespace = namespace @service_account_name = service_account_name @@ -21,14 +20,16 @@ module Gitlab private + attr_reader :name, :role_name, :namespace, :service_account_name + def metadata - { name: "gitlab-#{namespace}", namespace: namespace } + { name: name, namespace: namespace } end def role_ref { apiGroup: 'rbac.authorization.k8s.io', - kind: 'Role', + kind: 'ClusterRole', name: role_name } end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index 142b7d1a472..d419fa66e57 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -4,10 +4,11 @@ module Gitlab module MarkupHelper extend self - MARKDOWN_EXTENSIONS = %w(mdown mkd mkdn md markdown).freeze - ASCIIDOC_EXTENSIONS = %w(adoc ad asciidoc).freeze - OTHER_EXTENSIONS = %w(textile rdoc org creole wiki mediawiki rst).freeze + MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown].freeze + ASCIIDOC_EXTENSIONS = %w[adoc ad asciidoc].freeze + OTHER_EXTENSIONS = %w[textile rdoc org creole wiki mediawiki rst].freeze EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS + PLAIN_FILENAMES = %w[readme index].freeze # Public: Determines if a given filename is compatible with GitHub::Markup. # @@ -43,7 +44,7 @@ module Gitlab # # Returns boolean def plain?(filename) - extension(filename) == 'txt' || filename.casecmp('readme').zero? + extension(filename) == 'txt' || plain_filename?(filename) end def previewable?(filename) @@ -55,5 +56,9 @@ module Gitlab def extension(filename) File.extname(filename).downcase.delete('.') end + + def plain_filename?(filename) + PLAIN_FILENAMES.include?(filename.downcase) + end end end diff --git a/lib/gitlab/private_commit_email.rb b/lib/gitlab/private_commit_email.rb new file mode 100644 index 00000000000..bade2248ccd --- /dev/null +++ b/lib/gitlab/private_commit_email.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module PrivateCommitEmail + TOKEN = "_private".freeze + + class << self + def regex + hostname_regexp = Regexp.escape(Gitlab::CurrentSettings.current_application_settings.commit_email_hostname) + + /\A(?<id>([0-9]+))\-([^@]+)@#{hostname_regexp}\z/ + end + + def user_id_for_email(email) + match = email&.match(regex) + return unless match + + match[:id].to_i + end + + def for_user(user) + hostname = Gitlab::CurrentSettings.current_application_settings.commit_email_hostname + + "#{user.id}-#{user.username}@#{hostname}" + end + end + end +end diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 30c6806b68e..59f8dd889aa 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -29,7 +29,7 @@ module Gitlab # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" # ``` - def extract_commands(content) + def extract_commands(content, only: nil) return [content, []] unless content content = content.dup @@ -37,7 +37,7 @@ module Gitlab commands = [] content.delete!("\r") - content.gsub!(commands_regex) do + content.gsub!(commands_regex(only: only)) do if $~[:cmd] commands << [$~[:cmd].downcase, $~[:arg]].reject(&:blank?) '' @@ -60,8 +60,8 @@ module Gitlab # It looks something like: # # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ - def commands_regex - names = command_names.map(&:to_s) + def commands_regex(only:) + names = command_names(limit_to_commands: only).map(&:to_s) @commands_regex ||= %r{ (?<code> @@ -133,10 +133,14 @@ module Gitlab [content, commands] end - def command_names + def command_names(limit_to_commands:) command_definitions.flat_map do |command| next if command.noop? + if limit_to_commands && (command.all_names & limit_to_commands).empty? + next + end + command.all_names end.compact end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 16c1edb2f11..c6a6fb9b5ce 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -108,23 +108,6 @@ module Gitlab success end - # Fetch remote for repository - # - # repository - an instance of Git::Repository - # remote - remote name - # ssh_auth - SSH known_hosts data and a private key to use for public-key authentication - # forced - should we use --force flag? - # no_tags - should we use --no-tags flag? - # - # Ex. - # fetch_remote(my_repo, "upstream") - # - def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) - wrapped_gitaly_errors do - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) - end - end - # Move repository reroutes to mv_directory which is an alias for # mv_namespace. Given the underlying implementation is a move action, # indescriminate of what the folders might be. diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index cc0817bdcd2..069cd1f802a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -62,6 +62,7 @@ module Gitlab clusters_applications_ingress: count(::Clusters::Applications::Ingress.installed), clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.installed), clusters_applications_runner: count(::Clusters::Applications::Runner.installed), + clusters_applications_knative: count(::Clusters::Applications::Knative.installed), in_review_folder: count(::Environment.in_review_folder), groups: count(Group), issues: count(Issue), @@ -126,7 +127,6 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def services_usage types = { - JiraService: :projects_jira_active, SlackService: :projects_slack_notifications_active, SlackSlashCommandsService: :projects_slack_slash_active, PrometheusService: :projects_prometheus_active @@ -134,6 +134,23 @@ module Gitlab results = count(Service.unscoped.where(type: types.keys, active: true).group(:type), fallback: Hash.new(-1)) types.each_with_object({}) { |(klass, key), response| response[key] = results[klass.to_s] || 0 } + .merge(jira_usage) + end + + def jira_usage + # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999 + # so we can just check for subdomains of atlassian.net + services = count( + Service.unscoped.where(type: :JiraService, active: true) + .group("CASE WHEN properties LIKE '%.atlassian.net%' THEN 'cloud' ELSE 'server' END"), + fallback: Hash.new(-1) + ) + + { + projects_jira_server_active: services['server'] || 0, + projects_jira_cloud_active: services['cloud'] || 0, + projects_jira_active: services['server'] == -1 ? -1 : services.values.sum + } end def count(relation, fallback: -1) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 2c92458f777..9e59137a2c0 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -16,6 +16,11 @@ module Gitlab str.force_encoding(Encoding::UTF_8) end + # Append path to host, making sure there's one single / in between + def append_path(host, path) + "#{host.to_s.sub(%r{\/+$}, '')}/#{path.to_s.sub(%r{^\/+}, '')}" + end + # A slugified version of the string, suitable for inclusion in URLs and # domain names. Rules: # |